From 7af2edec1e0e8b3d6d8db8d3d05b477d541015ed Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 14 Sep 2026 13:53:28 +0000 Subject: [PATCH 01/51] Optimize packed code extraction --- src/packed_string/impls_for_access.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/packed_string/impls_for_access.rs b/src/packed_string/impls_for_access.rs index 3ee54d3..714136c 100644 --- a/src/packed_string/impls_for_access.rs +++ b/src/packed_string/impls_for_access.rs @@ -48,8 +48,18 @@ where #[inline] fn code_at(&self, index: usize) -> u8 { - let start = index * usize::from(BITS); - (self.bits.get_chunk(start) & u64::from(code_mask::())) as u8 + let bits = usize::from(BITS); + let bit_index = index * bits; + let word_index = bit_index / 64; + let offset = bit_index % 64; + let words = self.bits.words(); + + let mut code = words[word_index] >> offset; + if 64 % bits != 0 && offset + bits > 64 { + code |= words[word_index + 1] << (64 - offset); + } + + (code & u64::from(code_mask::())) as u8 } } From bbf04d9e9b389d4d13c7257abf933ef1c424f52a Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 16 Sep 2026 13:02:59 +0000 Subject: [PATCH 02/51] Optimize packed view access --- src/packed_str/impls_for_access.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs index 613e80a..84e88ca 100644 --- a/src/packed_str/impls_for_access.rs +++ b/src/packed_str/impls_for_access.rs @@ -17,11 +17,8 @@ where return None; } Some( - C::from_code( - (self.bits.get_chunk(index * usize::from(BITS)) - & u64::from(crate::code_mask::())) as u8, - ) - .expect("PackedChar rejected a code in a PackedStr invariant"), + C::from_code(self.code_at(index)) + .expect("PackedChar rejected a code in a PackedStr invariant"), ) } @@ -34,4 +31,20 @@ where .checked_sub(1) .and_then(|index| self.get(index)) } + + #[inline] + fn code_at(&self, index: usize) -> u8 { + let bits = usize::from(BITS); + let bit_index = self.bits.start() + index * bits; + let word_index = bit_index / 64; + let offset = bit_index % 64; + let words = self.bits.source().words(); + + let mut code = words[word_index] >> offset; + if 64 % bits != 0 && offset + bits > 64 { + code |= words[word_index + 1] << (64 - offset); + } + + (code & u64::from(crate::code_mask::())) as u8 + } } From 151e8a55a32b7f492f7d708b627d2c76b8f963bb Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 16 Sep 2026 13:07:47 +0000 Subject: [PATCH 03/51] Share packed code extraction --- src/funcs_for_packed.rs | 16 ++++++++++++++++ src/packed_str/impls_for_access.rs | 14 ++------------ src/packed_string/impls_for_access.rs | 14 ++------------ 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/funcs_for_packed.rs b/src/funcs_for_packed.rs index 65464f9..d4fea12 100644 --- a/src/funcs_for_packed.rs +++ b/src/funcs_for_packed.rs @@ -16,5 +16,21 @@ pub(crate) const fn code_mask() -> u8 { } } +/// Extracts one fixed-width packed code from a word-aligned or offset view. +#[inline] +pub(crate) fn extract_code(words: &[u64], bit_start: usize, index: usize) -> u8 { + let bits = usize::from(BITS); + let bit_index = bit_start + index * bits; + let word_index = bit_index / 64; + let offset = bit_index % 64; + + let mut code = words[word_index] >> offset; + if 64 % bits != 0 && offset + bits > 64 { + code |= words[word_index + 1] << (64 - offset); + } + + (code & u64::from(code_mask::())) as u8 +} + #[cfg(test)] mod tests_for_packed; diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs index 84e88ca..16bb2bd 100644 --- a/src/packed_str/impls_for_access.rs +++ b/src/packed_str/impls_for_access.rs @@ -1,4 +1,5 @@ use super::*; +use crate::extract_code; impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> where @@ -34,17 +35,6 @@ where #[inline] fn code_at(&self, index: usize) -> u8 { - let bits = usize::from(BITS); - let bit_index = self.bits.start() + index * bits; - let word_index = bit_index / 64; - let offset = bit_index % 64; - let words = self.bits.source().words(); - - let mut code = words[word_index] >> offset; - if 64 % bits != 0 && offset + bits > 64 { - code |= words[word_index + 1] << (64 - offset); - } - - (code & u64::from(crate::code_mask::())) as u8 + extract_code::(self.bits.source().words(), self.bits.start(), index) } } diff --git a/src/packed_string/impls_for_access.rs b/src/packed_string/impls_for_access.rs index 714136c..9f6d241 100644 --- a/src/packed_string/impls_for_access.rs +++ b/src/packed_string/impls_for_access.rs @@ -1,4 +1,5 @@ use super::*; +use crate::extract_code; impl PackedString where @@ -48,18 +49,7 @@ where #[inline] fn code_at(&self, index: usize) -> u8 { - let bits = usize::from(BITS); - let bit_index = index * bits; - let word_index = bit_index / 64; - let offset = bit_index % 64; - let words = self.bits.words(); - - let mut code = words[word_index] >> offset; - if 64 % bits != 0 && offset + bits > 64 { - code |= words[word_index + 1] << (64 - offset); - } - - (code & u64::from(code_mask::())) as u8 + extract_code::(self.bits.words(), 0, index) } } From 215e980af00ee76e839abdfbabe422db670d3933 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 16 Sep 2026 14:17:13 +0000 Subject: [PATCH 04/51] Optimize packed construction --- .../impls_for_editing/impls_for_push_pop.rs | 46 +++++++++++++++ src/packed_string/funcs_for_code.rs | 47 +++++++++++++++ src/packed_string/impls_for_construction.rs | 5 +- .../tests_for_construction.rs | 58 ++++++++++++++++++- src/packed_string/impls_for_editing.rs | 8 ++- 5 files changed, 159 insertions(+), 5 deletions(-) diff --git a/src/bit_string/impls_for_editing/impls_for_push_pop.rs b/src/bit_string/impls_for_editing/impls_for_push_pop.rs index 7745e27..1db1061 100644 --- a/src/bit_string/impls_for_editing/impls_for_push_pop.rs +++ b/src/bit_string/impls_for_editing/impls_for_push_pop.rs @@ -1,9 +1,55 @@ use crate::funcs_for_bits::*; use crate::traits::*; +use crate::{WORD_BITS, low_mask}; use super::*; impl BitString { + /// Reserves backing storage for an additional number of bits. + #[inline] + pub(crate) fn reserve_bits(&mut self, additional_bits: usize) { + if let Some(total_bits) = self.bit_len.checked_add(additional_bits) { + let additional_words = word_len(total_bits).saturating_sub(self.words.len()); + self.words.reserve(additional_words); + } + } + + /// Appends the low `bits` bits of `value` in one or two word writes. + #[inline] + pub(crate) fn push_bits(&mut self, value: u64, bits: usize) { + assert!( + bits <= WORD_BITS, + "cannot append more than one word of bits" + ); + if bits == 0 { + return; + } + + let new_bit_len = self + .bit_len + .checked_add(bits) + .expect("bit string length overflow"); + let word_index = self.bit_len / WORD_BITS; + let bit_offset = self.bit_len % WORD_BITS; + + if word_index == self.words.len() { + self.words.push(0); + } + + let value = value & low_mask(bits); + self.words[word_index] |= value << bit_offset; + + let next_offset = bit_offset + bits; + if next_offset > WORD_BITS { + if word_index + 1 == self.words.len() { + self.words.push(0); + } + self.words[word_index + 1] |= value >> (WORD_BITS - bit_offset); + } + + self.bit_len = new_bit_len; + } + /// Appends a single bit to the end. /// /// Panics if the bit string length would overflow `usize`. diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 04d47bb..032ff98 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -1,5 +1,52 @@ +use alloc::vec::Vec; + +use crate::{BitString, WORD_BITS, code_mask, word_len}; + use super::*; +pub(super) fn pack_codes(chars: I) -> BitString +where + C: PackedChar, + I: IntoIterator, +{ + let width = usize::from(BITS); + let value_mask = u64::from(code_mask::()); + let chars = chars.into_iter(); + let (lower_bound, _) = chars.size_hint(); + let mut words = Vec::new(); + if let Some(bit_len) = lower_bound.checked_mul(width) { + words.reserve(word_len(bit_len)); + } + + let mut bit_len = 0_usize; + for character in chars { + let new_bit_len = bit_len + .checked_add(width) + .expect("packed string length overflow"); + let word_index = bit_len / WORD_BITS; + let bit_offset = bit_len % WORD_BITS; + + if word_index == words.len() { + words.push(0); + } + + let code = u64::from(character.code()) & value_mask; + words[word_index] |= code << bit_offset; + + let next_offset = bit_offset + width; + if next_offset > WORD_BITS { + if word_index + 1 == words.len() { + words.push(0); + } + words[word_index + 1] |= code >> (WORD_BITS - bit_offset); + } + + bit_len = new_bit_len; + } + + BitString::from_words(&words, bit_len).expect("packed words have an invalid length") +} + #[inline] pub(super) fn write_code(bits: &mut BitString, position: usize, code: u8) { let start = position * usize::from(BITS); diff --git a/src/packed_string/impls_for_construction.rs b/src/packed_string/impls_for_construction.rs index df209f8..9371a58 100644 --- a/src/packed_string/impls_for_construction.rs +++ b/src/packed_string/impls_for_construction.rs @@ -1,4 +1,5 @@ use super::*; +use crate::packed_string::funcs_for_code::pack_codes; impl PackedString where @@ -18,9 +19,7 @@ where where I: IntoIterator, { - let mut result = Self::new(); - result.extend(chars); - result + Self::from_valid_bits(pack_codes::(chars)) } } diff --git a/src/packed_string/impls_for_construction/tests_for_construction.rs b/src/packed_string/impls_for_construction/tests_for_construction.rs index d39fedb..ad3b894 100644 --- a/src/packed_string/impls_for_construction/tests_for_construction.rs +++ b/src/packed_string/impls_for_construction/tests_for_construction.rs @@ -1,7 +1,20 @@ #[cfg(debug_assertions)] use crate::BitString; -use crate::PackedString; use crate::packed_string::tests_for_support::{Letter, LetterString}; +use crate::{PackedString, traits::PackedChar}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} #[test] fn enum_discriminants_are_stored_directly() { @@ -18,6 +31,49 @@ fn collect_constructs_a_packed_string() { assert_eq!(value.get(0), Some(Letter::A)); } +#[test] +fn construction_and_extend_preserve_values_across_word_boundaries() { + fn check() { + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let initial = (0..7) + .map(|index| Code(((index * 3 + 1) as u8) & mask)) + .collect::>(); + let suffix = (7..30) + .map(|index| Code(((index * 5 + 2) as u8) & mask)) + .collect::>(); + let expected = initial + .iter() + .chain(&suffix) + .map(|code| code.0) + .collect::>(); + + let mut value = PackedString::::from_chars(initial.iter().copied()); + value.extend(suffix.iter().copied()); + + assert_eq!(value.bits().bit_len(), expected.len() * usize::from(BITS)); + assert_eq!( + value + .iter() + .map(|code| code.0) + .collect::>(), + expected + ); + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} + #[test] fn trusted_constructor_adopts_aligned_payload_unchanged() { let source = LetterString::from_chars([Letter::A, Letter::D, Letter::B, Letter::C]); diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 000f4f1..2ed7592 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -46,8 +46,14 @@ where C: PackedChar, { fn extend>(&mut self, iter: I) { + let iter = iter.into_iter(); + let width = usize::from(BITS); + let (lower_bound, _) = iter.size_hint(); + if let Some(additional_bits) = lower_bound.checked_mul(width) { + self.bits.reserve_bits(additional_bits); + } for character in iter { - self.push(character); + self.bits.push_bits(u64::from(character.code()), width); } } } From 2ec5fcf49b9ae911ef64bddc6aafeeacbf213d29 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 07:42:03 +0000 Subject: [PATCH 05/51] Adopt owned bit-string words --- .../impls_for_from_words.rs | 20 +++++++++++++++++ .../tests_for_from_words.rs | 22 +++++++++++++++++++ src/packed_string/funcs_for_code.rs | 2 +- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/bit_string/impls_for_construction/impls_for_from_words.rs b/src/bit_string/impls_for_construction/impls_for_from_words.rs index 9ecb30d..1467e18 100644 --- a/src/bit_string/impls_for_construction/impls_for_from_words.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_words.rs @@ -1,3 +1,5 @@ +use alloc::vec::Vec; + use super::*; use crate::funcs_for_bits::*; use crate::traits::*; @@ -22,6 +24,24 @@ impl BitString { bit_len: len, }) } + + /// Constructs a bit string by taking ownership of packed little-endian words. + /// + /// The input must contain exactly enough words for `len`. Unused high bits + /// in the last word are masked out before the vector is adopted. + #[inline] + pub(crate) fn from_owned_words(mut words: Vec, len: usize) -> Option { + if words.len() != word_len(len) { + return None; + } + + words.mask_unused_bits(len); + + Some(Self { + words, + bit_len: len, + }) + } } #[cfg(test)] diff --git a/src/bit_string/impls_for_construction/impls_for_from_words/tests_for_from_words.rs b/src/bit_string/impls_for_construction/impls_for_from_words/tests_for_from_words.rs index 2c56314..dd9672e 100644 --- a/src/bit_string/impls_for_construction/impls_for_from_words/tests_for_from_words.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_words/tests_for_from_words.rs @@ -1,4 +1,5 @@ use alloc::string::ToString; +use alloc::vec; use crate::BitString; @@ -12,6 +13,14 @@ fn constructs_empty_from_empty_words() { assert_eq!(bits.to_string(), ""); } +#[test] +fn constructs_from_owned_words() { + let bits = BitString::from_owned_words(vec![0b1011], 4).unwrap(); + + assert_eq!(bits.bit_len(), 4); + assert_eq!(bits.to_string(), "1101"); +} + #[test] fn constructs_single_partial_word() { let bits = BitString::from_words(&[0b1011], 4).unwrap(); @@ -31,6 +40,14 @@ fn masks_unused_high_bits_in_last_word() { assert!(bits.is_all_ones()); } +#[test] +fn masks_unused_high_bits_in_owned_last_word() { + let bits = BitString::from_owned_words(vec![u64::MAX], 3).unwrap(); + + assert_eq!(bits.words(), &[0b111]); + assert_eq!(bits.count_ones(), 3); +} + #[test] fn constructs_word_aligned_bits() { let bits = BitString::from_words(&[u64::MAX], 64).unwrap(); @@ -56,6 +73,8 @@ fn constructs_across_multiple_words() { fn rejects_too_few_words() { assert!(BitString::from_words(&[], 1).is_none()); assert!(BitString::from_words(&[0], 65).is_none()); + assert!(BitString::from_owned_words(vec![], 1).is_none()); + assert!(BitString::from_owned_words(vec![0], 65).is_none()); } #[test] @@ -63,4 +82,7 @@ fn rejects_too_many_words() { assert!(BitString::from_words(&[0], 0).is_none()); assert!(BitString::from_words(&[0, 0], 64).is_none()); assert!(BitString::from_words(&[0, 0, 0], 65).is_none()); + assert!(BitString::from_owned_words(vec![0], 0).is_none()); + assert!(BitString::from_owned_words(vec![0, 0], 64).is_none()); + assert!(BitString::from_owned_words(vec![0, 0, 0], 65).is_none()); } diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 032ff98..4fd74d3 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -44,7 +44,7 @@ where bit_len = new_bit_len; } - BitString::from_words(&words, bit_len).expect("packed words have an invalid length") + BitString::from_owned_words(words, bit_len).expect("packed words have an invalid length") } #[inline] From 09a7759de456b8fdb5f2093f5c47a63f41ede290 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 07:48:29 +0000 Subject: [PATCH 06/51] Add packed word block backend --- src/packed_string/funcs_for_code.rs | 78 +++++++++++++++++++ .../tests_for_construction.rs | 35 +++++++++ src/traits.rs | 2 + src/traits/words_pack.rs | 13 ++++ .../words_pack/funcs_for_pack_codes_core.rs | 40 ++++++++++ .../funcs_for_pack_codes_core/scalar.rs | 28 +++++++ .../tests_for_backend_equivalence.rs | 58 ++++++++++++++ src/traits/words_pack/impls_for_u64_slice.rs | 9 +++ 8 files changed, 263 insertions(+) create mode 100644 src/traits/words_pack.rs create mode 100644 src/traits/words_pack/funcs_for_pack_codes_core.rs create mode 100644 src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs create mode 100644 src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs create mode 100644 src/traits/words_pack/impls_for_u64_slice.rs diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 4fd74d3..a56ac18 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -1,10 +1,23 @@ use alloc::vec::Vec; +use crate::traits::{WordsPack, layout_block_len}; use crate::{BitString, WORD_BITS, code_mask, word_len}; use super::*; pub(super) fn pack_codes(chars: I) -> BitString +where + C: PackedChar, + I: IntoIterator, +{ + if BITS == 8 { + return pack_codes_in_blocks::(chars); + } + + pack_codes_scalar::(chars) +} + +fn pack_codes_scalar(chars: I) -> BitString where C: PackedChar, I: IntoIterator, @@ -47,6 +60,71 @@ where BitString::from_owned_words(words, bit_len).expect("packed words have an invalid length") } +fn pack_codes_in_blocks(chars: I) -> BitString +where + C: PackedChar, + I: IntoIterator, +{ + let width = usize::from(BITS); + let value_mask = u64::from(code_mask::()); + let chars = chars.into_iter(); + let (lower_bound, _) = chars.size_hint(); + let mut words = Vec::new(); + if let Some(bit_len) = lower_bound.checked_mul(width) { + words.reserve(word_len(bit_len)); + } + + let block_len = layout_block_len::(); + let block_word_len = block_len * width / WORD_BITS; + let mut block = [0u8; 64]; + let mut block_count = 0; + + for character in chars { + block[block_count] = character.code() & value_mask as u8; + block_count += 1; + + if block_count == block_len { + let start = words.len(); + words.resize(start + block_word_len, 0); + words[start..].pack_codes::(&block[..block_len]); + block_count = 0; + } + } + + let mut bit_len = words.len() * WORD_BITS; + for &code in &block[..block_count] { + append_code::(&mut words, &mut bit_len, u64::from(code)); + } + + BitString::from_owned_words(words, bit_len).expect("packed words have an invalid length") +} + +#[inline] +fn append_code(words: &mut Vec, bit_len: &mut usize, code: u64) { + let width = usize::from(BITS); + let new_bit_len = bit_len + .checked_add(width) + .expect("packed string length overflow"); + let word_index = *bit_len / WORD_BITS; + let bit_offset = *bit_len % WORD_BITS; + + if word_index == words.len() { + words.push(0); + } + + words[word_index] |= code << bit_offset; + + let next_offset = bit_offset + width; + if next_offset > WORD_BITS { + if word_index + 1 == words.len() { + words.push(0); + } + words[word_index + 1] |= code >> (WORD_BITS - bit_offset); + } + + *bit_len = new_bit_len; +} + #[inline] pub(super) fn write_code(bits: &mut BitString, position: usize, code: u8) { let start = position * usize::from(BITS); diff --git a/src/packed_string/impls_for_construction/tests_for_construction.rs b/src/packed_string/impls_for_construction/tests_for_construction.rs index ad3b894..6702868 100644 --- a/src/packed_string/impls_for_construction/tests_for_construction.rs +++ b/src/packed_string/impls_for_construction/tests_for_construction.rs @@ -1,6 +1,7 @@ #[cfg(debug_assertions)] use crate::BitString; use crate::packed_string::tests_for_support::{Letter, LetterString}; +use crate::traits::layout_block_len; use crate::{PackedString, traits::PackedChar}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -74,6 +75,40 @@ fn construction_and_extend_preserve_values_across_word_boundaries() { check::<8>(); } +#[test] +fn construction_preserves_values_across_complete_pack_blocks() { + fn check() { + let block_len = layout_block_len::(); + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let expected = (0..(block_len * 2 + 3)) + .map(|index| ((index * 11 + 5) as u8) & mask) + .collect::>(); + let input = expected.iter().copied().map(Code); + let value = PackedString::::from_chars(input); + + assert_eq!( + value + .iter() + .map(|code| code.0) + .collect::>(), + expected + ); + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} + #[test] fn trusted_constructor_adopts_aligned_payload_unchanged() { let source = LetterString::from_chars([Letter::A, Letter::D, Letter::B, Letter::C]); diff --git a/src/traits.rs b/src/traits.rs index c682bf3..992ac06 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -5,6 +5,7 @@ pub(crate) mod words_edit; pub(crate) mod words_eq; pub(crate) mod words_find; pub(crate) mod words_ord; +pub(crate) mod words_pack; pub(crate) mod words_scan; pub use packed_char::PackedChar; @@ -14,4 +15,5 @@ pub(crate) use words_edit::*; pub(crate) use words_eq::*; pub(crate) use words_find::*; pub(crate) use words_ord::*; +pub(crate) use words_pack::*; pub(crate) use words_scan::*; diff --git a/src/traits/words_pack.rs b/src/traits/words_pack.rs new file mode 100644 index 0000000..c7082ed --- /dev/null +++ b/src/traits/words_pack.rs @@ -0,0 +1,13 @@ +/// Fixed-width code packing operations on `[u64]` backing storage. +/// +/// The destination is expected to contain exactly the words produced by the +/// complete layout blocks in `codes`. Partial blocks and bit offsets are +/// handled by the packed-domain caller instead. +pub(crate) trait WordsPack { + fn pack_codes(&mut self, codes: &[u8]); +} + +pub(crate) mod funcs_for_pack_codes_core; +mod impls_for_u64_slice; + +pub(crate) use funcs_for_pack_codes_core::layout_block_len; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs new file mode 100644 index 0000000..e3c66fe --- /dev/null +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -0,0 +1,40 @@ +use crate::{assert_valid_width, word_len}; + +#[inline] +pub(crate) const fn layout_block_len() -> usize { + 64 / gcd(64, BITS as usize) +} + +#[inline] +const fn gcd(mut lhs: usize, mut rhs: usize) -> usize { + while rhs != 0 { + let remainder = lhs % rhs; + lhs = rhs; + rhs = remainder; + } + lhs +} + +#[inline] +pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { + assert_valid_width::(); + let block_len = layout_block_len::(); + let expected_words = word_len(codes.len() * usize::from(BITS)); + + assert!( + codes.len().is_multiple_of(block_len), + "code count must contain complete packed blocks" + ); + assert_eq!( + dst.len(), + expected_words, + "destination must contain exactly the packed block words" + ); + + scalar::pack_codes::(dst, codes); +} + +mod scalar; + +#[cfg(test)] +mod tests_for_backend_equivalence; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs new file mode 100644 index 0000000..fb9ec4e --- /dev/null +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -0,0 +1,28 @@ +use crate::code_mask; + +/// Packs complete layout blocks into complete little-endian words. +pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { + let width = usize::from(BITS); + let mask = u64::from(code_mask::()); + + dst.fill(0); + + let mut word_index = 0; + let mut bit_offset = 0; + for &code in codes { + let code = u64::from(code) & mask; + dst[word_index] |= code << bit_offset; + + let next_offset = bit_offset + width; + if next_offset > 64 { + dst[word_index + 1] |= code >> (64 - bit_offset); + word_index += 1; + bit_offset = next_offset - 64; + } else if next_offset == 64 { + word_index += 1; + bit_offset = 0; + } else { + bit_offset = next_offset; + } + } +} diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs new file mode 100644 index 0000000..84e8b01 --- /dev/null +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -0,0 +1,58 @@ +use alloc::vec; +use alloc::vec::Vec; + +use super::*; +use crate::traits::words_pack::funcs_for_pack_codes_core::scalar; + +fn codes(len: usize) -> Vec { + let mask = crate::code_mask::(); + (0..len) + .map(|index| ((index.wrapping_mul(29) + 7) as u8) & mask) + .collect() +} + +fn run(codes: &[u8]) -> Vec { + let words = codes.len() * usize::from(BITS) / 64; + let mut dst = vec![0; words]; + scalar::pack_codes::(&mut dst, codes); + dst +} + +#[test] +fn scalar_packs_complete_layout_blocks_for_all_widths() { + for bits in 1..=8 { + match bits { + 1 => assert_width::<1>(), + 2 => assert_width::<2>(), + 3 => assert_width::<3>(), + 4 => assert_width::<4>(), + 5 => assert_width::<5>(), + 6 => assert_width::<6>(), + 7 => assert_width::<7>(), + 8 => assert_width::<8>(), + _ => unreachable!(), + } + } +} + +fn assert_width() { + let block_len = layout_block_len::(); + for blocks in 0..=3 { + let input = codes::(blocks * block_len); + let actual = run::(&input); + + let mut expected = vec![0; actual.len()]; + let mut bit = 0; + for &code in &input { + let word = bit / 64; + let offset = bit % 64; + expected[word] |= u64::from(code) << offset; + if offset + usize::from(BITS) > 64 { + expected[word + 1] |= u64::from(code) >> (64 - offset); + } + bit += usize::from(BITS); + } + + assert_eq!(actual, expected, "BITS={BITS}, blocks={blocks}"); + } +} diff --git a/src/traits/words_pack/impls_for_u64_slice.rs b/src/traits/words_pack/impls_for_u64_slice.rs new file mode 100644 index 0000000..fef45fc --- /dev/null +++ b/src/traits/words_pack/impls_for_u64_slice.rs @@ -0,0 +1,9 @@ +use super::WordsPack; +use super::funcs_for_pack_codes_core; + +impl WordsPack for [u64] { + #[inline] + fn pack_codes(&mut self, codes: &[u8]) { + funcs_for_pack_codes_core::pack_codes::(self, codes); + } +} From b15488526aa77a5c0304c4582f79102185578eb0 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 07:50:44 +0000 Subject: [PATCH 07/51] Specialize scalar packing for eight-bit codes --- .../words_pack/funcs_for_pack_codes_core/scalar.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs index fb9ec4e..b273109 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -2,6 +2,11 @@ use crate::code_mask; /// Packs complete layout blocks into complete little-endian words. pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { + if BITS == 8 { + pack_bytes(dst, codes); + return; + } + let width = usize::from(BITS); let mask = u64::from(code_mask::()); @@ -26,3 +31,12 @@ pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { } } } + +#[inline] +fn pack_bytes(dst: &mut [u64], codes: &[u8]) { + debug_assert_eq!(codes.len(), dst.len() * 8); + + for (word, chunk) in dst.iter_mut().zip(codes.chunks_exact(8)) { + *word = u64::from_le_bytes(chunk.try_into().expect("chunk has eight bytes")); + } +} From 7c45c3c334ad165ffe4f3a733cea890304bcc273 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 07:57:10 +0000 Subject: [PATCH 08/51] Specialize scalar packing for four-bit codes --- src/packed_string/funcs_for_code.rs | 2 +- .../funcs_for_pack_codes_core/scalar.rs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index a56ac18..80e9872 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -10,7 +10,7 @@ where C: PackedChar, I: IntoIterator, { - if BITS == 8 { + if BITS == 4 || BITS == 8 { return pack_codes_in_blocks::(chars); } diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs index b273109..9752206 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -6,6 +6,10 @@ pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { pack_bytes(dst, codes); return; } + if BITS == 4 { + pack_nibbles(dst, codes); + return; + } let width = usize::from(BITS); let mask = u64::from(code_mask::()); @@ -40,3 +44,16 @@ fn pack_bytes(dst: &mut [u64], codes: &[u8]) { *word = u64::from_le_bytes(chunk.try_into().expect("chunk has eight bytes")); } } + +#[inline] +fn pack_nibbles(dst: &mut [u64], codes: &[u8]) { + debug_assert_eq!(codes.len(), dst.len() * 16); + + for (word, chunk) in dst.iter_mut().zip(codes.chunks_exact(16)) { + let mut packed = 0u64; + for (index, &code) in chunk.iter().enumerate() { + packed |= u64::from(code & 0x0f) << (index * 4); + } + *word = packed; + } +} From b5d43243dea02533220a9c9fb66dcf158bcde38e Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 07:58:44 +0000 Subject: [PATCH 09/51] Specialize scalar packing for two-bit codes --- src/packed_string/funcs_for_code.rs | 2 +- .../funcs_for_pack_codes_core/scalar.rs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 80e9872..e32e2a2 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -10,7 +10,7 @@ where C: PackedChar, I: IntoIterator, { - if BITS == 4 || BITS == 8 { + if BITS == 2 || BITS == 4 || BITS == 8 { return pack_codes_in_blocks::(chars); } diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs index 9752206..7095576 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -10,6 +10,10 @@ pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { pack_nibbles(dst, codes); return; } + if BITS == 2 { + pack_pairs(dst, codes); + return; + } let width = usize::from(BITS); let mask = u64::from(code_mask::()); @@ -57,3 +61,16 @@ fn pack_nibbles(dst: &mut [u64], codes: &[u8]) { *word = packed; } } + +#[inline] +fn pack_pairs(dst: &mut [u64], codes: &[u8]) { + debug_assert_eq!(codes.len(), dst.len() * 32); + + for (word, chunk) in dst.iter_mut().zip(codes.chunks_exact(32)) { + let mut packed = 0u64; + for (index, &code) in chunk.iter().enumerate() { + packed |= u64::from(code & 0x03) << (index * 2); + } + *word = packed; + } +} From 42d82eb963f8d9b8a2d83fd8efe48ee517659d64 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 08:05:54 +0000 Subject: [PATCH 10/51] Optimize irregular packed widths in blocks --- src/packed_string/funcs_for_code.rs | 2 +- .../funcs_for_pack_codes_core/scalar.rs | 41 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index e32e2a2..3ae1368 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -10,7 +10,7 @@ where C: PackedChar, I: IntoIterator, { - if BITS == 2 || BITS == 4 || BITS == 8 { + if matches!(BITS, 2 | 3 | 4 | 5 | 7 | 8) { return pack_codes_in_blocks::(chars); } diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs index 7095576..fd0afe3 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -14,7 +14,10 @@ pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { pack_pairs(dst, codes); return; } - + if matches!(BITS, 1 | 3 | 5 | 7) { + pack_groups::(dst, codes); + return; + } let width = usize::from(BITS); let mask = u64::from(code_mask::()); @@ -74,3 +77,39 @@ fn pack_pairs(dst: &mut [u64], codes: &[u8]) { *word = packed; } } + +#[inline] +fn pack_groups(dst: &mut [u64], codes: &[u8]) { + let width = usize::from(BITS); + let mask = u64::from(code_mask::()); + let group_bits = width * 8; + let mut accumulator = 0u64; + let mut accumulated_bits = 0; + let mut word_index = 0; + + for group in codes.chunks_exact(8) { + let mut packed_group = 0u64; + for (index, &code) in group.iter().enumerate() { + packed_group |= (u64::from(code) & mask) << (index * width); + } + + let next_bits = accumulated_bits + group_bits; + if next_bits < 64 { + accumulator |= packed_group << accumulated_bits; + accumulated_bits = next_bits; + } else if next_bits == 64 { + dst[word_index] = accumulator | (packed_group << accumulated_bits); + word_index += 1; + accumulator = 0; + accumulated_bits = 0; + } else { + dst[word_index] = accumulator | (packed_group << accumulated_bits); + word_index += 1; + accumulator = packed_group >> (64 - accumulated_bits); + accumulated_bits = next_bits - 64; + } + } + + debug_assert_eq!(accumulated_bits, 0); + debug_assert_eq!(word_index, dst.len()); +} From 50884f6fa84ae3e9c30d6558d01d0941bcff65e6 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 08:08:13 +0000 Subject: [PATCH 11/51] Specialize scalar packing for one-bit codes --- src/packed_string/funcs_for_code.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 3ae1368..d9679b4 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -10,7 +10,7 @@ where C: PackedChar, I: IntoIterator, { - if matches!(BITS, 2 | 3 | 4 | 5 | 7 | 8) { + if matches!(BITS, 1 | 2 | 3 | 4 | 5 | 7 | 8) { return pack_codes_in_blocks::(chars); } From be42264fdc3721d4809db50c455b84a78a1478f6 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 08:23:14 +0000 Subject: [PATCH 12/51] Batch packed construction before SIMD --- src/packed_string/funcs_for_code.rs | 67 ++++--------------- .../words_pack/funcs_for_pack_codes_core.rs | 1 + .../funcs_for_pack_codes_core/scalar.rs | 43 ++++++++++++ 3 files changed, 57 insertions(+), 54 deletions(-) diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index d9679b4..e378f3c 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -10,54 +10,7 @@ where C: PackedChar, I: IntoIterator, { - if matches!(BITS, 1 | 2 | 3 | 4 | 5 | 7 | 8) { - return pack_codes_in_blocks::(chars); - } - - pack_codes_scalar::(chars) -} - -fn pack_codes_scalar(chars: I) -> BitString -where - C: PackedChar, - I: IntoIterator, -{ - let width = usize::from(BITS); - let value_mask = u64::from(code_mask::()); - let chars = chars.into_iter(); - let (lower_bound, _) = chars.size_hint(); - let mut words = Vec::new(); - if let Some(bit_len) = lower_bound.checked_mul(width) { - words.reserve(word_len(bit_len)); - } - - let mut bit_len = 0_usize; - for character in chars { - let new_bit_len = bit_len - .checked_add(width) - .expect("packed string length overflow"); - let word_index = bit_len / WORD_BITS; - let bit_offset = bit_len % WORD_BITS; - - if word_index == words.len() { - words.push(0); - } - - let code = u64::from(character.code()) & value_mask; - words[word_index] |= code << bit_offset; - - let next_offset = bit_offset + width; - if next_offset > WORD_BITS { - if word_index + 1 == words.len() { - words.push(0); - } - words[word_index + 1] |= code >> (WORD_BITS - bit_offset); - } - - bit_len = new_bit_len; - } - - BitString::from_owned_words(words, bit_len).expect("packed words have an invalid length") + pack_codes_in_blocks::(chars) } fn pack_codes_in_blocks(chars: I) -> BitString @@ -74,8 +27,7 @@ where words.reserve(word_len(bit_len)); } - let block_len = layout_block_len::(); - let block_word_len = block_len * width / WORD_BITS; + let layout_block_len = layout_block_len::(); let mut block = [0u8; 64]; let mut block_count = 0; @@ -83,16 +35,23 @@ where block[block_count] = character.code() & value_mask as u8; block_count += 1; - if block_count == block_len { + if block_count == block.len() { let start = words.len(); - words.resize(start + block_word_len, 0); - words[start..].pack_codes::(&block[..block_len]); + words.resize(start + block.len() * width / WORD_BITS, 0); + words[start..].pack_codes::(&block); block_count = 0; } } + let aligned_count = block_count / layout_block_len * layout_block_len; + if aligned_count != 0 { + let start = words.len(); + words.resize(start + aligned_count * width / WORD_BITS, 0); + words[start..].pack_codes::(&block[..aligned_count]); + } + let mut bit_len = words.len() * WORD_BITS; - for &code in &block[..block_count] { + for &code in &block[aligned_count..block_count] { append_code::(&mut words, &mut bit_len, u64::from(code)); } diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index e3c66fe..ae2d1a4 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -2,6 +2,7 @@ use crate::{assert_valid_width, word_len}; #[inline] pub(crate) const fn layout_block_len() -> usize { + assert_valid_width::(); 64 / gcd(64, BITS as usize) } diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs index fd0afe3..6bcbf5f 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -14,6 +14,10 @@ pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { pack_pairs(dst, codes); return; } + if BITS == 6 { + pack_six_bits(dst, codes); + return; + } if matches!(BITS, 1 | 3 | 5 | 7) { pack_groups::(dst, codes); return; @@ -78,6 +82,45 @@ fn pack_pairs(dst: &mut [u64], codes: &[u8]) { } } +#[inline] +fn pack_six_bits(dst: &mut [u64], codes: &[u8]) { + debug_assert_eq!(codes.len() % 8, 0); + + let mut accumulator = 0u64; + let mut accumulated_bits = 0; + let mut word_index = 0; + + for group in codes.chunks_exact(8) { + let packed_group = u64::from(group[0] & 0x3f) + | (u64::from(group[1] & 0x3f) << 6) + | (u64::from(group[2] & 0x3f) << 12) + | (u64::from(group[3] & 0x3f) << 18) + | (u64::from(group[4] & 0x3f) << 24) + | (u64::from(group[5] & 0x3f) << 30) + | (u64::from(group[6] & 0x3f) << 36) + | (u64::from(group[7] & 0x3f) << 42); + let next_bits = accumulated_bits + 48; + + if next_bits < 64 { + accumulator |= packed_group << accumulated_bits; + accumulated_bits = next_bits; + } else if next_bits == 64 { + dst[word_index] = accumulator | (packed_group << accumulated_bits); + word_index += 1; + accumulator = 0; + accumulated_bits = 0; + } else { + dst[word_index] = accumulator | (packed_group << accumulated_bits); + word_index += 1; + accumulator = packed_group >> (64 - accumulated_bits); + accumulated_bits = next_bits - 64; + } + } + + debug_assert_eq!(accumulated_bits, 0); + debug_assert_eq!(word_index, dst.len()); +} + #[inline] fn pack_groups(dst: &mut [u64], codes: &[u8]) { let width = usize::from(BITS); From e9cbfb891d2366b07389c2a88e632260675b8f46 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 08:40:25 +0000 Subject: [PATCH 13/51] Prepare packed word packing backends --- src/packed_string/funcs_for_code.rs | 37 ++++----- src/traits/words_pack.rs | 24 +++++- .../words_pack/funcs_for_pack_codes_core.rs | 32 ++++---- .../funcs_for_pack_codes_core/scalar.rs | 75 ++++++------------- .../tests_for_backend_equivalence.rs | 2 +- 5 files changed, 73 insertions(+), 97 deletions(-) diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index e378f3c..e2e9956 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -1,25 +1,16 @@ use alloc::vec::Vec; use crate::traits::{WordsPack, layout_block_len}; -use crate::{BitString, WORD_BITS, code_mask, word_len}; +use crate::{BitString, WORD_BITS, word_len}; use super::*; pub(super) fn pack_codes(chars: I) -> BitString -where - C: PackedChar, - I: IntoIterator, -{ - pack_codes_in_blocks::(chars) -} - -fn pack_codes_in_blocks(chars: I) -> BitString where C: PackedChar, I: IntoIterator, { let width = usize::from(BITS); - let value_mask = u64::from(code_mask::()); let chars = chars.into_iter(); let (lower_bound, _) = chars.size_hint(); let mut words = Vec::new(); @@ -27,31 +18,33 @@ where words.reserve(word_len(bit_len)); } - let layout_block_len = layout_block_len::(); - let mut block = [0u8; 64]; - let mut block_count = 0; + let layout_len = layout_block_len::(); + const CODE_BATCH_LEN: usize = 64; + // 64 is a multiple of every layout block for BITS=1..8. + let mut batch = [0u8; CODE_BATCH_LEN]; + let mut batch_len = 0; for character in chars { - block[block_count] = character.code() & value_mask as u8; - block_count += 1; + batch[batch_len] = character.code(); + batch_len += 1; - if block_count == block.len() { + if batch_len == CODE_BATCH_LEN { let start = words.len(); - words.resize(start + block.len() * width / WORD_BITS, 0); - words[start..].pack_codes::(&block); - block_count = 0; + words.resize(start + CODE_BATCH_LEN * width / WORD_BITS, 0); + words[start..].pack_codes::(&batch); + batch_len = 0; } } - let aligned_count = block_count / layout_block_len * layout_block_len; + let aligned_count = batch_len / layout_len * layout_len; if aligned_count != 0 { let start = words.len(); words.resize(start + aligned_count * width / WORD_BITS, 0); - words[start..].pack_codes::(&block[..aligned_count]); + words[start..].pack_codes::(&batch[..aligned_count]); } let mut bit_len = words.len() * WORD_BITS; - for &code in &block[aligned_count..block_count] { + for &code in &batch[aligned_count..batch_len] { append_code::(&mut words, &mut bit_len, u64::from(code)); } diff --git a/src/traits/words_pack.rs b/src/traits/words_pack.rs index c7082ed..870816f 100644 --- a/src/traits/words_pack.rs +++ b/src/traits/words_pack.rs @@ -1,13 +1,31 @@ +use crate::assert_valid_width; + /// Fixed-width code packing operations on `[u64]` backing storage. /// /// The destination is expected to contain exactly the words produced by the /// complete layout blocks in `codes`. Partial blocks and bit offsets are -/// handled by the packed-domain caller instead. +/// handled by the packed-domain caller instead. Every code must fit in +/// `BITS` bits; callers are responsible for preserving this invariant. pub(crate) trait WordsPack { fn pack_codes(&mut self, codes: &[u8]); } +/// Returns the smallest number of codes that occupies a whole number of words. +#[inline] +pub(crate) const fn layout_block_len() -> usize { + assert_valid_width::(); + 64 / gcd(64, BITS as usize) +} + +#[inline] +const fn gcd(mut lhs: usize, mut rhs: usize) -> usize { + while rhs != 0 { + let remainder = lhs % rhs; + lhs = rhs; + rhs = remainder; + } + lhs +} + pub(crate) mod funcs_for_pack_codes_core; mod impls_for_u64_slice; - -pub(crate) use funcs_for_pack_codes_core::layout_block_len; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index ae2d1a4..77b8530 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -1,29 +1,13 @@ -use crate::{assert_valid_width, word_len}; - -#[inline] -pub(crate) const fn layout_block_len() -> usize { - assert_valid_width::(); - 64 / gcd(64, BITS as usize) -} - -#[inline] -const fn gcd(mut lhs: usize, mut rhs: usize) -> usize { - while rhs != 0 { - let remainder = lhs % rhs; - lhs = rhs; - rhs = remainder; - } - lhs -} +use crate::{assert_valid_width, code_mask, word_len}; #[inline] pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { assert_valid_width::(); - let block_len = layout_block_len::(); + let layout_len = super::layout_block_len::(); let expected_words = word_len(codes.len() * usize::from(BITS)); assert!( - codes.len().is_multiple_of(block_len), + codes.len().is_multiple_of(layout_len), "code count must contain complete packed blocks" ); assert_eq!( @@ -32,6 +16,16 @@ pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { "destination must contain exactly the packed block words" ); + debug_assert!( + codes.iter().all(|&code| code <= code_mask::()), + "packed code does not fit in BITS bits" + ); + + dispatch::(dst, codes); +} + +#[inline] +fn dispatch(dst: &mut [u64], codes: &[u8]) { scalar::pack_codes::(dst, codes); } diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs index 6bcbf5f..9dde865 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -1,5 +1,3 @@ -use crate::code_mask; - /// Packs complete layout blocks into complete little-endian words. pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { if BITS == 8 { @@ -14,23 +12,18 @@ pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { pack_pairs(dst, codes); return; } - if BITS == 6 { - pack_six_bits(dst, codes); - return; - } - if matches!(BITS, 1 | 3 | 5 | 7) { + if matches!(BITS, 1 | 3 | 5 | 6 | 7) { pack_groups::(dst, codes); return; } let width = usize::from(BITS); - let mask = u64::from(code_mask::()); dst.fill(0); let mut word_index = 0; let mut bit_offset = 0; for &code in codes { - let code = u64::from(code) & mask; + let code = u64::from(code); dst[word_index] |= code << bit_offset; let next_offset = bit_offset + width; @@ -83,24 +76,17 @@ fn pack_pairs(dst: &mut [u64], codes: &[u8]) { } #[inline] -fn pack_six_bits(dst: &mut [u64], codes: &[u8]) { - debug_assert_eq!(codes.len() % 8, 0); - +fn pack_groups(dst: &mut [u64], codes: &[u8]) { + let width = usize::from(BITS); + let group_bits = width * 8; let mut accumulator = 0u64; let mut accumulated_bits = 0; let mut word_index = 0; for group in codes.chunks_exact(8) { - let packed_group = u64::from(group[0] & 0x3f) - | (u64::from(group[1] & 0x3f) << 6) - | (u64::from(group[2] & 0x3f) << 12) - | (u64::from(group[3] & 0x3f) << 18) - | (u64::from(group[4] & 0x3f) << 24) - | (u64::from(group[5] & 0x3f) << 30) - | (u64::from(group[6] & 0x3f) << 36) - | (u64::from(group[7] & 0x3f) << 42); - let next_bits = accumulated_bits + 48; + let packed_group = pack_group::(group); + let next_bits = accumulated_bits + group_bits; if next_bits < 64 { accumulator |= packed_group << accumulated_bits; accumulated_bits = next_bits; @@ -122,37 +108,22 @@ fn pack_six_bits(dst: &mut [u64], codes: &[u8]) { } #[inline] -fn pack_groups(dst: &mut [u64], codes: &[u8]) { - let width = usize::from(BITS); - let mask = u64::from(code_mask::()); - let group_bits = width * 8; - let mut accumulator = 0u64; - let mut accumulated_bits = 0; - let mut word_index = 0; - - for group in codes.chunks_exact(8) { - let mut packed_group = 0u64; - for (index, &code) in group.iter().enumerate() { - packed_group |= (u64::from(code) & mask) << (index * width); - } - - let next_bits = accumulated_bits + group_bits; - if next_bits < 64 { - accumulator |= packed_group << accumulated_bits; - accumulated_bits = next_bits; - } else if next_bits == 64 { - dst[word_index] = accumulator | (packed_group << accumulated_bits); - word_index += 1; - accumulator = 0; - accumulated_bits = 0; - } else { - dst[word_index] = accumulator | (packed_group << accumulated_bits); - word_index += 1; - accumulator = packed_group >> (64 - accumulated_bits); - accumulated_bits = next_bits - 64; - } +fn pack_group(group: &[u8]) -> u64 { + if BITS == 6 { + return u64::from(group[0]) + | (u64::from(group[1]) << 6) + | (u64::from(group[2]) << 12) + | (u64::from(group[3]) << 18) + | (u64::from(group[4]) << 24) + | (u64::from(group[5]) << 30) + | (u64::from(group[6]) << 36) + | (u64::from(group[7]) << 42); } - debug_assert_eq!(accumulated_bits, 0); - debug_assert_eq!(word_index, dst.len()); + let width = usize::from(BITS); + let mut packed = 0u64; + for (index, &code) in group.iter().enumerate() { + packed |= u64::from(code) << (index * width); + } + packed } diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs index 84e8b01..814b57a 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -1,7 +1,7 @@ use alloc::vec; use alloc::vec::Vec; -use super::*; +use crate::traits::layout_block_len; use crate::traits::words_pack::funcs_for_pack_codes_core::scalar; fn codes(len: usize) -> Vec { From eb3b172c571586639cf2a1efef2bde83396951fa Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 19 Sep 2026 12:58:48 +0000 Subject: [PATCH 14/51] Finalize packed word packing contract --- src/packed_string/funcs_for_code.rs | 3 +- src/traits/words_pack.rs | 8 ++-- .../funcs_for_pack_codes_core/scalar.rs | 47 ++++--------------- .../tests_for_backend_equivalence.rs | 2 +- 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index e2e9956..87662ab 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -5,6 +5,8 @@ use crate::{BitString, WORD_BITS, word_len}; use super::*; +const CODE_BATCH_LEN: usize = 64; + pub(super) fn pack_codes(chars: I) -> BitString where C: PackedChar, @@ -19,7 +21,6 @@ where } let layout_len = layout_block_len::(); - const CODE_BATCH_LEN: usize = 64; // 64 is a multiple of every layout block for BITS=1..8. let mut batch = [0u8; CODE_BATCH_LEN]; let mut batch_len = 0; diff --git a/src/traits/words_pack.rs b/src/traits/words_pack.rs index 870816f..c517bfd 100644 --- a/src/traits/words_pack.rs +++ b/src/traits/words_pack.rs @@ -1,11 +1,13 @@ -use crate::assert_valid_width; +use crate::{WORD_BITS, assert_valid_width}; /// Fixed-width code packing operations on `[u64]` backing storage. /// /// The destination is expected to contain exactly the words produced by the /// complete layout blocks in `codes`. Partial blocks and bit offsets are /// handled by the packed-domain caller instead. Every code must fit in -/// `BITS` bits; callers are responsible for preserving this invariant. +/// `BITS` bits; callers are responsible for preserving this invariant. Every +/// destination word is fully overwritten, and its previous contents are +/// ignored. pub(crate) trait WordsPack { fn pack_codes(&mut self, codes: &[u8]); } @@ -14,7 +16,7 @@ pub(crate) trait WordsPack { #[inline] pub(crate) const fn layout_block_len() -> usize { assert_valid_width::(); - 64 / gcd(64, BITS as usize) + WORD_BITS / gcd(WORD_BITS, BITS as usize) } #[inline] diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs index 9dde865..bc91414 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -1,42 +1,11 @@ /// Packs complete layout blocks into complete little-endian words. pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { - if BITS == 8 { - pack_bytes(dst, codes); - return; - } - if BITS == 4 { - pack_nibbles(dst, codes); - return; - } - if BITS == 2 { - pack_pairs(dst, codes); - return; - } - if matches!(BITS, 1 | 3 | 5 | 6 | 7) { - pack_groups::(dst, codes); - return; - } - let width = usize::from(BITS); - - dst.fill(0); - - let mut word_index = 0; - let mut bit_offset = 0; - for &code in codes { - let code = u64::from(code); - dst[word_index] |= code << bit_offset; - - let next_offset = bit_offset + width; - if next_offset > 64 { - dst[word_index + 1] |= code >> (64 - bit_offset); - word_index += 1; - bit_offset = next_offset - 64; - } else if next_offset == 64 { - word_index += 1; - bit_offset = 0; - } else { - bit_offset = next_offset; - } + match BITS { + 8 => pack_bytes(dst, codes), + 4 => pack_nibbles(dst, codes), + 2 => pack_pairs(dst, codes), + 1 | 3 | 5 | 6 | 7 => pack_groups::(dst, codes), + _ => unreachable!("BITS validated by WordsPack core"), } } @@ -56,7 +25,7 @@ fn pack_nibbles(dst: &mut [u64], codes: &[u8]) { for (word, chunk) in dst.iter_mut().zip(codes.chunks_exact(16)) { let mut packed = 0u64; for (index, &code) in chunk.iter().enumerate() { - packed |= u64::from(code & 0x0f) << (index * 4); + packed |= u64::from(code) << (index * 4); } *word = packed; } @@ -69,7 +38,7 @@ fn pack_pairs(dst: &mut [u64], codes: &[u8]) { for (word, chunk) in dst.iter_mut().zip(codes.chunks_exact(32)) { let mut packed = 0u64; for (index, &code) in chunk.iter().enumerate() { - packed |= u64::from(code & 0x03) << (index * 2); + packed |= u64::from(code) << (index * 2); } *word = packed; } diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs index 814b57a..8827d8a 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -1,8 +1,8 @@ use alloc::vec; use alloc::vec::Vec; +use super::scalar; use crate::traits::layout_block_len; -use crate::traits::words_pack::funcs_for_pack_codes_core::scalar; fn codes(len: usize) -> Vec { let mask = crate::code_mask::(); From 9476d3ae192e2eab410bd285aca184e5e9ddf98f Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 12:45:09 +0000 Subject: [PATCH 15/51] Add AVX2 nibble packing backend --- .../words_pack/funcs_for_pack_codes_core.rs | 16 ++++ .../funcs_for_pack_codes_core/avx2.rs | 78 +++++++++++++++++++ .../tests_for_backend_equivalence.rs | 29 +++++++ 3 files changed, 123 insertions(+) create mode 100644 src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index 77b8530..6c2105a 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -26,10 +26,26 @@ pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { #[inline] fn dispatch(dst: &mut [u64], codes: &[u8]) { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + if BITS == 4 && codes.len() >= 32 { + // SAFETY: This branch is compiled only when AVX2 is enabled. + unsafe { avx2::pack_codes::(dst, codes) }; + return; + } + } + scalar::pack_codes::(dst, codes); } mod scalar; +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2; + #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs new file mode 100644 index 0000000..7a02206 --- /dev/null +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -0,0 +1,78 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_loadu_si256, + _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_setr_epi8, _mm256_shuffle_epi8, + _mm256_slli_epi16, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_loadu_si256, + _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_setr_epi8, _mm256_shuffle_epi8, + _mm256_slli_epi16, +}; + +const CODES_PER_SIMD_BLOCK: usize = 32; +const WORDS_PER_SIMD_BLOCK: usize = 2; + +/// Packs the BITS=4 SIMD prefix and delegates complete remaining blocks to scalar. +/// +/// # Safety +/// +/// The caller must only invoke this function when AVX2 is available. The +/// slices must satisfy the `WordsPack` contract, and `BITS` must be 4. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { + debug_assert_eq!(BITS, 4); + debug_assert_eq!(codes.len() % 16, 0); + debug_assert_eq!(dst.len(), codes.len() / 16); + + let simd_code_len = codes.len() / CODES_PER_SIMD_BLOCK * CODES_PER_SIMD_BLOCK; + let simd_word_len = simd_code_len / 16; + + for (code_chunk, word_chunk) in codes[..simd_code_len] + .chunks_exact(CODES_PER_SIMD_BLOCK) + .zip(dst[..simd_word_len].chunks_exact_mut(WORDS_PER_SIMD_BLOCK)) + { + // SAFETY: The chunk contains 32 initialized bytes, and the backend is + // compiled with AVX2 enabled. + unsafe { pack_32_codes(code_chunk, word_chunk) }; + } + + if simd_code_len < codes.len() { + scalar::pack_codes::(&mut dst[simd_word_len..], &codes[simd_code_len..]); + } +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_codes(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); + debug_assert_eq!(words.len(), WORDS_PER_SIMD_BLOCK); + + let even_indices = _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, // + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, + ); + let odd_indices = _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, // + 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let even = _mm256_shuffle_epi8(input, even_indices); + let odd = _mm256_shuffle_epi8(input, odd_indices); + let packed = _mm256_or_si256(even, _mm256_slli_epi16(odd, 4)); + let compacted = _mm256_permute4x64_epi64(packed, 0x88); + + // SAFETY: `compacted` contains two output words in its low 128 bits, and + // `words` has space for exactly those two words. + unsafe { + _mm_storeu_si128( + words.as_mut_ptr().cast::<__m128i>(), + _mm256_castsi256_si128(compacted), + ); + } +} diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs index 8827d8a..9dac93d 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -56,3 +56,32 @@ fn assert_width() { assert_eq!(actual, expected, "BITS={BITS}, blocks={blocks}"); } } + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +mod tests_for_avx2 { + use super::super::avx2; + use super::*; + + #[test] + fn bits_four_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 16, 32, 48, 64, 96, 128] { + for input in [ + vec![0; code_len], + vec![0x0f; code_len], + codes::<4>(code_len), + ] { + let expected = run::<4>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<4>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } +} From 3d6db1f6824ff6bc607584ba1999008fa47a828f Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 13:13:27 +0000 Subject: [PATCH 16/51] Add AVX2 bit-mask packing backend --- .../words_pack/funcs_for_pack_codes_core.rs | 2 +- .../funcs_for_pack_codes_core/avx2.rs | 59 ++++++++++++++----- .../tests_for_backend_equivalence.rs | 16 +++++ 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index 6c2105a..2daa762 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -31,7 +31,7 @@ fn dispatch(dst: &mut [u64], codes: &[u8]) { target_feature = "avx2" ))] { - if BITS == 4 && codes.len() >= 32 { + if matches!(BITS, 1 | 4) && codes.len() >= 32 { // SAFETY: This branch is compiled only when AVX2 is enabled. unsafe { avx2::pack_codes::(dst, codes) }; return; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs index 7a02206..6370982 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -3,42 +3,52 @@ use super::scalar; #[cfg(target_arch = "x86")] use core::arch::x86::{ __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_loadu_si256, - _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_setr_epi8, _mm256_shuffle_epi8, - _mm256_slli_epi16, + _mm256_movemask_epi8, _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_setr_epi8, + _mm256_shuffle_epi8, _mm256_slli_epi16, }; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{ __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_loadu_si256, - _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_setr_epi8, _mm256_shuffle_epi8, - _mm256_slli_epi16, + _mm256_movemask_epi8, _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_setr_epi8, + _mm256_shuffle_epi8, _mm256_slli_epi16, }; const CODES_PER_SIMD_BLOCK: usize = 32; const WORDS_PER_SIMD_BLOCK: usize = 2; -/// Packs the BITS=4 SIMD prefix and delegates complete remaining blocks to scalar. +/// Packs the BITS=1 or BITS=4 SIMD prefix and delegates complete remaining +/// blocks to scalar. /// /// # Safety /// /// The caller must only invoke this function when AVX2 is available. The -/// slices must satisfy the `WordsPack` contract, and `BITS` must be 4. +/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1 or 4. #[target_feature(enable = "avx2")] pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { - debug_assert_eq!(BITS, 4); - debug_assert_eq!(codes.len() % 16, 0); - debug_assert_eq!(dst.len(), codes.len() / 16); + debug_assert!(matches!(BITS, 1 | 4)); + let codes_per_word = 64 / usize::from(BITS); + debug_assert_eq!(codes.len() % codes_per_word, 0); + debug_assert_eq!(dst.len(), codes.len() / codes_per_word); - let simd_code_len = codes.len() / CODES_PER_SIMD_BLOCK * CODES_PER_SIMD_BLOCK; - let simd_word_len = simd_code_len / 16; + let simd_code_block_len = if BITS == 1 { 64 } else { CODES_PER_SIMD_BLOCK }; + let simd_code_len = codes.len() / simd_code_block_len * simd_code_block_len; + let simd_word_len = simd_code_len * usize::from(BITS) / 64; + let simd_word_block_len = if BITS == 1 { 1 } else { WORDS_PER_SIMD_BLOCK }; for (code_chunk, word_chunk) in codes[..simd_code_len] - .chunks_exact(CODES_PER_SIMD_BLOCK) - .zip(dst[..simd_word_len].chunks_exact_mut(WORDS_PER_SIMD_BLOCK)) + .chunks_exact(simd_code_block_len) + .zip(dst[..simd_word_len].chunks_exact_mut(simd_word_block_len)) { - // SAFETY: The chunk contains 32 initialized bytes, and the backend is - // compiled with AVX2 enabled. - unsafe { pack_32_codes(code_chunk, word_chunk) }; + // SAFETY: The selected kernel reads only initialized bytes from this + // chunk, and the backend is compiled with AVX2 enabled. + unsafe { + if BITS == 1 { + pack_64_bits(code_chunk, word_chunk); + } else { + pack_32_codes(code_chunk, word_chunk); + } + } } if simd_code_len < codes.len() { @@ -46,6 +56,23 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { } } +#[target_feature(enable = "avx2")] +unsafe fn pack_64_bits(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), 64); + debug_assert_eq!(words.len(), 1); + + // SAFETY: Both loads read exactly 32 initialized bytes. + let (low, high) = unsafe { + ( + _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()), + _mm256_loadu_si256(codes.as_ptr().add(32).cast::<__m256i>()), + ) + }; + let low = _mm256_movemask_epi8(_mm256_slli_epi16(low, 7)) as u32; + let high = _mm256_movemask_epi8(_mm256_slli_epi16(high, 7)) as u32; + words[0] = u64::from(low) | (u64::from(high) << 32); +} + #[target_feature(enable = "avx2")] unsafe fn pack_32_codes(codes: &[u8], words: &mut [u64]) { debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs index 9dac93d..b1846de 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -84,4 +84,20 @@ mod tests_for_avx2 { } } } + + #[test] + fn bits_one_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 64, 128, 192, 256] { + for input in [vec![0; code_len], vec![1; code_len], codes::<1>(code_len)] { + let expected = run::<1>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<1>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } } From 028c8c0bf7a12f087d921a244c08e33e334dd241 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 13:18:17 +0000 Subject: [PATCH 17/51] Document AVX2 packing support matrix --- src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs index 6370982..d55457c 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -1,3 +1,12 @@ +//! AVX2 packing is intentionally implemented only for `BITS=1` and `BITS=4`. +//! +//! A `BITS=2` compression prototype and a `BITS=8` byte-copy prototype were +//! benchmarked against the scalar backend without a stable construction +//! speedup. Their extra dispatch and kernel complexity therefore was not +//! retained. Other widths continue to use scalar packing until a new kernel +//! demonstrates a measured benefit; re-run the construction benchmarks before +//! revisiting this support matrix. + use super::scalar; #[cfg(target_arch = "x86")] From da342d31ad8f446f4fa597f76da67f0ae51dba49 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 13:32:04 +0000 Subject: [PATCH 18/51] Add AVX2 six-bit packing backend --- .../words_pack/funcs_for_pack_codes_core.rs | 2 +- .../funcs_for_pack_codes_core/avx2.rs | 89 +++++++++++++++---- .../tests_for_backend_equivalence.rs | 20 +++++ 3 files changed, 92 insertions(+), 19 deletions(-) diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index 2daa762..9bb8425 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -31,7 +31,7 @@ fn dispatch(dst: &mut [u64], codes: &[u8]) { target_feature = "avx2" ))] { - if matches!(BITS, 1 | 4) && codes.len() >= 32 { + if matches!(BITS, 1 | 4 | 6) && codes.len() >= 32 { // SAFETY: This branch is compiled only when AVX2 is enabled. unsafe { avx2::pack_codes::(dst, codes) }; return; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs index d55457c..1493b62 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -1,4 +1,5 @@ -//! AVX2 packing is intentionally implemented only for `BITS=1` and `BITS=4`. +//! AVX2 packing is intentionally implemented only for `BITS=1`, `BITS=4`, and +//! `BITS=6`. //! //! A `BITS=2` compression prototype and a `BITS=8` byte-copy prototype were //! benchmarked against the scalar backend without a stable construction @@ -8,42 +9,51 @@ //! revisiting this support matrix. use super::scalar; +use crate::traits::words_pack::layout_block_len; #[cfg(target_arch = "x86")] use core::arch::x86::{ - __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_loadu_si256, - _mm256_movemask_epi8, _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_setr_epi8, - _mm256_shuffle_epi8, _mm256_slli_epi16, + __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_cvtepu32_epi64, + _mm256_loadu_si256, _mm256_madd_epi16, _mm256_maddubs_epi16, _mm256_movemask_epi8, + _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_permutevar8x32_epi32, _mm256_setr_epi8, + _mm256_setr_epi16, _mm256_setr_epi32, _mm256_shuffle_epi8, _mm256_slli_epi16, + _mm256_slli_epi64, _mm256_storeu_si256, }; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{ - __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_loadu_si256, - _mm256_movemask_epi8, _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_setr_epi8, - _mm256_shuffle_epi8, _mm256_slli_epi16, + __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_cvtepu32_epi64, + _mm256_loadu_si256, _mm256_madd_epi16, _mm256_maddubs_epi16, _mm256_movemask_epi8, + _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_permutevar8x32_epi32, _mm256_setr_epi8, + _mm256_setr_epi16, _mm256_setr_epi32, _mm256_shuffle_epi8, _mm256_slli_epi16, + _mm256_slli_epi64, _mm256_storeu_si256, }; const CODES_PER_SIMD_BLOCK: usize = 32; const WORDS_PER_SIMD_BLOCK: usize = 2; -/// Packs the BITS=1 or BITS=4 SIMD prefix and delegates complete remaining +/// Packs the BITS=1, BITS=4, or BITS=6 SIMD prefix and delegates complete remaining /// blocks to scalar. /// /// # Safety /// /// The caller must only invoke this function when AVX2 is available. The -/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1 or 4. +/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1, 4, or 6. #[target_feature(enable = "avx2")] pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { - debug_assert!(matches!(BITS, 1 | 4)); - let codes_per_word = 64 / usize::from(BITS); - debug_assert_eq!(codes.len() % codes_per_word, 0); - debug_assert_eq!(dst.len(), codes.len() / codes_per_word); + debug_assert!(matches!(BITS, 1 | 4 | 6)); + debug_assert_eq!(codes.len() % layout_block_len::(), 0); + debug_assert_eq!(dst.len(), codes.len() * usize::from(BITS) / 64); let simd_code_block_len = if BITS == 1 { 64 } else { CODES_PER_SIMD_BLOCK }; let simd_code_len = codes.len() / simd_code_block_len * simd_code_block_len; let simd_word_len = simd_code_len * usize::from(BITS) / 64; - let simd_word_block_len = if BITS == 1 { 1 } else { WORDS_PER_SIMD_BLOCK }; + let simd_word_block_len = match BITS { + 1 => 1, + 4 => WORDS_PER_SIMD_BLOCK, + 6 => 3, + _ => unreachable!(), + }; for (code_chunk, word_chunk) in codes[..simd_code_len] .chunks_exact(simd_code_block_len) @@ -52,10 +62,11 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { // SAFETY: The selected kernel reads only initialized bytes from this // chunk, and the backend is compiled with AVX2 enabled. unsafe { - if BITS == 1 { - pack_64_bits(code_chunk, word_chunk); - } else { - pack_32_codes(code_chunk, word_chunk); + match BITS { + 1 => pack_64_bits(code_chunk, word_chunk), + 4 => pack_32_codes(code_chunk, word_chunk), + 6 => pack_32_six_bits(code_chunk, word_chunk), + _ => unreachable!(), } } } @@ -112,3 +123,45 @@ unsafe fn pack_32_codes(codes: &[u8], words: &mut [u64]) { ); } } + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_six_bits(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); + debug_assert_eq!(words.len(), 3); + + // First combine adjacent codes into 12-bit pairs, then combine adjacent + // pairs into 24-bit values. Two values make one complete 48-bit group. + let pair_weights = _mm256_setr_epi8( + 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, // + 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, + ); + let group_weights = _mm256_setr_epi16( + 1, 4096, 1, 4096, 1, 4096, 1, 4096, // + 1, 4096, 1, 4096, 1, 4096, 1, 4096, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_maddubs_epi16(input, pair_weights); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let groups = _mm256_or_si256(even, _mm256_slli_epi64(odd, 24)); + + let mut packed_groups = [0u64; 4]; + // SAFETY: `packed_groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(packed_groups.as_mut_ptr().cast::<__m256i>(), groups) }; + + words[0] = packed_groups[0] | (packed_groups[1] << 48); + words[1] = (packed_groups[1] >> 16) | (packed_groups[2] << 32); + words[2] = (packed_groups[2] >> 32) | (packed_groups[3] << 16); +} diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs index b1846de..c285cc9 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -100,4 +100,24 @@ mod tests_for_avx2 { } } } + + #[test] + fn bits_six_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 32, 64, 96, 128] { + for input in [ + vec![0; code_len], + vec![0x3f; code_len], + codes::<6>(code_len), + ] { + let expected = run::<6>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<6>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } } From 58b63d7d41e29fd70b54004c6acca6f0034bc1f0 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 13:38:29 +0000 Subject: [PATCH 19/51] Add AVX2 three-bit packing backend --- .../words_pack/funcs_for_pack_codes_core.rs | 2 +- .../funcs_for_pack_codes_core/avx2.rs | 93 +++++++++++++++++-- .../tests_for_backend_equivalence.rs | 20 ++++ 3 files changed, 107 insertions(+), 8 deletions(-) diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index 9bb8425..0aa57f9 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -31,7 +31,7 @@ fn dispatch(dst: &mut [u64], codes: &[u8]) { target_feature = "avx2" ))] { - if matches!(BITS, 1 | 4 | 6) && codes.len() >= 32 { + if matches!(BITS, 1 | 3 | 4 | 6) && codes.len() >= 32 { // SAFETY: This branch is compiled only when AVX2 is enabled. unsafe { avx2::pack_codes::(dst, codes) }; return; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs index 1493b62..50e60c4 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -1,5 +1,5 @@ -//! AVX2 packing is intentionally implemented only for `BITS=1`, `BITS=4`, and -//! `BITS=6`. +//! AVX2 packing is intentionally implemented only for `BITS=1`, `BITS=3`, +//! `BITS=4`, and `BITS=6`. //! //! A `BITS=2` compression prototype and a `BITS=8` byte-copy prototype were //! benchmarked against the scalar backend without a stable construction @@ -32,24 +32,29 @@ use core::arch::x86_64::{ const CODES_PER_SIMD_BLOCK: usize = 32; const WORDS_PER_SIMD_BLOCK: usize = 2; -/// Packs the BITS=1, BITS=4, or BITS=6 SIMD prefix and delegates complete remaining -/// blocks to scalar. +/// Packs the BITS=1, BITS=3, BITS=4, or BITS=6 SIMD prefix and delegates +/// complete remaining blocks to scalar. /// /// # Safety /// /// The caller must only invoke this function when AVX2 is available. The -/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1, 4, or 6. +/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1, 3, 4, or 6. #[target_feature(enable = "avx2")] pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { - debug_assert!(matches!(BITS, 1 | 4 | 6)); + debug_assert!(matches!(BITS, 1 | 3 | 4 | 6)); debug_assert_eq!(codes.len() % layout_block_len::(), 0); debug_assert_eq!(dst.len(), codes.len() * usize::from(BITS) / 64); - let simd_code_block_len = if BITS == 1 { 64 } else { CODES_PER_SIMD_BLOCK }; + let simd_code_block_len = if matches!(BITS, 1 | 3) { + 64 + } else { + CODES_PER_SIMD_BLOCK + }; let simd_code_len = codes.len() / simd_code_block_len * simd_code_block_len; let simd_word_len = simd_code_len * usize::from(BITS) / 64; let simd_word_block_len = match BITS { 1 => 1, + 3 => 3, 4 => WORDS_PER_SIMD_BLOCK, 6 => 3, _ => unreachable!(), @@ -64,6 +69,7 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { unsafe { match BITS { 1 => pack_64_bits(code_chunk, word_chunk), + 3 => pack_64_three_bits(code_chunk, word_chunk), 4 => pack_32_codes(code_chunk, word_chunk), 6 => pack_32_six_bits(code_chunk, word_chunk), _ => unreachable!(), @@ -124,6 +130,79 @@ unsafe fn pack_32_codes(codes: &[u8], words: &mut [u64]) { } } +#[target_feature(enable = "avx2")] +unsafe fn pack_64_three_bits(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), 64); + debug_assert_eq!(words.len(), 3); + + let mut groups = [0u64; 8]; + // SAFETY: Each half contains exactly 32 initialized codes and has space + // for the four 24-bit groups produced by the kernel. + unsafe { + pack_32_three_bit_groups(&codes[..32], &mut groups[..4]); + pack_32_three_bit_groups(&codes[32..], &mut groups[4..]); + } + + let mut accumulator = 0u64; + let mut accumulated_bits = 0; + let mut word_index = 0; + for group in groups { + let next_bits = accumulated_bits + 24; + if next_bits < 64 { + accumulator |= group << accumulated_bits; + accumulated_bits = next_bits; + } else if next_bits == 64 { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = 0; + accumulated_bits = 0; + } else { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = group >> (64 - accumulated_bits); + accumulated_bits = next_bits - 64; + } + } + + debug_assert_eq!(accumulated_bits, 0); + debug_assert_eq!(word_index, 3); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_three_bit_groups(codes: &[u8], groups: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); + debug_assert_eq!(groups.len(), 4); + + let pair_weights = _mm256_setr_epi8( + 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, // + 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, + ); + let group_weights = _mm256_setr_epi16( + 1, 64, 1, 64, 1, 64, 1, 64, // + 1, 64, 1, 64, 1, 64, 1, 64, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_maddubs_epi16(input, pair_weights); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let packed = _mm256_or_si256(even, _mm256_slli_epi64(odd, 12)); + + // SAFETY: `groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; +} + #[target_feature(enable = "avx2")] unsafe fn pack_32_six_bits(codes: &[u8], words: &mut [u64]) { debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs index c285cc9..7c35b70 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -85,6 +85,26 @@ mod tests_for_avx2 { } } + #[test] + fn bits_three_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 64, 128, 192, 256] { + for input in [ + vec![0; code_len], + vec![0x07; code_len], + codes::<3>(code_len), + ] { + let expected = run::<3>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<3>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } + #[test] fn bits_one_matches_scalar_across_simd_prefix_and_tail() { for code_len in [0, 64, 128, 192, 256] { From 6e081051f0223793b03e4c92e5d371227bae0e5d Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 13:43:14 +0000 Subject: [PATCH 20/51] Add AVX2 five-bit packing backend --- .../words_pack/funcs_for_pack_codes_core.rs | 2 +- .../funcs_for_pack_codes_core/avx2.rs | 87 +++++++++++++++++-- .../tests_for_backend_equivalence.rs | 20 +++++ 3 files changed, 102 insertions(+), 7 deletions(-) diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index 0aa57f9..1ac1633 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -31,7 +31,7 @@ fn dispatch(dst: &mut [u64], codes: &[u8]) { target_feature = "avx2" ))] { - if matches!(BITS, 1 | 3 | 4 | 6) && codes.len() >= 32 { + if matches!(BITS, 1 | 3 | 4 | 5 | 6) && codes.len() >= 32 { // SAFETY: This branch is compiled only when AVX2 is enabled. unsafe { avx2::pack_codes::(dst, codes) }; return; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs index 50e60c4..8705fe0 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -1,5 +1,5 @@ //! AVX2 packing is intentionally implemented only for `BITS=1`, `BITS=3`, -//! `BITS=4`, and `BITS=6`. +//! `BITS=4`, `BITS=5`, and `BITS=6`. //! //! A `BITS=2` compression prototype and a `BITS=8` byte-copy prototype were //! benchmarked against the scalar backend without a stable construction @@ -32,20 +32,20 @@ use core::arch::x86_64::{ const CODES_PER_SIMD_BLOCK: usize = 32; const WORDS_PER_SIMD_BLOCK: usize = 2; -/// Packs the BITS=1, BITS=3, BITS=4, or BITS=6 SIMD prefix and delegates -/// complete remaining blocks to scalar. +/// Packs the BITS=1, BITS=3, BITS=4, BITS=5, or BITS=6 SIMD prefix and +/// delegates complete remaining blocks to scalar. /// /// # Safety /// /// The caller must only invoke this function when AVX2 is available. The -/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1, 3, 4, or 6. +/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1, 3, 4, 5, or 6. #[target_feature(enable = "avx2")] pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { - debug_assert!(matches!(BITS, 1 | 3 | 4 | 6)); + debug_assert!(matches!(BITS, 1 | 3 | 4 | 5 | 6)); debug_assert_eq!(codes.len() % layout_block_len::(), 0); debug_assert_eq!(dst.len(), codes.len() * usize::from(BITS) / 64); - let simd_code_block_len = if matches!(BITS, 1 | 3) { + let simd_code_block_len = if matches!(BITS, 1 | 3 | 5) { 64 } else { CODES_PER_SIMD_BLOCK @@ -56,6 +56,7 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { 1 => 1, 3 => 3, 4 => WORDS_PER_SIMD_BLOCK, + 5 => 5, 6 => 3, _ => unreachable!(), }; @@ -71,6 +72,7 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { 1 => pack_64_bits(code_chunk, word_chunk), 3 => pack_64_three_bits(code_chunk, word_chunk), 4 => pack_32_codes(code_chunk, word_chunk), + 5 => pack_64_five_bits(code_chunk, word_chunk), 6 => pack_32_six_bits(code_chunk, word_chunk), _ => unreachable!(), } @@ -203,6 +205,79 @@ unsafe fn pack_32_three_bit_groups(codes: &[u8], groups: &mut [u64]) { unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; } +#[target_feature(enable = "avx2")] +unsafe fn pack_64_five_bits(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), 64); + debug_assert_eq!(words.len(), 5); + + let mut groups = [0u64; 8]; + // SAFETY: Each half contains exactly 32 initialized codes and has space + // for the four 40-bit groups produced by the kernel. + unsafe { + pack_32_five_bit_groups(&codes[..32], &mut groups[..4]); + pack_32_five_bit_groups(&codes[32..], &mut groups[4..]); + } + + let mut accumulator = 0u64; + let mut accumulated_bits = 0; + let mut word_index = 0; + for group in groups { + let next_bits = accumulated_bits + 40; + if next_bits < 64 { + accumulator |= group << accumulated_bits; + accumulated_bits = next_bits; + } else if next_bits == 64 { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = 0; + accumulated_bits = 0; + } else { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = group >> (64 - accumulated_bits); + accumulated_bits = next_bits - 64; + } + } + + debug_assert_eq!(accumulated_bits, 0); + debug_assert_eq!(word_index, 5); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_five_bit_groups(codes: &[u8], groups: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); + debug_assert_eq!(groups.len(), 4); + + let pair_weights = _mm256_setr_epi8( + 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, // + 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, + ); + let group_weights = _mm256_setr_epi16( + 1, 1024, 1, 1024, 1, 1024, 1, 1024, // + 1, 1024, 1, 1024, 1, 1024, 1, 1024, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_maddubs_epi16(input, pair_weights); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let packed = _mm256_or_si256(even, _mm256_slli_epi64(odd, 20)); + + // SAFETY: `groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; +} + #[target_feature(enable = "avx2")] unsafe fn pack_32_six_bits(codes: &[u8], words: &mut [u64]) { debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs index 7c35b70..10975f1 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -105,6 +105,26 @@ mod tests_for_avx2 { } } + #[test] + fn bits_five_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 64, 128, 192, 256] { + for input in [ + vec![0; code_len], + vec![0x1f; code_len], + codes::<5>(code_len), + ] { + let expected = run::<5>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<5>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } + #[test] fn bits_one_matches_scalar_across_simd_prefix_and_tail() { for code_len in [0, 64, 128, 192, 256] { From a10ad6f2c481f0a6e6d5ecf666c009e3cb1398f1 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 13:49:28 +0000 Subject: [PATCH 21/51] Add AVX2 seven-bit packing backend --- .../words_pack/funcs_for_pack_codes_core.rs | 2 +- .../funcs_for_pack_codes_core/avx2.rs | 112 +++++++++++++++--- .../tests_for_backend_equivalence.rs | 20 ++++ 3 files changed, 119 insertions(+), 15 deletions(-) diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index 1ac1633..d3b87f0 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -31,7 +31,7 @@ fn dispatch(dst: &mut [u64], codes: &[u8]) { target_feature = "avx2" ))] { - if matches!(BITS, 1 | 3 | 4 | 5 | 6) && codes.len() >= 32 { + if matches!(BITS, 1 | 3 | 4 | 5 | 6 | 7) && codes.len() >= 32 { // SAFETY: This branch is compiled only when AVX2 is enabled. unsafe { avx2::pack_codes::(dst, codes) }; return; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs index 8705fe0..09f3402 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -1,5 +1,5 @@ //! AVX2 packing is intentionally implemented only for `BITS=1`, `BITS=3`, -//! `BITS=4`, `BITS=5`, and `BITS=6`. +//! `BITS=4`, `BITS=5`, `BITS=6`, and `BITS=7`. //! //! A `BITS=2` compression prototype and a `BITS=8` byte-copy prototype were //! benchmarked against the scalar backend without a stable construction @@ -13,39 +13,39 @@ use crate::traits::words_pack::layout_block_len; #[cfg(target_arch = "x86")] use core::arch::x86::{ - __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_cvtepu32_epi64, - _mm256_loadu_si256, _mm256_madd_epi16, _mm256_maddubs_epi16, _mm256_movemask_epi8, - _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_permutevar8x32_epi32, _mm256_setr_epi8, - _mm256_setr_epi16, _mm256_setr_epi32, _mm256_shuffle_epi8, _mm256_slli_epi16, + __m128i, __m256i, _mm_storeu_si128, _mm256_add_epi16, _mm256_castsi256_si128, + _mm256_cvtepu32_epi64, _mm256_loadu_si256, _mm256_madd_epi16, _mm256_maddubs_epi16, + _mm256_movemask_epi8, _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_permutevar8x32_epi32, + _mm256_setr_epi8, _mm256_setr_epi16, _mm256_setr_epi32, _mm256_shuffle_epi8, _mm256_slli_epi16, _mm256_slli_epi64, _mm256_storeu_si256, }; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{ - __m128i, __m256i, _mm_storeu_si128, _mm256_castsi256_si128, _mm256_cvtepu32_epi64, - _mm256_loadu_si256, _mm256_madd_epi16, _mm256_maddubs_epi16, _mm256_movemask_epi8, - _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_permutevar8x32_epi32, _mm256_setr_epi8, - _mm256_setr_epi16, _mm256_setr_epi32, _mm256_shuffle_epi8, _mm256_slli_epi16, + __m128i, __m256i, _mm_storeu_si128, _mm256_add_epi16, _mm256_castsi256_si128, + _mm256_cvtepu32_epi64, _mm256_loadu_si256, _mm256_madd_epi16, _mm256_maddubs_epi16, + _mm256_movemask_epi8, _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_permutevar8x32_epi32, + _mm256_setr_epi8, _mm256_setr_epi16, _mm256_setr_epi32, _mm256_shuffle_epi8, _mm256_slli_epi16, _mm256_slli_epi64, _mm256_storeu_si256, }; const CODES_PER_SIMD_BLOCK: usize = 32; const WORDS_PER_SIMD_BLOCK: usize = 2; -/// Packs the BITS=1, BITS=3, BITS=4, BITS=5, or BITS=6 SIMD prefix and -/// delegates complete remaining blocks to scalar. +/// Packs the BITS=1, BITS=3, BITS=4, BITS=5, BITS=6, or BITS=7 SIMD prefix +/// and delegates complete remaining blocks to scalar. /// /// # Safety /// /// The caller must only invoke this function when AVX2 is available. The -/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1, 3, 4, 5, or 6. +/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1, 3, 4, 5, 6, or 7. #[target_feature(enable = "avx2")] pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { - debug_assert!(matches!(BITS, 1 | 3 | 4 | 5 | 6)); + debug_assert!(matches!(BITS, 1 | 3 | 4 | 5 | 6 | 7)); debug_assert_eq!(codes.len() % layout_block_len::(), 0); debug_assert_eq!(dst.len(), codes.len() * usize::from(BITS) / 64); - let simd_code_block_len = if matches!(BITS, 1 | 3 | 5) { + let simd_code_block_len = if matches!(BITS, 1 | 3 | 5 | 7) { 64 } else { CODES_PER_SIMD_BLOCK @@ -58,6 +58,7 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { 4 => WORDS_PER_SIMD_BLOCK, 5 => 5, 6 => 3, + 7 => 7, _ => unreachable!(), }; @@ -74,6 +75,7 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { 4 => pack_32_codes(code_chunk, word_chunk), 5 => pack_64_five_bits(code_chunk, word_chunk), 6 => pack_32_six_bits(code_chunk, word_chunk), + 7 => pack_64_seven_bits(code_chunk, word_chunk), _ => unreachable!(), } } @@ -278,6 +280,88 @@ unsafe fn pack_32_five_bit_groups(codes: &[u8], groups: &mut [u64]) { unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; } +#[target_feature(enable = "avx2")] +unsafe fn pack_64_seven_bits(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), 64); + debug_assert_eq!(words.len(), 7); + + let mut groups = [0u64; 8]; + // SAFETY: Each half contains exactly 32 initialized codes and has space + // for the four 56-bit groups produced by the kernel. + unsafe { + pack_32_seven_bit_groups(&codes[..32], &mut groups[..4]); + pack_32_seven_bit_groups(&codes[32..], &mut groups[4..]); + } + + let mut accumulator = 0u64; + let mut accumulated_bits = 0; + let mut word_index = 0; + for group in groups { + let next_bits = accumulated_bits + 56; + if next_bits < 64 { + accumulator |= group << accumulated_bits; + accumulated_bits = next_bits; + } else if next_bits == 64 { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = 0; + accumulated_bits = 0; + } else { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = group >> (64 - accumulated_bits); + accumulated_bits = next_bits - 64; + } + } + + debug_assert_eq!(accumulated_bits, 0); + debug_assert_eq!(word_index, 7); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_seven_bit_groups(codes: &[u8], groups: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); + debug_assert_eq!(groups.len(), 4); + + // `maddubs` uses signed byte weights, so 128 cannot be represented + // directly. The first product uses 127 and the second adds the odd code. + let pair_weights = _mm256_setr_epi8( + 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, // + 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, + ); + let odd_weights = _mm256_setr_epi8( + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, // + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + ); + let group_weights = _mm256_setr_epi16( + 1, 16384, 1, 16384, 1, 16384, 1, 16384, // + 1, 16384, 1, 16384, 1, 16384, 1, 16384, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_add_epi16( + _mm256_maddubs_epi16(input, pair_weights), + _mm256_maddubs_epi16(input, odd_weights), + ); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let packed = _mm256_or_si256(even, _mm256_slli_epi64(odd, 28)); + + // SAFETY: `groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; +} + #[target_feature(enable = "avx2")] unsafe fn pack_32_six_bits(codes: &[u8], words: &mut [u64]) { debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs index 10975f1..e2afb37 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -125,6 +125,26 @@ mod tests_for_avx2 { } } + #[test] + fn bits_seven_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 64, 128, 192, 256] { + for input in [ + vec![0; code_len], + vec![0x7f; code_len], + codes::<7>(code_len), + ] { + let expected = run::<7>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<7>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } + #[test] fn bits_one_matches_scalar_across_simd_prefix_and_tail() { for code_len in [0, 64, 128, 192, 256] { From cc36fbfcfe0f1d5d5977ec0879482ebc498b717b Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 13:58:15 +0000 Subject: [PATCH 22/51] Freeze AVX2 packing thresholds --- Cargo.toml | 4 + benches/packed_thresholds.rs | 137 ++++++++++++++++++ .../words_pack/funcs_for_pack_codes_core.rs | 20 ++- 3 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 benches/packed_thresholds.rs diff --git a/Cargo.toml b/Cargo.toml index 3150b2e..8b9ab1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,10 @@ harness = false name = "packed_construction" harness = false +[[bench]] +name = "packed_thresholds" +harness = false + [[bench]] name = "packed_access" harness = false diff --git a/benches/packed_thresholds.rs b/benches/packed_thresholds.rs new file mode 100644 index 0000000..2d08e80 --- /dev/null +++ b/benches/packed_thresholds.rs @@ -0,0 +1,137 @@ +//! Compare packed construction with and without AVX2 enabled. +//! +//! Run this benchmark without `-C target-feature=+avx2` for the scalar +//! baseline, then with that target feature to measure the AVX2 dispatch. +//! Widths 2 and 8 are intentionally omitted because their AVX2 prototypes did +//! not show a stable construction-speed improvement; see the AVX2 backend +//! documentation. + +#[path = "packed_support/mod.rs"] +mod support; + +use divan::{Bencher, black_box}; +use support::{codes, packed}; + +fn main() { + divan::main(); +} + +macro_rules! define_width { + ($module:ident, $bits:literal, [$($name:ident => $len:literal),+ $(,)?]) => { + mod $module { + use super::*; + + $( + #[divan::bench( + name = concat!( + "packed_thresholds/bits_", + stringify!($bits), + "/len_", + stringify!($len) + ) + )] + fn $name(bencher: Bencher) { + let input = codes($bits, $len); + bencher.bench(|| black_box(packed::<$bits>(&input))); + } + )+ + } + }; +} + +define_width!( + bits_1, + 1, + [ + len_64 => 64, + len_128 => 128, + len_192 => 192, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_3, + 3, + [ + len_64 => 64, + len_128 => 128, + len_192 => 192, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_4, + 4, + [ + len_16 => 16, + len_32 => 32, + len_48 => 48, + len_64 => 64, + len_96 => 96, + len_128 => 128, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_5, + 5, + [ + len_64 => 64, + len_128 => 128, + len_192 => 192, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_6, + 6, + [ + len_16 => 16, + len_32 => 32, + len_48 => 48, + len_64 => 64, + len_96 => 96, + len_128 => 128, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_7, + 7, + [ + len_64 => 64, + len_128 => 128, + len_192 => 192, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index d3b87f0..1774088 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -31,7 +31,25 @@ fn dispatch(dst: &mut [u64], codes: &[u8]) { target_feature = "avx2" ))] { - if matches!(BITS, 1 | 3 | 4 | 5 | 6 | 7) && codes.len() >= 32 { + // Conservative thresholds selected from `benches/packed_thresholds.rs`. + const AVX2_BITS_1_THRESHOLD: usize = 64; + const AVX2_BITS_3_THRESHOLD: usize = 64; + const AVX2_BITS_4_THRESHOLD: usize = 64; + const AVX2_BITS_5_THRESHOLD: usize = 64; + const AVX2_BITS_6_THRESHOLD: usize = 32; + const AVX2_BITS_7_THRESHOLD: usize = 64; + + let use_avx2 = match BITS { + 1 => codes.len() >= AVX2_BITS_1_THRESHOLD, + 3 => codes.len() >= AVX2_BITS_3_THRESHOLD, + 4 => codes.len() >= AVX2_BITS_4_THRESHOLD, + 5 => codes.len() >= AVX2_BITS_5_THRESHOLD, + 6 => codes.len() >= AVX2_BITS_6_THRESHOLD, + 7 => codes.len() >= AVX2_BITS_7_THRESHOLD, + _ => false, + }; + + if use_avx2 { // SAFETY: This branch is compiled only when AVX2 is enabled. unsafe { avx2::pack_codes::(dst, codes) }; return; From 1d65434830e2d5fe6644d1e582d278e5b7f4a9be Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 20 Sep 2026 14:03:22 +0000 Subject: [PATCH 23/51] Organize AVX2 packing kernels --- .../funcs_for_pack_codes_core/avx2.rs | 298 ++++++++---------- 1 file changed, 126 insertions(+), 172 deletions(-) diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs index 09f3402..5f909f4 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -1,15 +1,12 @@ -//! AVX2 packing is intentionally implemented only for `BITS=1`, `BITS=3`, -//! `BITS=4`, `BITS=5`, `BITS=6`, and `BITS=7`. +//! AVX2 kernels are retained for `BITS=1`, `BITS=3`, `BITS=4`, `BITS=5`, +//! `BITS=6`, and `BITS=7`. //! -//! A `BITS=2` compression prototype and a `BITS=8` byte-copy prototype were -//! benchmarked against the scalar backend without a stable construction -//! speedup. Their extra dispatch and kernel complexity therefore was not -//! retained. Other widths continue to use scalar packing until a new kernel -//! demonstrates a measured benefit; re-run the construction benchmarks before -//! revisiting this support matrix. +//! `BITS=2` and `BITS=8` intentionally remain scalar: tested AVX2 +//! implementations showed no stable construction-speed improvement. Re-run +//! the construction benchmarks before changing this support matrix. use super::scalar; -use crate::traits::words_pack::layout_block_len; +use crate::{WORD_BITS, traits::words_pack::layout_block_len}; #[cfg(target_arch = "x86")] use core::arch::x86::{ @@ -29,8 +26,11 @@ use core::arch::x86_64::{ _mm256_slli_epi64, _mm256_storeu_si256, }; -const CODES_PER_SIMD_BLOCK: usize = 32; -const WORDS_PER_SIMD_BLOCK: usize = 2; +const CODES_PER_VECTOR: usize = 32; +// Each irregular-width group contains eight codes. +const BITS_3_GROUP_BITS: usize = 3 * 8; +const BITS_5_GROUP_BITS: usize = 5 * 8; +const BITS_7_GROUP_BITS: usize = 7 * 8; /// Packs the BITS=1, BITS=3, BITS=4, BITS=5, BITS=6, or BITS=7 SIMD prefix /// and delegates complete remaining blocks to scalar. @@ -48,19 +48,11 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { let simd_code_block_len = if matches!(BITS, 1 | 3 | 5 | 7) { 64 } else { - CODES_PER_SIMD_BLOCK + CODES_PER_VECTOR }; let simd_code_len = codes.len() / simd_code_block_len * simd_code_block_len; let simd_word_len = simd_code_len * usize::from(BITS) / 64; - let simd_word_block_len = match BITS { - 1 => 1, - 3 => 3, - 4 => WORDS_PER_SIMD_BLOCK, - 5 => 5, - 6 => 3, - 7 => 7, - _ => unreachable!(), - }; + let simd_word_block_len = simd_code_block_len * usize::from(BITS) / WORD_BITS; for (code_chunk, word_chunk) in codes[..simd_code_len] .chunks_exact(simd_code_block_len) @@ -70,12 +62,12 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { // chunk, and the backend is compiled with AVX2 enabled. unsafe { match BITS { - 1 => pack_64_bits(code_chunk, word_chunk), - 3 => pack_64_three_bits(code_chunk, word_chunk), - 4 => pack_32_codes(code_chunk, word_chunk), - 5 => pack_64_five_bits(code_chunk, word_chunk), - 6 => pack_32_six_bits(code_chunk, word_chunk), - 7 => pack_64_seven_bits(code_chunk, word_chunk), + 1 => pack_64_one_bit(code_chunk, word_chunk), + 3 => pack_64_three_bit(code_chunk, word_chunk), + 4 => pack_32_four_bit(code_chunk, word_chunk), + 5 => pack_64_five_bit(code_chunk, word_chunk), + 6 => pack_32_six_bit(code_chunk, word_chunk), + 7 => pack_64_seven_bit(code_chunk, word_chunk), _ => unreachable!(), } } @@ -87,7 +79,7 @@ pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { } #[target_feature(enable = "avx2")] -unsafe fn pack_64_bits(codes: &[u8], words: &mut [u64]) { +unsafe fn pack_64_one_bit(codes: &[u8], words: &mut [u64]) { debug_assert_eq!(codes.len(), 64); debug_assert_eq!(words.len(), 1); @@ -104,38 +96,7 @@ unsafe fn pack_64_bits(codes: &[u8], words: &mut [u64]) { } #[target_feature(enable = "avx2")] -unsafe fn pack_32_codes(codes: &[u8], words: &mut [u64]) { - debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); - debug_assert_eq!(words.len(), WORDS_PER_SIMD_BLOCK); - - let even_indices = _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, // - 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, - ); - let odd_indices = _mm256_setr_epi8( - 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, // - 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, - ); - - // SAFETY: `codes` contains exactly 32 initialized bytes. - let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; - let even = _mm256_shuffle_epi8(input, even_indices); - let odd = _mm256_shuffle_epi8(input, odd_indices); - let packed = _mm256_or_si256(even, _mm256_slli_epi16(odd, 4)); - let compacted = _mm256_permute4x64_epi64(packed, 0x88); - - // SAFETY: `compacted` contains two output words in its low 128 bits, and - // `words` has space for exactly those two words. - unsafe { - _mm_storeu_si128( - words.as_mut_ptr().cast::<__m128i>(), - _mm256_castsi256_si128(compacted), - ); - } -} - -#[target_feature(enable = "avx2")] -unsafe fn pack_64_three_bits(codes: &[u8], words: &mut [u64]) { +unsafe fn pack_64_three_bit(codes: &[u8], words: &mut [u64]) { debug_assert_eq!(codes.len(), 64); debug_assert_eq!(words.len(), 3); @@ -147,34 +108,12 @@ unsafe fn pack_64_three_bits(codes: &[u8], words: &mut [u64]) { pack_32_three_bit_groups(&codes[32..], &mut groups[4..]); } - let mut accumulator = 0u64; - let mut accumulated_bits = 0; - let mut word_index = 0; - for group in groups { - let next_bits = accumulated_bits + 24; - if next_bits < 64 { - accumulator |= group << accumulated_bits; - accumulated_bits = next_bits; - } else if next_bits == 64 { - words[word_index] = accumulator | (group << accumulated_bits); - word_index += 1; - accumulator = 0; - accumulated_bits = 0; - } else { - words[word_index] = accumulator | (group << accumulated_bits); - word_index += 1; - accumulator = group >> (64 - accumulated_bits); - accumulated_bits = next_bits - 64; - } - } - - debug_assert_eq!(accumulated_bits, 0); - debug_assert_eq!(word_index, 3); + write_groups::(words, &groups); } #[target_feature(enable = "avx2")] unsafe fn pack_32_three_bit_groups(codes: &[u8], groups: &mut [u64]) { - debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); debug_assert_eq!(groups.len(), 4); let pair_weights = _mm256_setr_epi8( @@ -208,7 +147,38 @@ unsafe fn pack_32_three_bit_groups(codes: &[u8], groups: &mut [u64]) { } #[target_feature(enable = "avx2")] -unsafe fn pack_64_five_bits(codes: &[u8], words: &mut [u64]) { +unsafe fn pack_32_four_bit(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); + debug_assert_eq!(words.len(), 2); + + let even_indices = _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, // + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, + ); + let odd_indices = _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, // + 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let even = _mm256_shuffle_epi8(input, even_indices); + let odd = _mm256_shuffle_epi8(input, odd_indices); + let packed = _mm256_or_si256(even, _mm256_slli_epi16(odd, 4)); + let compacted = _mm256_permute4x64_epi64(packed, 0x88); + + // SAFETY: `compacted` contains two output words in its low 128 bits, and + // `words` has space for exactly those two words. + unsafe { + _mm_storeu_si128( + words.as_mut_ptr().cast::<__m128i>(), + _mm256_castsi256_si128(compacted), + ); + } +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_64_five_bit(codes: &[u8], words: &mut [u64]) { debug_assert_eq!(codes.len(), 64); debug_assert_eq!(words.len(), 5); @@ -220,34 +190,12 @@ unsafe fn pack_64_five_bits(codes: &[u8], words: &mut [u64]) { pack_32_five_bit_groups(&codes[32..], &mut groups[4..]); } - let mut accumulator = 0u64; - let mut accumulated_bits = 0; - let mut word_index = 0; - for group in groups { - let next_bits = accumulated_bits + 40; - if next_bits < 64 { - accumulator |= group << accumulated_bits; - accumulated_bits = next_bits; - } else if next_bits == 64 { - words[word_index] = accumulator | (group << accumulated_bits); - word_index += 1; - accumulator = 0; - accumulated_bits = 0; - } else { - words[word_index] = accumulator | (group << accumulated_bits); - word_index += 1; - accumulator = group >> (64 - accumulated_bits); - accumulated_bits = next_bits - 64; - } - } - - debug_assert_eq!(accumulated_bits, 0); - debug_assert_eq!(word_index, 5); + write_groups::(words, &groups); } #[target_feature(enable = "avx2")] unsafe fn pack_32_five_bit_groups(codes: &[u8], groups: &mut [u64]) { - debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); debug_assert_eq!(groups.len(), 4); let pair_weights = _mm256_setr_epi8( @@ -281,7 +229,49 @@ unsafe fn pack_32_five_bit_groups(codes: &[u8], groups: &mut [u64]) { } #[target_feature(enable = "avx2")] -unsafe fn pack_64_seven_bits(codes: &[u8], words: &mut [u64]) { +unsafe fn pack_32_six_bit(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); + debug_assert_eq!(words.len(), 3); + + // First combine adjacent codes into 12-bit pairs, then combine adjacent + // pairs into 24-bit values. Two values make one complete 48-bit group. + let pair_weights = _mm256_setr_epi8( + 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, // + 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, + ); + let group_weights = _mm256_setr_epi16( + 1, 4096, 1, 4096, 1, 4096, 1, 4096, // + 1, 4096, 1, 4096, 1, 4096, 1, 4096, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_maddubs_epi16(input, pair_weights); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let groups = _mm256_or_si256(even, _mm256_slli_epi64(odd, 24)); + + let mut packed_groups = [0u64; 4]; + // SAFETY: `packed_groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(packed_groups.as_mut_ptr().cast::<__m256i>(), groups) }; + + words[0] = packed_groups[0] | (packed_groups[1] << 48); + words[1] = (packed_groups[1] >> 16) | (packed_groups[2] << 32); + words[2] = (packed_groups[2] >> 32) | (packed_groups[3] << 16); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_64_seven_bit(codes: &[u8], words: &mut [u64]) { debug_assert_eq!(codes.len(), 64); debug_assert_eq!(words.len(), 7); @@ -293,34 +283,12 @@ unsafe fn pack_64_seven_bits(codes: &[u8], words: &mut [u64]) { pack_32_seven_bit_groups(&codes[32..], &mut groups[4..]); } - let mut accumulator = 0u64; - let mut accumulated_bits = 0; - let mut word_index = 0; - for group in groups { - let next_bits = accumulated_bits + 56; - if next_bits < 64 { - accumulator |= group << accumulated_bits; - accumulated_bits = next_bits; - } else if next_bits == 64 { - words[word_index] = accumulator | (group << accumulated_bits); - word_index += 1; - accumulator = 0; - accumulated_bits = 0; - } else { - words[word_index] = accumulator | (group << accumulated_bits); - word_index += 1; - accumulator = group >> (64 - accumulated_bits); - accumulated_bits = next_bits - 64; - } - } - - debug_assert_eq!(accumulated_bits, 0); - debug_assert_eq!(word_index, 7); + write_groups::(words, &groups); } #[target_feature(enable = "avx2")] unsafe fn pack_32_seven_bit_groups(codes: &[u8], groups: &mut [u64]) { - debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); debug_assert_eq!(groups.len(), 4); // `maddubs` uses signed byte weights, so 128 cannot be represented @@ -362,44 +330,30 @@ unsafe fn pack_32_seven_bit_groups(codes: &[u8], groups: &mut [u64]) { unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; } -#[target_feature(enable = "avx2")] -unsafe fn pack_32_six_bits(codes: &[u8], words: &mut [u64]) { - debug_assert_eq!(codes.len(), CODES_PER_SIMD_BLOCK); - debug_assert_eq!(words.len(), 3); - - // First combine adjacent codes into 12-bit pairs, then combine adjacent - // pairs into 24-bit values. Two values make one complete 48-bit group. - let pair_weights = _mm256_setr_epi8( - 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, // - 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, - ); - let group_weights = _mm256_setr_epi16( - 1, 4096, 1, 4096, 1, 4096, 1, 4096, // - 1, 4096, 1, 4096, 1, 4096, 1, 4096, - ); - - // SAFETY: `codes` contains exactly 32 initialized bytes. - let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; - let pairs = _mm256_maddubs_epi16(input, pair_weights); - let values = _mm256_madd_epi16(pairs, group_weights); - - let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); - let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); - let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( - values, - even_indices, - ))); - let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( - values, - odd_indices, - ))); - let groups = _mm256_or_si256(even, _mm256_slli_epi64(odd, 24)); +#[inline] +fn write_groups(words: &mut [u64], groups: &[u64]) { + let mut accumulator = 0u64; + let mut accumulated_bits = 0; + let mut word_index = 0; - let mut packed_groups = [0u64; 4]; - // SAFETY: `packed_groups` has space for exactly four u64 values. - unsafe { _mm256_storeu_si256(packed_groups.as_mut_ptr().cast::<__m256i>(), groups) }; + for &group in groups { + let next_bits = accumulated_bits + GROUP_BITS; + if next_bits < WORD_BITS { + accumulator |= group << accumulated_bits; + accumulated_bits = next_bits; + } else if next_bits == WORD_BITS { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = 0; + accumulated_bits = 0; + } else { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = group >> (WORD_BITS - accumulated_bits); + accumulated_bits = next_bits - WORD_BITS; + } + } - words[0] = packed_groups[0] | (packed_groups[1] << 48); - words[1] = (packed_groups[1] >> 16) | (packed_groups[2] << 32); - words[2] = (packed_groups[2] >> 32) | (packed_groups[3] << 16); + debug_assert_eq!(accumulated_bits, 0); + debug_assert_eq!(word_index, words.len()); } From 854cb1a2e2b761aebf24b6949a24c3301625ab0d Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 26 Sep 2026 22:42:31 +0800 Subject: [PATCH 24/51] Optimize sequential packed iteration --- src/packed_string/impls_for_iter.rs | 26 +++++++- .../impls_for_iter/tests_for_iter.rs | 65 +++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index 48273e3..65a7bf8 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -1,5 +1,7 @@ use core::iter::FusedIterator; +use crate::{WORD_BITS, code_mask}; + use super::*; impl PackedString @@ -12,6 +14,8 @@ where string: self, front: 0, back: self.char_len(), + front_word: 0, + front_bit_offset: 0, } } @@ -29,6 +33,8 @@ where string: &'a PackedString, front: usize, back: usize, + front_word: usize, + front_bit_offset: usize, } impl Iterator for Iter<'_, C, BITS> @@ -41,9 +47,25 @@ where if self.front == self.back { return None; } - let character = self.string.get(self.front); + + let width = usize::from(BITS); + let words = self.string.bits.words(); + let mut code = words[self.front_word] >> self.front_bit_offset; + if self.front_bit_offset + width > WORD_BITS { + code |= words[self.front_word + 1] << (WORD_BITS - self.front_bit_offset); + } + + let character = C::from_code((code & u64::from(code_mask::())) as u8) + .expect("PackedChar rejected a code it previously produced"); self.front += 1; - character + + self.front_bit_offset += width; + if self.front_bit_offset >= WORD_BITS { + self.front_word += 1; + self.front_bit_offset -= WORD_BITS; + } + + Some(character) } fn size_hint(&self) -> (usize, Option) { diff --git a/src/packed_string/impls_for_iter/tests_for_iter.rs b/src/packed_string/impls_for_iter/tests_for_iter.rs index 22f755b..273848c 100644 --- a/src/packed_string/impls_for_iter/tests_for_iter.rs +++ b/src/packed_string/impls_for_iter/tests_for_iter.rs @@ -1,4 +1,6 @@ +use crate::PackedString; use crate::packed_string::tests_for_support::{Letter, LetterString}; +use crate::traits::PackedChar; #[test] fn iter_is_double_ended_and_exact_size() { @@ -10,3 +12,66 @@ fn iter_is_double_ended_and_exact_size() { assert_eq!(iter.next(), Some(Letter::B)); assert_eq!(iter.next(), None); } + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} + +#[test] +fn mixed_iteration_preserves_order_across_word_boundaries() { + fn assert_front_cursor(iter: &super::Iter<'_, Code, BITS>, front: usize) { + assert_eq!( + front * usize::from(BITS), + iter.front_word * crate::WORD_BITS + iter.front_bit_offset + ); + } + + fn check() { + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let expected = (0..100) + .map(|index| Code(((index * 13 + 7) as u8) & mask)) + .collect::>(); + let value = PackedString::::from_chars(expected.iter().copied()); + let mut iter = value.iter(); + assert_front_cursor(&iter, 0); + + for (front, &code) in expected[..25].iter().enumerate() { + assert_eq!(iter.next(), Some(code)); + assert_front_cursor(&iter, front + 1); + } + for &code in expected[75..].iter().rev() { + assert_eq!(iter.next_back(), Some(code)); + assert_front_cursor(&iter, 25); + } + for (front, &code) in expected[25..75].iter().enumerate() { + assert_eq!(iter.next(), Some(code)); + assert_front_cursor(&iter, front + 26); + } + + assert_front_cursor(&iter, 75); + assert_eq!(iter.next(), None); + assert_eq!(iter.next_back(), None); + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} From e8f7d60e0ef6c1991a473dd6752dc11e1522f8d7 Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 29 Sep 2026 13:09:18 +0800 Subject: [PATCH 25/51] Add packed benchmarks for widths five and six --- benches/packed_support/mod.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/benches/packed_support/mod.rs b/benches/packed_support/mod.rs index e234fd2..e1feb95 100644 --- a/benches/packed_support/mod.rs +++ b/benches/packed_support/mod.rs @@ -39,6 +39,16 @@ macro_rules! for_each_packed_case { $macro!(bits_4_len_1024, 4, 1_024); $macro!(bits_4_len_65536, 4, 65_536); $macro!(bits_4_len_1048576, 4, 1_048_576); + $macro!(bits_5_len_16, 5, 16); + $macro!(bits_5_len_64, 5, 64); + $macro!(bits_5_len_1024, 5, 1_024); + $macro!(bits_5_len_65536, 5, 65_536); + $macro!(bits_5_len_1048576, 5, 1_048_576); + $macro!(bits_6_len_16, 6, 16); + $macro!(bits_6_len_64, 6, 64); + $macro!(bits_6_len_1024, 6, 1_024); + $macro!(bits_6_len_65536, 6, 65_536); + $macro!(bits_6_len_1048576, 6, 1_048_576); $macro!(bits_7_len_16, 7, 16); $macro!(bits_7_len_64, 7, 64); $macro!(bits_7_len_1024, 7, 1_024); @@ -75,6 +85,16 @@ macro_rules! for_each_grit_case { $macro!(bits_4_len_1024, 4, 1_024, grit_bitvec::u8_as_u4); $macro!(bits_4_len_65536, 4, 65_536, grit_bitvec::u8_as_u4); $macro!(bits_4_len_1048576, 4, 1_048_576, grit_bitvec::u8_as_u4); + $macro!(bits_5_len_16, 5, 16, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_64, 5, 64, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_1024, 5, 1_024, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_65536, 5, 65_536, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_1048576, 5, 1_048_576, grit_bitvec::u8_as_u5); + $macro!(bits_6_len_16, 6, 16, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_64, 6, 64, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_1024, 6, 1_024, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_65536, 6, 65_536, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_1048576, 6, 1_048_576, grit_bitvec::u8_as_u6); $macro!(bits_7_len_16, 7, 16, grit_bitvec::u8_as_u7); $macro!(bits_7_len_64, 7, 64, grit_bitvec::u8_as_u7); $macro!(bits_7_len_1024, 7, 1_024, grit_bitvec::u8_as_u7); @@ -98,6 +118,10 @@ macro_rules! for_each_packed_regression_case { $macro!(bits_3_len_1048576, 3, 1_048_576); $macro!(bits_4_len_64, 4, 64); $macro!(bits_4_len_1048576, 4, 1_048_576); + $macro!(bits_5_len_1024, 5, 1_024); + $macro!(bits_5_len_1048576, 5, 1_048_576); + $macro!(bits_6_len_1024, 6, 1_024); + $macro!(bits_6_len_1048576, 6, 1_048_576); $macro!(bits_7_len_1024, 7, 1_024); $macro!(bits_7_len_1048576, 7, 1_048_576); $macro!(bits_8_len_64, 8, 64); @@ -116,6 +140,10 @@ macro_rules! for_each_grit_regression_case { $macro!(bits_3_len_1048576, 3, 1_048_576, grit_bitvec::u8_as_u3); $macro!(bits_4_len_64, 4, 64, grit_bitvec::u8_as_u4); $macro!(bits_4_len_1048576, 4, 1_048_576, grit_bitvec::u8_as_u4); + $macro!(bits_5_len_1024, 5, 1_024, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_1048576, 5, 1_048_576, grit_bitvec::u8_as_u5); + $macro!(bits_6_len_1024, 6, 1_024, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_1048576, 6, 1_048_576, grit_bitvec::u8_as_u6); $macro!(bits_7_len_1024, 7, 1_024, grit_bitvec::u8_as_u7); $macro!(bits_7_len_1048576, 7, 1_048_576, grit_bitvec::u8_as_u7); }; From 5aaa220312a99eead67094a5dfbecc4524761a50 Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 29 Sep 2026 18:25:07 +0800 Subject: [PATCH 26/51] Optimize packed iteration with a rolling window --- src/packed_string/impls_for_iter.rs | 41 ++++++--- .../impls_for_iter/tests_for_iter.rs | 85 ++++++++++++++++--- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index 65a7bf8..d20ff84 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -14,8 +14,9 @@ where string: self, front: 0, back: self.char_len(), - front_word: 0, - front_bit_offset: 0, + front_pending_bits: 0, + front_pending_bit_count: 0, + front_next_word_index: 0, } } @@ -33,8 +34,14 @@ where string: &'a PackedString, front: usize, back: usize, - front_word: usize, - front_bit_offset: usize, + // Inspired by sux's bit-field iterator: cache unread bits and refill on demand. + // Loaded but unconsumed bits; the next code starts at bit 0. + front_pending_bits: u64, + // Number of buffered, unconsumed low bits; the final word may include tail padding. + front_pending_bit_count: usize, + // Index of the next backing word not yet loaded into `front_pending_bits`. + front_next_word_index: usize, + // Invariant: `front * BITS + front_pending_bit_count == front_next_word_index * WORD_BITS`. } impl Iterator for Iter<'_, C, BITS> @@ -50,21 +57,27 @@ where let width = usize::from(BITS); let words = self.string.bits.words(); - let mut code = words[self.front_word] >> self.front_bit_offset; - if self.front_bit_offset + width > WORD_BITS { - code |= words[self.front_word + 1] << (WORD_BITS - self.front_bit_offset); - } + let code = if self.front_pending_bit_count >= width { + let code = self.front_pending_bits; + self.front_pending_bits >>= width; + self.front_pending_bit_count -= width; + code + } else { + let old_fill = self.front_pending_bit_count; + let next_word = words[self.front_next_word_index]; + self.front_next_word_index += 1; + + let code = self.front_pending_bits | (next_word << old_fill); + let used_from_next = width - old_fill; + self.front_pending_bits = next_word >> used_from_next; + self.front_pending_bit_count = WORD_BITS - used_from_next; + code + }; let character = C::from_code((code & u64::from(code_mask::())) as u8) .expect("PackedChar rejected a code it previously produced"); self.front += 1; - self.front_bit_offset += width; - if self.front_bit_offset >= WORD_BITS { - self.front_word += 1; - self.front_bit_offset -= WORD_BITS; - } - Some(character) } diff --git a/src/packed_string/impls_for_iter/tests_for_iter.rs b/src/packed_string/impls_for_iter/tests_for_iter.rs index 273848c..4a3622a 100644 --- a/src/packed_string/impls_for_iter/tests_for_iter.rs +++ b/src/packed_string/impls_for_iter/tests_for_iter.rs @@ -26,15 +26,16 @@ impl PackedChar for Code { } } +fn assert_front_window(iter: &super::Iter<'_, Code, BITS>, front: usize) { + assert_eq!( + iter.front_next_word_index * crate::WORD_BITS, + front * usize::from(BITS) + iter.front_pending_bit_count + ); + assert!(iter.front_pending_bit_count <= crate::WORD_BITS); +} + #[test] fn mixed_iteration_preserves_order_across_word_boundaries() { - fn assert_front_cursor(iter: &super::Iter<'_, Code, BITS>, front: usize) { - assert_eq!( - front * usize::from(BITS), - iter.front_word * crate::WORD_BITS + iter.front_bit_offset - ); - } - fn check() { let mask = if BITS == 8 { u8::MAX @@ -46,22 +47,22 @@ fn mixed_iteration_preserves_order_across_word_boundaries() { .collect::>(); let value = PackedString::::from_chars(expected.iter().copied()); let mut iter = value.iter(); - assert_front_cursor(&iter, 0); + assert_front_window(&iter, 0); for (front, &code) in expected[..25].iter().enumerate() { assert_eq!(iter.next(), Some(code)); - assert_front_cursor(&iter, front + 1); + assert_front_window(&iter, front + 1); } for &code in expected[75..].iter().rev() { assert_eq!(iter.next_back(), Some(code)); - assert_front_cursor(&iter, 25); + assert_front_window(&iter, 25); } for (front, &code) in expected[25..75].iter().enumerate() { assert_eq!(iter.next(), Some(code)); - assert_front_cursor(&iter, front + 26); + assert_front_window(&iter, front + 26); } - assert_front_cursor(&iter, 75); + assert_front_window(&iter, 75); assert_eq!(iter.next(), None); assert_eq!(iter.next_back(), None); } @@ -75,3 +76,63 @@ fn mixed_iteration_preserves_order_across_word_boundaries() { check::<7>(); check::<8>(); } + +#[test] +fn rolling_window_handles_word_boundaries_and_mixed_iteration() { + fn check() { + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let codes_per_word = crate::WORD_BITS / usize::from(BITS); + let codes_per_two_words = (crate::WORD_BITS * 2).div_ceil(usize::from(BITS)); + let lengths = [ + 0, + 1, + codes_per_word.saturating_sub(1), + codes_per_word, + codes_per_word + 1, + codes_per_word * 2 - 1, + codes_per_word * 2, + codes_per_word * 2 + 1, + codes_per_two_words - 1, + codes_per_two_words, + codes_per_two_words + 1, + ]; + + for len in lengths { + let expected = (0..len) + .map(|index| Code(((index * 17 + 3) as u8) & mask)) + .collect::>(); + let value = PackedString::::from_chars(expected.iter().copied()); + let mut iter = value.iter(); + let mut front = 0; + let mut back = len; + assert_front_window(&iter, front); + + while front < back { + assert_eq!(iter.next(), Some(expected[front])); + front += 1; + assert_front_window(&iter, front); + if front < back { + back -= 1; + assert_eq!(iter.next_back(), Some(expected[back])); + assert_front_window(&iter, front); + } + } + + assert_eq!(iter.next(), None); + assert_eq!(iter.next_back(), None); + } + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} From ae9d1073a85083e29d50a1c5c24baca3f3e50827 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 30 Sep 2026 11:33:32 +0800 Subject: [PATCH 27/51] Add AVX2 packed bulk unpack for conversion --- Cargo.toml | 4 + benches/packed_conversion.rs | 41 ++++++++++ src/packed_string.rs | 1 + src/packed_string/impls_for_conversion.rs | 43 +++++++++++ .../tests_for_conversion.rs | 38 +++++++++ src/packed_string/impls_for_iter.rs | 5 -- src/traits.rs | 4 + src/traits/packed_layout.rs | 18 +++++ src/traits/words_pack.rs | 19 ----- .../words_pack/funcs_for_pack_codes_core.rs | 4 +- .../funcs_for_pack_codes_core/avx2.rs | 2 +- src/traits/words_unpack.rs | 14 ++++ .../funcs_for_unpack_codes_core.rs | 48 ++++++++++++ .../funcs_for_unpack_codes_core/avx2.rs | 77 +++++++++++++++++++ .../funcs_for_unpack_codes_core/scalar.rs | 30 ++++++++ .../tests_for_backend_equivalence.rs | 46 +++++++++++ .../words_unpack/impls_for_u64_slice.rs | 9 +++ .../words_unpack/tests_for_words_unpack.rs | 37 +++++++++ 18 files changed, 413 insertions(+), 27 deletions(-) create mode 100644 benches/packed_conversion.rs create mode 100644 src/packed_string/impls_for_conversion.rs create mode 100644 src/packed_string/impls_for_conversion/tests_for_conversion.rs create mode 100644 src/traits/packed_layout.rs create mode 100644 src/traits/words_unpack.rs create mode 100644 src/traits/words_unpack/funcs_for_unpack_codes_core.rs create mode 100644 src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs create mode 100644 src/traits/words_unpack/funcs_for_unpack_codes_core/scalar.rs create mode 100644 src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs create mode 100644 src/traits/words_unpack/impls_for_u64_slice.rs create mode 100644 src/traits/words_unpack/tests_for_words_unpack.rs diff --git a/Cargo.toml b/Cargo.toml index 8b9ab1f..ef5b665 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,6 +140,10 @@ harness = false name = "packed_iteration" harness = false +[[bench]] +name = "packed_conversion" +harness = false + [[bench]] name = "packed_editing" harness = false diff --git a/benches/packed_conversion.rs b/benches/packed_conversion.rs new file mode 100644 index 0000000..b16296f --- /dev/null +++ b/benches/packed_conversion.rs @@ -0,0 +1,41 @@ +#[path = "packed_support/mod.rs"] +mod support; + +use divan::{Bencher, black_box}; +use support::{codes, packed}; + +fn main() { + divan::main(); +} + +fn collect_via_iter(bencher: Bencher, len: usize) { + let input = codes(BITS, len); + let value = packed::(&input); + bencher.bench(|| black_box(&value).iter().collect::>()); +} + +fn convert_to_vec(bencher: Bencher, len: usize) { + let input = codes(BITS, len); + let value = packed::(&input); + bencher.bench(|| black_box(&value).to_vec()); +} + +macro_rules! define_case { + ($case:ident, $bits:literal, $len:literal) => { + mod $case { + use super::*; + + #[divan::bench(name = concat!("packed_conversion/to_vec/", stringify!($case), "/iter_collect"))] + fn iter_collect(bencher: Bencher) { + super::collect_via_iter::<$bits>(bencher, $len); + } + + #[divan::bench(name = concat!("packed_conversion/to_vec/", stringify!($case), "/to_vec"))] + fn to_vec(bencher: Bencher) { + super::convert_to_vec::<$bits>(bencher, $len); + } + } + }; +} + +crate::for_each_packed_bench_case!(define_case); diff --git a/src/packed_string.rs b/src/packed_string.rs index 9e4b78e..8b2e645 100644 --- a/src/packed_string.rs +++ b/src/packed_string.rs @@ -21,6 +21,7 @@ where mod funcs_for_code; mod impls_for_access; mod impls_for_construction; +mod impls_for_conversion; mod impls_for_editing; mod impls_for_eq; mod impls_for_fmt; diff --git a/src/packed_string/impls_for_conversion.rs b/src/packed_string/impls_for_conversion.rs new file mode 100644 index 0000000..a332f3d --- /dev/null +++ b/src/packed_string/impls_for_conversion.rs @@ -0,0 +1,43 @@ +use crate::traits::{WordsUnpack, layout_block_len}; + +use super::*; + +impl PackedString +where + C: PackedChar, +{ + /// Collects the decoded characters into a vector. + pub fn to_vec(&self) -> alloc::vec::Vec { + // 64 codes is a multiple of every layout block for BITS=1..=8. + const CODE_BATCH_LEN: usize = 64; + + debug_assert!(CODE_BATCH_LEN.is_multiple_of(layout_block_len::())); + + let char_len = self.char_len(); + let bulk_code_len = char_len / CODE_BATCH_LEN * CODE_BATCH_LEN; + let words = self.bits.words(); + let words_per_batch = CODE_BATCH_LEN * usize::from(BITS) / crate::WORD_BITS; + let mut decoded = [0u8; CODE_BATCH_LEN]; + let mut result = alloc::vec::Vec::with_capacity(char_len); + + for batch_index in 0..bulk_code_len / CODE_BATCH_LEN { + let word_start = batch_index * words_per_batch; + let word_end = word_start + words_per_batch; + words[word_start..word_end].unpack_codes::(&mut decoded); + + result.extend(decoded.iter().map(|&code| { + C::from_code(code).expect("PackedChar rejected a code it previously produced") + })); + } + + result.extend((bulk_code_len..char_len).map(|index| { + self.get(index) + .expect("packed character index is within the string") + })); + + result + } +} + +#[cfg(test)] +mod tests_for_conversion; diff --git a/src/packed_string/impls_for_conversion/tests_for_conversion.rs b/src/packed_string/impls_for_conversion/tests_for_conversion.rs new file mode 100644 index 0000000..235dec3 --- /dev/null +++ b/src/packed_string/impls_for_conversion/tests_for_conversion.rs @@ -0,0 +1,38 @@ +use crate::PackedString; +use crate::traits::PackedChar; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} + +#[test] +fn to_vec_matches_codes_across_batches_and_tails() { + fn check() { + let mask = crate::code_mask::(); + for len in [0, 1, 15, 16, 31, 32, 63, 64, 65, 127, 128, 129, 257] { + let expected = (0..len) + .map(|index| Code(((index * 29 + 5) as u8) & mask)) + .collect::>(); + let value = PackedString::::from_chars(expected.iter().copied()); + assert_eq!(value.to_vec(), expected, "BITS={BITS}, len={len}"); + } + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index d20ff84..68aaa34 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -19,11 +19,6 @@ where front_next_word_index: 0, } } - - /// Collects the decoded characters into a vector. - pub fn to_vec(&self) -> alloc::vec::Vec { - self.iter().collect() - } } #[derive(Clone)] diff --git a/src/traits.rs b/src/traits.rs index 992ac06..741c445 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,4 +1,5 @@ mod packed_char; +mod packed_layout; pub(crate) mod word_ord; pub(crate) mod words_arith; pub(crate) mod words_edit; @@ -7,8 +8,10 @@ pub(crate) mod words_find; pub(crate) mod words_ord; pub(crate) mod words_pack; pub(crate) mod words_scan; +pub(crate) mod words_unpack; pub use packed_char::PackedChar; +pub(crate) use packed_layout::layout_block_len; pub(crate) use word_ord::*; pub(crate) use words_arith::*; pub(crate) use words_edit::*; @@ -17,3 +20,4 @@ pub(crate) use words_find::*; pub(crate) use words_ord::*; pub(crate) use words_pack::*; pub(crate) use words_scan::*; +pub(crate) use words_unpack::*; diff --git a/src/traits/packed_layout.rs b/src/traits/packed_layout.rs new file mode 100644 index 0000000..d083826 --- /dev/null +++ b/src/traits/packed_layout.rs @@ -0,0 +1,18 @@ +use crate::{WORD_BITS, assert_valid_width}; + +/// Returns the smallest number of fixed-width codes that occupies whole words. +#[inline] +pub(crate) const fn layout_block_len() -> usize { + assert_valid_width::(); + WORD_BITS / gcd(WORD_BITS, BITS as usize) +} + +#[inline] +const fn gcd(mut lhs: usize, mut rhs: usize) -> usize { + while rhs != 0 { + let remainder = lhs % rhs; + lhs = rhs; + rhs = remainder; + } + lhs +} diff --git a/src/traits/words_pack.rs b/src/traits/words_pack.rs index c517bfd..73536e7 100644 --- a/src/traits/words_pack.rs +++ b/src/traits/words_pack.rs @@ -1,5 +1,3 @@ -use crate::{WORD_BITS, assert_valid_width}; - /// Fixed-width code packing operations on `[u64]` backing storage. /// /// The destination is expected to contain exactly the words produced by the @@ -12,22 +10,5 @@ pub(crate) trait WordsPack { fn pack_codes(&mut self, codes: &[u8]); } -/// Returns the smallest number of codes that occupies a whole number of words. -#[inline] -pub(crate) const fn layout_block_len() -> usize { - assert_valid_width::(); - WORD_BITS / gcd(WORD_BITS, BITS as usize) -} - -#[inline] -const fn gcd(mut lhs: usize, mut rhs: usize) -> usize { - while rhs != 0 { - let remainder = lhs % rhs; - lhs = rhs; - rhs = remainder; - } - lhs -} - pub(crate) mod funcs_for_pack_codes_core; mod impls_for_u64_slice; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs index 1774088..f1e35eb 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -1,9 +1,9 @@ -use crate::{assert_valid_width, code_mask, word_len}; +use crate::{assert_valid_width, code_mask, traits::layout_block_len, word_len}; #[inline] pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { assert_valid_width::(); - let layout_len = super::layout_block_len::(); + let layout_len = layout_block_len::(); let expected_words = word_len(codes.len() * usize::from(BITS)); assert!( diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs index 5f909f4..d1087fe 100644 --- a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -6,7 +6,7 @@ //! the construction benchmarks before changing this support matrix. use super::scalar; -use crate::{WORD_BITS, traits::words_pack::layout_block_len}; +use crate::{WORD_BITS, traits::layout_block_len}; #[cfg(target_arch = "x86")] use core::arch::x86::{ diff --git a/src/traits/words_unpack.rs b/src/traits/words_unpack.rs new file mode 100644 index 0000000..a263943 --- /dev/null +++ b/src/traits/words_unpack.rs @@ -0,0 +1,14 @@ +/// Fixed-width code unpacking operations on `[u64]` backing storage. +/// +/// The input must contain complete layout blocks and the output length must +/// equal the number of codes represented by those words. Every output code is +/// fully overwritten and contains only its low `BITS` bits. +pub(crate) trait WordsUnpack { + fn unpack_codes(&self, codes: &mut [u8]); +} + +pub(crate) mod funcs_for_unpack_codes_core; +mod impls_for_u64_slice; + +#[cfg(test)] +mod tests_for_words_unpack; diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs new file mode 100644 index 0000000..0642328 --- /dev/null +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs @@ -0,0 +1,48 @@ +use crate::{WORD_BITS, assert_valid_width, traits::layout_block_len}; + +#[inline] +pub(super) fn unpack_codes(words: &[u64], codes: &mut [u8]) { + assert_valid_width::(); + let expected_words = codes.len() * usize::from(BITS) / WORD_BITS; + + assert!( + codes.len().is_multiple_of(layout_block_len::()), + "code count must contain complete packed blocks" + ); + assert_eq!( + words.len(), + expected_words, + "input must contain exactly the words represented by the codes" + ); + + dispatch::(words, codes); +} + +#[inline] +fn dispatch(words: &[u64], codes: &mut [u8]) { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + const AVX2_BITS_4_THRESHOLD: usize = 64; + + if BITS == 4 && codes.len() >= AVX2_BITS_4_THRESHOLD { + // SAFETY: This branch is compiled only when AVX2 is enabled, and + // the core has validated the complete-block input/output contract. + unsafe { avx2::unpack_codes::(words, codes) }; + return; + } + } + + scalar::unpack_codes::(words, codes); +} + +mod scalar; + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2; + +#[cfg(test)] +mod tests_for_backend_equivalence; diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs new file mode 100644 index 0000000..1af7b0e --- /dev/null +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs @@ -0,0 +1,77 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_storeu_si128, _mm256_and_si256, _mm256_castsi256_si128, + _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_set1_epi8, _mm256_srli_epi16, + _mm256_unpackhi_epi8, _mm256_unpacklo_epi8, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_storeu_si128, _mm256_and_si256, _mm256_castsi256_si128, + _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_set1_epi8, _mm256_srli_epi16, + _mm256_unpackhi_epi8, _mm256_unpacklo_epi8, +}; + +const CODES_PER_BLOCK: usize = 64; +const WORDS_PER_BLOCK: usize = 4; + +/// AVX2 backend entry for BITS=4. It handles its SIMD prefix and scalar tail. +/// +/// # Safety +/// +/// The caller must ensure AVX2 is available and that the slices satisfy the +/// complete-layout-block `WordsUnpack` contract for `BITS=4`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn unpack_codes(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(BITS, 4); + + let simd_code_len = codes.len() / CODES_PER_BLOCK * CODES_PER_BLOCK; + let simd_word_len = simd_code_len / CODES_PER_BLOCK * WORDS_PER_BLOCK; + + for (word_block, code_block) in words[..simd_word_len] + .chunks_exact(WORDS_PER_BLOCK) + .zip(codes[..simd_code_len].chunks_exact_mut(CODES_PER_BLOCK)) + { + // SAFETY: Each paired block has exactly four initialized words and + // 64 initialized output bytes, as required by the kernel. + unsafe { unpack_64_four_bit(word_block, code_block) }; + } + + scalar::unpack_codes::(&words[simd_word_len..], &mut codes[simd_code_len..]); +} + +#[target_feature(enable = "avx2")] +unsafe fn unpack_64_four_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), WORDS_PER_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_BLOCK); + + let nibble_mask = _mm256_set1_epi8(0x0f); + // SAFETY: This block contains exactly 32 initialized bytes. + let packed = unsafe { _mm256_loadu_si256(words.as_ptr().cast::<__m256i>()) }; + let low = _mm256_and_si256(packed, nibble_mask); + let high = _mm256_and_si256(_mm256_srli_epi16(packed, 4), nibble_mask); + let first = _mm256_unpacklo_epi8(low, high); + let second = _mm256_unpackhi_epi8(low, high); + + // AVX2 byte unpack is lane-local. Store its four 128-bit halves in code order. + unsafe { + _mm_storeu_si128( + codes.as_mut_ptr().cast::<__m128i>(), + _mm256_castsi256_si128(first), + ); + _mm_storeu_si128( + codes.as_mut_ptr().add(16).cast::<__m128i>(), + _mm256_castsi256_si128(second), + ); + _mm_storeu_si128( + codes.as_mut_ptr().add(32).cast::<__m128i>(), + _mm256_extracti128_si256::<1>(first), + ); + _mm_storeu_si128( + codes.as_mut_ptr().add(48).cast::<__m128i>(), + _mm256_extracti128_si256::<1>(second), + ); + } +} diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/scalar.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/scalar.rs new file mode 100644 index 0000000..a3d19f5 --- /dev/null +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/scalar.rs @@ -0,0 +1,30 @@ +use crate::WORD_BITS; + +pub(super) fn unpack_codes(words: &[u64], codes: &mut [u8]) { + let width = usize::from(BITS); + let mask = (1u64 << BITS) - 1; + let mut next_word_index = 0; + let mut pending_bits = 0u64; + let mut pending_bit_count = 0; + + for code in codes { + let value = if pending_bit_count >= width { + let value = pending_bits; + pending_bits >>= width; + pending_bit_count -= width; + value + } else { + let old_fill = pending_bit_count; + let next_word = words[next_word_index]; + next_word_index += 1; + + let value = pending_bits | (next_word << old_fill); + let used_from_next = width - old_fill; + pending_bits = next_word >> used_from_next; + pending_bit_count = WORD_BITS - used_from_next; + value + }; + + *code = (value & mask) as u8; + } +} diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs new file mode 100644 index 0000000..47b6254 --- /dev/null +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs @@ -0,0 +1,46 @@ +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +use super::scalar; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +use crate::traits::WordsPack; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +use alloc::vec; + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_four_matches_scalar_across_prefix_and_tail() { + fn codes(len: usize) -> alloc::vec::Vec { + let mask = crate::code_mask::(); + (0..len) + .map(|index| ((index.wrapping_mul(29) + 7) as u8) & mask) + .collect() + } + + for code_len in [0, 16, 32, 48, 64, 80, 96, 112, 128, 144] { + let input = codes::<4>(code_len); + let words_len = code_len / 16; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<4>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<4>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<4>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} diff --git a/src/traits/words_unpack/impls_for_u64_slice.rs b/src/traits/words_unpack/impls_for_u64_slice.rs new file mode 100644 index 0000000..8d3344f --- /dev/null +++ b/src/traits/words_unpack/impls_for_u64_slice.rs @@ -0,0 +1,9 @@ +use super::WordsUnpack; +use super::funcs_for_unpack_codes_core; + +impl WordsUnpack for [u64] { + #[inline] + fn unpack_codes(&self, codes: &mut [u8]) { + funcs_for_unpack_codes_core::unpack_codes::(self, codes); + } +} diff --git a/src/traits/words_unpack/tests_for_words_unpack.rs b/src/traits/words_unpack/tests_for_words_unpack.rs new file mode 100644 index 0000000..bf291f1 --- /dev/null +++ b/src/traits/words_unpack/tests_for_words_unpack.rs @@ -0,0 +1,37 @@ +use alloc::vec; + +use super::WordsUnpack; +use crate::traits::{WordsPack, layout_block_len}; + +#[test] +fn unpack_matches_codes_for_all_widths_and_layout_tails() { + fn check() { + let mask = crate::code_mask::(); + let codes = (0usize..257) + .map(|index| ((index.wrapping_mul(37) + 11) as u8) & mask) + .collect::>(); + let block_len = layout_block_len::(); + + for requested_len in [ + 0, 1, 15, 16, 31, 32, 63, 64, 65, 79, 80, 95, 96, 127, 128, 129, 257, + ] { + let code_len = requested_len / block_len * block_len; + let word_len = code_len * usize::from(BITS) / crate::WORD_BITS; + let mut words = vec![0; word_len]; + words.as_mut_slice().pack_codes::(&codes[..code_len]); + + let mut actual = vec![u8::MAX; code_len]; + words.as_slice().unpack_codes::(&mut actual); + assert_eq!(actual, codes[..code_len], "BITS={BITS}, len={code_len}"); + } + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} From 8ab8113a64919dbe3286ce1115975766ee8359f9 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 30 Sep 2026 11:50:11 +0800 Subject: [PATCH 28/51] Avoid slow random access for short packed conversion --- benches/packed_conversion.rs | 65 +++++++++++++++++++++++ src/packed_string/impls_for_conversion.rs | 4 ++ 2 files changed, 69 insertions(+) diff --git a/benches/packed_conversion.rs b/benches/packed_conversion.rs index b16296f..7f8aad9 100644 --- a/benches/packed_conversion.rs +++ b/benches/packed_conversion.rs @@ -39,3 +39,68 @@ macro_rules! define_case { } crate::for_each_packed_bench_case!(define_case); + +macro_rules! define_boundary_case { + ($case:ident, $width:ident, $bits:literal, $len:literal) => { + mod $case { + use super::*; + + #[divan::bench(name = concat!("packed_conversion/to_vec_boundary/", stringify!($width), "/", stringify!($case), "/iter_collect"))] + fn iter_collect(bencher: Bencher) { + super::super::collect_via_iter::<$bits>(bencher, $len); + } + + #[divan::bench(name = concat!("packed_conversion/to_vec_boundary/", stringify!($width), "/", stringify!($case), "/to_vec"))] + fn to_vec(bencher: Bencher) { + super::super::convert_to_vec::<$bits>(bencher, $len); + } + } + }; +} + +macro_rules! define_width_boundaries { + ($width:ident, $bits:literal; $($case:ident: $len:literal),+ $(,)?) => { + mod $width { + use super::*; + + $(define_boundary_case!($case, $width, $bits, $len);)+ + } + }; +} + +mod boundaries { + use super::*; + + define_width_boundaries!(bits_1, 1; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_2, 2; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_3, 3; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_4, 4; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_5, 5; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_6, 6; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_7, 7; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_8, 8; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + ); +} diff --git a/src/packed_string/impls_for_conversion.rs b/src/packed_string/impls_for_conversion.rs index a332f3d..c526329 100644 --- a/src/packed_string/impls_for_conversion.rs +++ b/src/packed_string/impls_for_conversion.rs @@ -14,6 +14,10 @@ where debug_assert!(CODE_BATCH_LEN.is_multiple_of(layout_block_len::())); let char_len = self.char_len(); + if char_len < CODE_BATCH_LEN { + return self.iter().collect(); + } + let bulk_code_len = char_len / CODE_BATCH_LEN * CODE_BATCH_LEN; let words = self.bits.words(); let words_per_batch = CODE_BATCH_LEN * usize::from(BITS) / crate::WORD_BITS; From bfd3f12cc4f6b310ef59e1877061c759c1ce60ad Mon Sep 17 00:00:00 2001 From: juncheng Date: Thu, 1 Oct 2026 22:02:07 +0800 Subject: [PATCH 29/51] Unpack complete layout blocks in conversion tail --- src/packed_string/impls_for_conversion.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/packed_string/impls_for_conversion.rs b/src/packed_string/impls_for_conversion.rs index c526329..021db03 100644 --- a/src/packed_string/impls_for_conversion.rs +++ b/src/packed_string/impls_for_conversion.rs @@ -34,7 +34,25 @@ where })); } - result.extend((bulk_code_len..char_len).map(|index| { + let layout_block_len = layout_block_len::(); + let remainder_code_len = char_len - bulk_code_len; + let unpacked_remainder_code_len = remainder_code_len / layout_block_len * layout_block_len; + + if unpacked_remainder_code_len > 0 { + let remainder_word_start = bulk_code_len * usize::from(BITS) / crate::WORD_BITS; + let remainder_word_len = + unpacked_remainder_code_len * usize::from(BITS) / crate::WORD_BITS; + let remainder_word_end = remainder_word_start + remainder_word_len; + words[remainder_word_start..remainder_word_end] + .unpack_codes::(&mut decoded[..unpacked_remainder_code_len]); + + result.extend(decoded[..unpacked_remainder_code_len].iter().map(|&code| { + C::from_code(code).expect("PackedChar rejected a code it previously produced") + })); + } + + let tail_start = bulk_code_len + unpacked_remainder_code_len; + result.extend((tail_start..char_len).map(|index| { self.get(index) .expect("packed character index is within the string") })); From 3803ddb3dffb09bd6bcca323e3eb1f25647274c1 Mon Sep 17 00:00:00 2001 From: juncheng Date: Thu, 1 Oct 2026 22:42:24 +0800 Subject: [PATCH 30/51] Clarify conversion layout length name --- src/packed_string/impls_for_conversion.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packed_string/impls_for_conversion.rs b/src/packed_string/impls_for_conversion.rs index 021db03..5381261 100644 --- a/src/packed_string/impls_for_conversion.rs +++ b/src/packed_string/impls_for_conversion.rs @@ -34,9 +34,9 @@ where })); } - let layout_block_len = layout_block_len::(); + let layout_len = layout_block_len::(); let remainder_code_len = char_len - bulk_code_len; - let unpacked_remainder_code_len = remainder_code_len / layout_block_len * layout_block_len; + let unpacked_remainder_code_len = remainder_code_len / layout_len * layout_len; if unpacked_remainder_code_len > 0 { let remainder_word_start = bulk_code_len * usize::from(BITS) / crate::WORD_BITS; From d4f689dbc33516a423906292192839486f79053c Mon Sep 17 00:00:00 2001 From: juncheng Date: Thu, 1 Oct 2026 23:05:58 +0800 Subject: [PATCH 31/51] Expand AVX2 unpack width support --- benches/packed_conversion.rs | 16 +- .../funcs_for_unpack_codes_core.rs | 13 +- .../funcs_for_unpack_codes_core/avx2.rs | 208 ++++++++++++++++-- .../tests_for_backend_equivalence.rs | 105 +++++++++ 4 files changed, 316 insertions(+), 26 deletions(-) diff --git a/benches/packed_conversion.rs b/benches/packed_conversion.rs index 7f8aad9..9f75637 100644 --- a/benches/packed_conversion.rs +++ b/benches/packed_conversion.rs @@ -73,34 +73,34 @@ mod boundaries { define_width_boundaries!(bits_1, 1; len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, - len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, ); define_width_boundaries!(bits_2, 2; len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, - len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, ); define_width_boundaries!(bits_3, 3; len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, - len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, ); define_width_boundaries!(bits_4, 4; len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, - len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, ); define_width_boundaries!(bits_5, 5; len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, - len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, ); define_width_boundaries!(bits_6, 6; len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, - len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, ); define_width_boundaries!(bits_7, 7; len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, - len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, ); define_width_boundaries!(bits_8, 8; len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, - len_65: 65, len_95: 95, len_127: 127, len_128: 128, len_129: 129, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, ); } diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs index 0642328..92cc603 100644 --- a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs @@ -25,9 +25,18 @@ fn dispatch(words: &[u64], codes: &mut [u8]) { target_feature = "avx2" ))] { + const AVX2_BITS_1_THRESHOLD: usize = 64; + const AVX2_BITS_2_THRESHOLD: usize = 64; const AVX2_BITS_4_THRESHOLD: usize = 64; - - if BITS == 4 && codes.len() >= AVX2_BITS_4_THRESHOLD { + const AVX2_BITS_6_THRESHOLD: usize = 32; + const AVX2_BITS_8_THRESHOLD: usize = 64; + + if (BITS == 1 && codes.len() >= AVX2_BITS_1_THRESHOLD) + || (BITS == 2 && codes.len() >= AVX2_BITS_2_THRESHOLD) + || (BITS == 4 && codes.len() >= AVX2_BITS_4_THRESHOLD) + || (BITS == 6 && codes.len() >= AVX2_BITS_6_THRESHOLD) + || (BITS == 8 && codes.len() >= AVX2_BITS_8_THRESHOLD) + { // SAFETY: This branch is compiled only when AVX2 is enabled, and // the core has validated the complete-block input/output contract. unsafe { avx2::unpack_codes::(words, codes) }; diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs index 1af7b0e..56dc2d8 100644 --- a/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs @@ -1,50 +1,209 @@ +//! AVX2 kernels are retained for BITS=1, BITS=2, BITS=4, BITS=6, and BITS=8. +//! BITS=3, BITS=5, and BITS=7 remain scalar pending width-specific wins. + use super::scalar; #[cfg(target_arch = "x86")] use core::arch::x86::{ - __m128i, __m256i, _mm_storeu_si128, _mm256_and_si256, _mm256_castsi256_si128, - _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_set1_epi8, _mm256_srli_epi16, - _mm256_unpackhi_epi8, _mm256_unpacklo_epi8, + __m128i, __m256i, _mm_and_si128, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_set1_epi8, + _mm_srli_epi16, _mm_srli_si128, _mm_storeu_si128, _mm_unpackhi_epi8, _mm_unpackhi_epi16, + _mm_unpacklo_epi8, _mm_unpacklo_epi16, _mm256_and_si256, _mm256_castsi256_si128, + _mm256_cvtepu8_epi64, _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_or_si256, + _mm256_packus_epi16, _mm256_packus_epi32, _mm256_set1_epi8, _mm256_set1_epi32, + _mm256_set1_epi64x, _mm256_setr_epi8, _mm256_shuffle_epi8, _mm256_slli_epi64, + _mm256_srli_epi16, _mm256_srli_epi32, _mm256_storeu_si256, _mm256_unpackhi_epi8, + _mm256_unpacklo_epi8, }; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{ - __m128i, __m256i, _mm_storeu_si128, _mm256_and_si256, _mm256_castsi256_si128, - _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_set1_epi8, _mm256_srli_epi16, - _mm256_unpackhi_epi8, _mm256_unpacklo_epi8, + __m128i, __m256i, _mm_and_si128, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_set1_epi8, + _mm_srli_epi16, _mm_srli_si128, _mm_storeu_si128, _mm_unpackhi_epi8, _mm_unpackhi_epi16, + _mm_unpacklo_epi8, _mm_unpacklo_epi16, _mm256_and_si256, _mm256_castsi256_si128, + _mm256_cvtepu8_epi64, _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_or_si256, + _mm256_packus_epi16, _mm256_packus_epi32, _mm256_set1_epi8, _mm256_set1_epi32, + _mm256_set1_epi64x, _mm256_setr_epi8, _mm256_shuffle_epi8, _mm256_slli_epi64, + _mm256_srli_epi16, _mm256_srli_epi32, _mm256_storeu_si256, _mm256_unpackhi_epi8, + _mm256_unpacklo_epi8, }; const CODES_PER_BLOCK: usize = 64; -const WORDS_PER_BLOCK: usize = 4; +const BITS_1_WORDS_PER_BLOCK: usize = 1; +const BITS_2_WORDS_PER_BLOCK: usize = 2; +const BITS_4_WORDS_PER_BLOCK: usize = 4; +const BITS_6_CODES_PER_BLOCK: usize = 32; +const BITS_6_WORDS_PER_BLOCK: usize = 3; +const BITS_8_WORDS_PER_BLOCK: usize = 8; -/// AVX2 backend entry for BITS=4. It handles its SIMD prefix and scalar tail. +/// AVX2 backend entry for BITS=1, BITS=2, BITS=4, BITS=6, and BITS=8. +/// It handles its SIMD prefix and scalar tail. /// /// # Safety /// /// The caller must ensure AVX2 is available and that the slices satisfy the -/// complete-layout-block `WordsUnpack` contract for `BITS=4`. +/// complete-layout-block `WordsUnpack` contract for `BITS=1`, `BITS=2`, `BITS=4`, `BITS=6`, or `BITS=8`. #[target_feature(enable = "avx2")] pub(super) unsafe fn unpack_codes(words: &[u64], codes: &mut [u8]) { - debug_assert_eq!(BITS, 4); + if BITS == 6 { + // BITS=6 has a 32-code AVX2 block, two codes per 64-code batch. + let simd_code_len = codes.len() / BITS_6_CODES_PER_BLOCK * BITS_6_CODES_PER_BLOCK; + let simd_word_len = simd_code_len / BITS_6_CODES_PER_BLOCK * BITS_6_WORDS_PER_BLOCK; + for (word_block, code_block) in words[..simd_word_len] + .chunks_exact(BITS_6_WORDS_PER_BLOCK) + .zip(codes[..simd_code_len].chunks_exact_mut(BITS_6_CODES_PER_BLOCK)) + { + // SAFETY: Each block has exactly three initialized words and 32 + // output bytes, as required by the BITS=6 kernel. + unsafe { unpack_32_six_bit(word_block, code_block) }; + } + + scalar::unpack_codes::(&words[simd_word_len..], &mut codes[simd_code_len..]); + return; + } + + debug_assert!(matches!(BITS, 1 | 2 | 4 | 8)); + let words_per_block = match BITS { + 1 => BITS_1_WORDS_PER_BLOCK, + 2 => BITS_2_WORDS_PER_BLOCK, + 4 => BITS_4_WORDS_PER_BLOCK, + 8 => BITS_8_WORDS_PER_BLOCK, + _ => unreachable!(), + }; let simd_code_len = codes.len() / CODES_PER_BLOCK * CODES_PER_BLOCK; - let simd_word_len = simd_code_len / CODES_PER_BLOCK * WORDS_PER_BLOCK; + let simd_word_len = simd_code_len / CODES_PER_BLOCK * words_per_block; for (word_block, code_block) in words[..simd_word_len] - .chunks_exact(WORDS_PER_BLOCK) + .chunks_exact(words_per_block) .zip(codes[..simd_code_len].chunks_exact_mut(CODES_PER_BLOCK)) { - // SAFETY: Each paired block has exactly four initialized words and - // 64 initialized output bytes, as required by the kernel. - unsafe { unpack_64_four_bit(word_block, code_block) }; + // SAFETY: Each paired block has the width-specific initialized input + // words and 64 initialized output bytes required by its kernel. + unsafe { + match BITS { + 1 => unpack_64_one_bit(word_block, code_block), + 2 => unpack_64_two_bit(word_block, code_block), + 4 => unpack_64_four_bit(word_block, code_block), + 8 => unpack_64_eight_bit(word_block, code_block), + _ => unreachable!(), + } + }; } scalar::unpack_codes::(&words[simd_word_len..], &mut codes[simd_code_len..]); } +#[target_feature(enable = "avx2")] +unsafe fn unpack_32_six_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), 3); + debug_assert_eq!(codes.len(), 32); + + let mut groups = [0u32; 8]; + for (group_index, group) in groups.iter_mut().enumerate() { + let bit_position = group_index * 24; + let word_index = bit_position / 64; + let bit_offset = bit_position % 64; + let mut packed = words[word_index] >> bit_offset; + if bit_offset + 24 > 64 { + packed |= words[word_index + 1] << (64 - bit_offset); + } + *group = packed as u32 & 0x00ff_ffff; + } + + // SAFETY: `groups` contains eight initialized 24-bit groups. + let packed = unsafe { _mm256_loadu_si256(groups.as_ptr().cast::<__m256i>()) }; + let code_mask = _mm256_set1_epi32(0x3f); + let first = _mm256_and_si256(packed, code_mask); + let second = _mm256_and_si256(_mm256_srli_epi32::<6>(packed), code_mask); + let third = _mm256_and_si256(_mm256_srli_epi32::<12>(packed), code_mask); + let fourth = _mm256_and_si256(_mm256_srli_epi32::<18>(packed), code_mask); + + let first_pair = _mm256_packus_epi32(first, second); + let second_pair = _mm256_packus_epi32(third, fourth); + let column_major = _mm256_packus_epi16(first_pair, second_pair); + let transpose = _mm256_setr_epi8( + 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, // + 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, + ); + let output = _mm256_shuffle_epi8(column_major, transpose); + + // SAFETY: `output` contains exactly 32 codes and the output block has + // exactly 32 initialized bytes. + unsafe { _mm256_storeu_si256(codes.as_mut_ptr().cast::<__m256i>(), output) }; +} + +#[target_feature(enable = "avx2")] +unsafe fn unpack_64_one_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), BITS_1_WORDS_PER_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_BLOCK); + + let packed = _mm_cvtsi64_si128(words[0] as i64); + let low = _mm256_cvtepu8_epi64(packed); + let high = _mm256_cvtepu8_epi64(_mm_srli_si128::<4>(packed)); + let spread_mask = _mm256_set1_epi64x(0x0101_0101_0101_0101); + let low = spread_one_bit_bytes(low, spread_mask); + let high = spread_one_bit_bytes(high, spread_mask); + + // SAFETY: Each vector contains exactly 32 output bytes, and the output + // slice contains exactly 64 initialized bytes. + unsafe { + _mm256_storeu_si256(codes.as_mut_ptr().cast::<__m256i>(), low); + _mm256_storeu_si256(codes.as_mut_ptr().add(32).cast::<__m256i>(), high); + } +} + +#[target_feature(enable = "avx2")] +fn spread_one_bit_bytes(mut bits: __m256i, output_mask: __m256i) -> __m256i { + bits = _mm256_and_si256( + _mm256_or_si256(bits, _mm256_slli_epi64::<28>(bits)), + _mm256_set1_epi64x(0x0000_000f_0000_000f), + ); + bits = _mm256_and_si256( + _mm256_or_si256(bits, _mm256_slli_epi64::<14>(bits)), + _mm256_set1_epi64x(0x0003_0003_0003_0003), + ); + _mm256_and_si256( + _mm256_or_si256(bits, _mm256_slli_epi64::<7>(bits)), + output_mask, + ) +} + +#[target_feature(enable = "avx2")] +unsafe fn unpack_64_two_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), BITS_2_WORDS_PER_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_BLOCK); + + let code_mask = _mm_set1_epi8(0x03); + // SAFETY: Two words contain exactly 16 initialized packed bytes. + let packed = unsafe { _mm_loadu_si128(words.as_ptr().cast::<__m128i>()) }; + let first = _mm_and_si128(packed, code_mask); + let second = _mm_and_si128(_mm_srli_epi16::<2>(packed), code_mask); + let third = _mm_and_si128(_mm_srli_epi16::<4>(packed), code_mask); + let fourth = _mm_and_si128(_mm_srli_epi16::<6>(packed), code_mask); + + let first_pair = _mm_unpacklo_epi8(first, second); + let second_pair = _mm_unpacklo_epi8(third, fourth); + let third_pair = _mm_unpackhi_epi8(first, second); + let fourth_pair = _mm_unpackhi_epi8(third, fourth); + + // Interleave four 2-bit codes from each packed byte into sequential order. + let output0 = _mm_unpacklo_epi16(first_pair, second_pair); + let output1 = _mm_unpackhi_epi16(first_pair, second_pair); + let output2 = _mm_unpacklo_epi16(third_pair, fourth_pair); + let output3 = _mm_unpackhi_epi16(third_pair, fourth_pair); + + // SAFETY: Four stores write exactly 64 bytes to the output slice. + unsafe { + _mm_storeu_si128(codes.as_mut_ptr().cast::<__m128i>(), output0); + _mm_storeu_si128(codes.as_mut_ptr().add(16).cast::<__m128i>(), output1); + _mm_storeu_si128(codes.as_mut_ptr().add(32).cast::<__m128i>(), output2); + _mm_storeu_si128(codes.as_mut_ptr().add(48).cast::<__m128i>(), output3); + } +} + #[target_feature(enable = "avx2")] unsafe fn unpack_64_four_bit(words: &[u64], codes: &mut [u8]) { - debug_assert_eq!(words.len(), WORDS_PER_BLOCK); + debug_assert_eq!(words.len(), BITS_4_WORDS_PER_BLOCK); debug_assert_eq!(codes.len(), CODES_PER_BLOCK); let nibble_mask = _mm256_set1_epi8(0x0f); @@ -75,3 +234,20 @@ unsafe fn unpack_64_four_bit(words: &[u64], codes: &mut [u8]) { ); } } + +#[target_feature(enable = "avx2")] +unsafe fn unpack_64_eight_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), BITS_8_WORDS_PER_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_BLOCK); + + // SAFETY: Eight words and 64 output bytes are initialized, so each load + // and store accesses exactly 32 bytes within its slice. + unsafe { + let input = words.as_ptr().cast::<__m256i>(); + let output = codes.as_mut_ptr().cast::<__m256i>(); + let first = _mm256_loadu_si256(input); + let second = _mm256_loadu_si256(input.add(1)); + _mm256_storeu_si256(output, first); + _mm256_storeu_si256(output.add(1), second); + } +} diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs index 47b6254..b455881 100644 --- a/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs @@ -44,3 +44,108 @@ fn avx2_bits_four_matches_scalar_across_prefix_and_tail() { assert_eq!(actual, expected, "code_len={code_len}"); } } + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_one_matches_scalar_across_blocks() { + for code_len in [0, 64, 128, 192, 256] { + let input = (0..code_len) + .map(|index: usize| ((index.wrapping_mul(29) + 7) as u8) & 0x01) + .collect::>(); + let words_len = code_len / 64; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<1>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<1>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<1>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_two_matches_scalar_across_prefix_and_tail() { + for code_len in [0, 32, 64, 96, 128, 160, 192] { + let input = (0..code_len) + .map(|index: usize| ((index.wrapping_mul(29) + 7) as u8) & 0x03) + .collect::>(); + let words_len = code_len / 32; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<2>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<2>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<2>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_eight_matches_scalar_across_prefix_and_tail() { + for code_len in [0, 8, 16, 32, 56, 64, 72, 96, 120, 128, 136] { + let input = (0..code_len) + .map(|index: usize| (index.wrapping_mul(29) + 7) as u8) + .collect::>(); + let words_len = code_len / 8; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<8>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<8>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<8>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_six_matches_scalar_across_prefix_and_tail() { + let mask = crate::code_mask::<6>(); + for code_len in [0, 32, 64, 96, 128, 160] { + let input = (0..code_len) + .map(|index: usize| (index.wrapping_mul(29) + 7) as u8 & mask) + .collect::>(); + let words_len = code_len * 6 / crate::WORD_BITS; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<6>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<6>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<6>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} From 6d3e690100e96d2bfc34f91c27055c7f3aa79a78 Mon Sep 17 00:00:00 2001 From: juncheng Date: Thu, 1 Oct 2026 23:18:41 +0800 Subject: [PATCH 32/51] Strengthen AVX2 unpack regression coverage --- .../funcs_for_unpack_codes_core.rs | 18 +++++--- .../funcs_for_unpack_codes_core/avx2.rs | 5 ++- .../tests_for_backend_equivalence.rs | 43 ++++++++++++++++++- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs index 92cc603..4b779c2 100644 --- a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs @@ -25,18 +25,24 @@ fn dispatch(words: &[u64], codes: &mut [u8]) { target_feature = "avx2" ))] { + // Thresholds match complete AVX2 kernel blocks. BITS=6 uses its + // 32-code block; the other retained kernels process 64 codes. const AVX2_BITS_1_THRESHOLD: usize = 64; const AVX2_BITS_2_THRESHOLD: usize = 64; const AVX2_BITS_4_THRESHOLD: usize = 64; const AVX2_BITS_6_THRESHOLD: usize = 32; const AVX2_BITS_8_THRESHOLD: usize = 64; - if (BITS == 1 && codes.len() >= AVX2_BITS_1_THRESHOLD) - || (BITS == 2 && codes.len() >= AVX2_BITS_2_THRESHOLD) - || (BITS == 4 && codes.len() >= AVX2_BITS_4_THRESHOLD) - || (BITS == 6 && codes.len() >= AVX2_BITS_6_THRESHOLD) - || (BITS == 8 && codes.len() >= AVX2_BITS_8_THRESHOLD) - { + let use_avx2 = match BITS { + 1 => codes.len() >= AVX2_BITS_1_THRESHOLD, + 2 => codes.len() >= AVX2_BITS_2_THRESHOLD, + 4 => codes.len() >= AVX2_BITS_4_THRESHOLD, + 6 => codes.len() >= AVX2_BITS_6_THRESHOLD, + 8 => codes.len() >= AVX2_BITS_8_THRESHOLD, + _ => false, + }; + + if use_avx2 { // SAFETY: This branch is compiled only when AVX2 is enabled, and // the core has validated the complete-block input/output contract. unsafe { avx2::unpack_codes::(words, codes) }; diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs index 56dc2d8..6f1f2f4 100644 --- a/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs @@ -1,5 +1,6 @@ //! AVX2 kernels are retained for BITS=1, BITS=2, BITS=4, BITS=6, and BITS=8. -//! BITS=3, BITS=5, and BITS=7 remain scalar pending width-specific wins. +//! BITS=3, BITS=5, and BITS=7 intentionally remain scalar: a tested generic +//! AVX2 gather regressed bulk conversion. Re-benchmark before adding kernels. use super::scalar; @@ -45,7 +46,7 @@ const BITS_8_WORDS_PER_BLOCK: usize = 8; #[target_feature(enable = "avx2")] pub(super) unsafe fn unpack_codes(words: &[u64], codes: &mut [u8]) { if BITS == 6 { - // BITS=6 has a 32-code AVX2 block, two codes per 64-code batch. + // BITS=6 has a 32-code AVX2 block, two blocks per 64-code batch. let simd_code_len = codes.len() / BITS_6_CODES_PER_BLOCK * BITS_6_CODES_PER_BLOCK; let simd_word_len = simd_code_len / BITS_6_CODES_PER_BLOCK * BITS_6_WORDS_PER_BLOCK; for (word_block, code_block) in words[..simd_word_len] diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs index b455881..abd56f5 100644 --- a/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs @@ -14,6 +14,34 @@ use crate::traits::WordsPack; ))] use alloc::vec; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +fn assert_avx2_matches_scalar_for_each_code_position(code_len: usize) { + let code_mask = crate::code_mask::(); + + for position in 0..code_len { + let mut input = vec![0; code_len]; + input[position] = code_mask; + + let words_len = code_len * usize::from(BITS) / crate::WORD_BITS; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::(&words, &mut expected); + assert_eq!(expected, input, "scalar decode at position={position}"); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::(&words, &mut actual) }; + + assert_eq!(actual, input, "AVX2 decode at position={position}"); + } +} + #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx2" @@ -128,7 +156,7 @@ fn avx2_bits_eight_matches_scalar_across_prefix_and_tail() { target_feature = "avx2" ))] #[test] -fn avx2_bits_six_matches_scalar_across_prefix_and_tail() { +fn avx2_bits_six_matches_scalar_across_blocks() { let mask = crate::code_mask::<6>(); for code_len in [0, 32, 64, 96, 128, 160] { let input = (0..code_len) @@ -149,3 +177,16 @@ fn avx2_bits_six_matches_scalar_across_prefix_and_tail() { assert_eq!(actual, expected, "code_len={code_len}"); } } + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_unpack_preserves_each_code_position() { + assert_avx2_matches_scalar_for_each_code_position::<1>(64); + assert_avx2_matches_scalar_for_each_code_position::<2>(64); + assert_avx2_matches_scalar_for_each_code_position::<4>(64); + assert_avx2_matches_scalar_for_each_code_position::<6>(32); + assert_avx2_matches_scalar_for_each_code_position::<8>(64); +} From 5ce70129d904b39139cf0e6f1a218272c3fa4a06 Mon Sep 17 00:00:00 2001 From: juncheng Date: Thu, 1 Oct 2026 23:29:23 +0800 Subject: [PATCH 33/51] Use bulk unpack only with accelerated backend --- src/packed_string/impls_for_conversion.rs | 6 +- src/traits/words_unpack.rs | 29 +++++++ .../funcs_for_unpack_codes_core.rs | 76 +++++++++++++++---- 3 files changed, 94 insertions(+), 17 deletions(-) diff --git a/src/packed_string/impls_for_conversion.rs b/src/packed_string/impls_for_conversion.rs index 5381261..dfe3e87 100644 --- a/src/packed_string/impls_for_conversion.rs +++ b/src/packed_string/impls_for_conversion.rs @@ -1,4 +1,4 @@ -use crate::traits::{WordsUnpack, layout_block_len}; +use crate::traits::{WordsUnpack, layout_block_len, prefer_bulk_unpack}; use super::*; @@ -11,6 +11,10 @@ where // 64 codes is a multiple of every layout block for BITS=1..=8. const CODE_BATCH_LEN: usize = 64; + if !prefer_bulk_unpack::() { + return self.iter().collect(); + } + debug_assert!(CODE_BATCH_LEN.is_multiple_of(layout_block_len::())); let char_len = self.char_len(); diff --git a/src/traits/words_unpack.rs b/src/traits/words_unpack.rs index a263943..6a67893 100644 --- a/src/traits/words_unpack.rs +++ b/src/traits/words_unpack.rs @@ -10,5 +10,34 @@ pub(crate) trait WordsUnpack { pub(crate) mod funcs_for_unpack_codes_core; mod impls_for_u64_slice; +pub(crate) use funcs_for_unpack_codes_core::prefer_bulk_unpack; + #[cfg(test)] mod tests_for_words_unpack; + +#[cfg(test)] +mod tests_for_bulk_preference { + use super::prefer_bulk_unpack; + + #[test] + fn preference_matches_available_avx2_widths() { + let has_avx2 = cfg!(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + )); + + for (bits, preferred) in [ + (1, prefer_bulk_unpack::<1>()), + (2, prefer_bulk_unpack::<2>()), + (3, prefer_bulk_unpack::<3>()), + (4, prefer_bulk_unpack::<4>()), + (5, prefer_bulk_unpack::<5>()), + (6, prefer_bulk_unpack::<6>()), + (7, prefer_bulk_unpack::<7>()), + (8, prefer_bulk_unpack::<8>()), + ] { + let has_kernel = matches!(bits, 1 | 2 | 4 | 6 | 8); + assert_eq!(preferred, has_avx2 && has_kernel, "BITS={bits}"); + } + } +} diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs index 4b779c2..8606291 100644 --- a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs @@ -1,5 +1,64 @@ use crate::{WORD_BITS, assert_valid_width, traits::layout_block_len}; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_1_THRESHOLD: usize = 64; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_2_THRESHOLD: usize = 64; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_4_THRESHOLD: usize = 64; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_6_THRESHOLD: usize = 32; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_8_THRESHOLD: usize = 64; + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const fn avx2_threshold() -> Option { + match BITS { + 1 => Some(AVX2_BITS_1_THRESHOLD), + 2 => Some(AVX2_BITS_2_THRESHOLD), + 4 => Some(AVX2_BITS_4_THRESHOLD), + 6 => Some(AVX2_BITS_6_THRESHOLD), + 8 => Some(AVX2_BITS_8_THRESHOLD), + _ => None, + } +} + +pub(crate) const fn prefer_bulk_unpack() -> bool { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + return avx2_threshold::().is_some(); + } + + #[cfg(not(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + )))] + { + false + } +} + #[inline] pub(super) fn unpack_codes(words: &[u64], codes: &mut [u8]) { assert_valid_width::(); @@ -27,22 +86,7 @@ fn dispatch(words: &[u64], codes: &mut [u8]) { { // Thresholds match complete AVX2 kernel blocks. BITS=6 uses its // 32-code block; the other retained kernels process 64 codes. - const AVX2_BITS_1_THRESHOLD: usize = 64; - const AVX2_BITS_2_THRESHOLD: usize = 64; - const AVX2_BITS_4_THRESHOLD: usize = 64; - const AVX2_BITS_6_THRESHOLD: usize = 32; - const AVX2_BITS_8_THRESHOLD: usize = 64; - - let use_avx2 = match BITS { - 1 => codes.len() >= AVX2_BITS_1_THRESHOLD, - 2 => codes.len() >= AVX2_BITS_2_THRESHOLD, - 4 => codes.len() >= AVX2_BITS_4_THRESHOLD, - 6 => codes.len() >= AVX2_BITS_6_THRESHOLD, - 8 => codes.len() >= AVX2_BITS_8_THRESHOLD, - _ => false, - }; - - if use_avx2 { + if avx2_threshold::().is_some_and(|threshold| codes.len() >= threshold) { // SAFETY: This branch is compiled only when AVX2 is enabled, and // the core has validated the complete-block input/output contract. unsafe { avx2::unpack_codes::(words, codes) }; From 8ab293f516ef45e322331cc7977ffd6b41cf5496 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 7 Oct 2026 08:50:31 +0800 Subject: [PATCH 34/51] Clarify bulk unpack capability naming --- src/packed_string/impls_for_conversion.rs | 4 +-- .../funcs_for_copy_words_shifted_core/avx2.rs | 4 +++ src/traits/words_unpack.rs | 28 +++++++++---------- .../funcs_for_unpack_codes_core.rs | 2 +- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/packed_string/impls_for_conversion.rs b/src/packed_string/impls_for_conversion.rs index dfe3e87..e93b162 100644 --- a/src/packed_string/impls_for_conversion.rs +++ b/src/packed_string/impls_for_conversion.rs @@ -1,4 +1,4 @@ -use crate::traits::{WordsUnpack, layout_block_len, prefer_bulk_unpack}; +use crate::traits::{WordsUnpack, has_accelerated_bulk_unpack, layout_block_len}; use super::*; @@ -11,7 +11,7 @@ where // 64 codes is a multiple of every layout block for BITS=1..=8. const CODE_BATCH_LEN: usize = 64; - if !prefer_bulk_unpack::() { + if !has_accelerated_bulk_unpack::() { return self.iter().collect(); } diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs index aa69a99..16c59ce 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs @@ -1,3 +1,5 @@ +//! AVX2 backend for copying shifted 64-bit windows. + use crate::WORD_BITS; #[cfg(target_arch = "x86")] @@ -19,6 +21,8 @@ pub(super) unsafe fn copy_words_shifted(dst: &mut [u64], src: &[u64], len: usize let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; let mut i = 0; while i + 4 <= len { + // The loads overlap by one word: lane j combines src[i + j] with + // src[i + j + 1], producing four consecutive shifted windows. // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm256_loadu_si256` uses unaligned loads so alignment is not required. let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; // SAFETY: Same as above. diff --git a/src/traits/words_unpack.rs b/src/traits/words_unpack.rs index 6a67893..1359ca1 100644 --- a/src/traits/words_unpack.rs +++ b/src/traits/words_unpack.rs @@ -10,34 +10,34 @@ pub(crate) trait WordsUnpack { pub(crate) mod funcs_for_unpack_codes_core; mod impls_for_u64_slice; -pub(crate) use funcs_for_unpack_codes_core::prefer_bulk_unpack; +pub(crate) use funcs_for_unpack_codes_core::has_accelerated_bulk_unpack; #[cfg(test)] mod tests_for_words_unpack; #[cfg(test)] -mod tests_for_bulk_preference { - use super::prefer_bulk_unpack; +mod tests_for_bulk_backend_availability { + use super::has_accelerated_bulk_unpack; #[test] - fn preference_matches_available_avx2_widths() { + fn availability_matches_compiled_avx2_widths() { let has_avx2 = cfg!(all( any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx2" )); - for (bits, preferred) in [ - (1, prefer_bulk_unpack::<1>()), - (2, prefer_bulk_unpack::<2>()), - (3, prefer_bulk_unpack::<3>()), - (4, prefer_bulk_unpack::<4>()), - (5, prefer_bulk_unpack::<5>()), - (6, prefer_bulk_unpack::<6>()), - (7, prefer_bulk_unpack::<7>()), - (8, prefer_bulk_unpack::<8>()), + for (bits, has_backend) in [ + (1, has_accelerated_bulk_unpack::<1>()), + (2, has_accelerated_bulk_unpack::<2>()), + (3, has_accelerated_bulk_unpack::<3>()), + (4, has_accelerated_bulk_unpack::<4>()), + (5, has_accelerated_bulk_unpack::<5>()), + (6, has_accelerated_bulk_unpack::<6>()), + (7, has_accelerated_bulk_unpack::<7>()), + (8, has_accelerated_bulk_unpack::<8>()), ] { let has_kernel = matches!(bits, 1 | 2 | 4 | 6 | 8); - assert_eq!(preferred, has_avx2 && has_kernel, "BITS={bits}"); + assert_eq!(has_backend, has_avx2 && has_kernel, "BITS={bits}"); } } } diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs index 8606291..eec2be2 100644 --- a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs @@ -41,7 +41,7 @@ const fn avx2_threshold() -> Option { } } -pub(crate) const fn prefer_bulk_unpack() -> bool { +pub(crate) const fn has_accelerated_bulk_unpack() -> bool { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx2" From b31ec0368211bdcd74491b6fb8050faf20ae8f2b Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 7 Oct 2026 09:15:46 +0800 Subject: [PATCH 35/51] Use unchecked word refill in packed iterator --- src/packed_string/impls_for_iter.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index 68aaa34..b2d890a 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -59,7 +59,12 @@ where code } else { let old_fill = self.front_pending_bit_count; - let next_word = words[self.front_next_word_index]; + // SAFETY: This refill branch requires `old_fill < width`, and + // `front < back`. The cursor invariant gives + // `front_next_word_index * WORD_BITS = front * BITS + old_fill`, + // which is less than `(front + 1) * BITS <= back * BITS <= + // words.len() * WORD_BITS`; therefore this word index is in bounds. + let next_word = unsafe { *words.get_unchecked(self.front_next_word_index) }; self.front_next_word_index += 1; let code = self.front_pending_bits | (next_word << old_fill); From b9f3b5f1234cf23f877df60bf3ba0f6b22b6063f Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 7 Oct 2026 19:03:12 +0800 Subject: [PATCH 36/51] Use unchecked word loads for packed get --- src/funcs_for_packed.rs | 24 ++++++++-- src/packed_str/impls_for_access.rs | 27 ++++++----- .../impls_for_access/tests_for_access.rs | 45 +++++++++++++++++++ src/packed_string/impls_for_access.rs | 17 +++---- 4 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 src/packed_str/impls_for_access/tests_for_access.rs diff --git a/src/funcs_for_packed.rs b/src/funcs_for_packed.rs index d4fea12..ff6eb22 100644 --- a/src/funcs_for_packed.rs +++ b/src/funcs_for_packed.rs @@ -17,16 +17,34 @@ pub(crate) const fn code_mask() -> u8 { } /// Extracts one fixed-width packed code from a word-aligned or offset view. +/// +/// # Safety +/// +/// `BITS` must be in `1..=8`, `bit_start` must be a multiple of `BITS`, and the +/// complete `BITS`-wide code at `bit_start + index * BITS` must lie within the +/// logical bit range represented by `words`. The arithmetic for that range +/// must fit in `usize`. These conditions ensure that the first backing word +/// exists and, when the code crosses a word boundary, that the following +/// backing word exists as well. #[inline] -pub(crate) fn extract_code(words: &[u64], bit_start: usize, index: usize) -> u8 { +pub(crate) unsafe fn extract_code_unchecked( + words: &[u64], + bit_start: usize, + index: usize, +) -> u8 { let bits = usize::from(BITS); let bit_index = bit_start + index * bits; let word_index = bit_index / 64; let offset = bit_index % 64; - let mut code = words[word_index] >> offset; + // SAFETY: The caller guarantees that the complete code lies in the logical + // bit range, so its first backing word is present. + let mut code = unsafe { *words.get_unchecked(word_index) } >> offset; if 64 % bits != 0 && offset + bits > 64 { - code |= words[word_index + 1] << (64 - offset); + // SAFETY: This branch is taken only when the code crosses the current + // word boundary. Since the complete code lies in the logical bit + // range, the following backing word is present. + code |= unsafe { *words.get_unchecked(word_index + 1) } << (64 - offset); } (code & u64::from(code_mask::())) as u8 diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs index 16bb2bd..e9fdbb9 100644 --- a/src/packed_str/impls_for_access.rs +++ b/src/packed_str/impls_for_access.rs @@ -1,5 +1,5 @@ use super::*; -use crate::extract_code; +use crate::extract_code_unchecked; impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> where @@ -17,10 +17,19 @@ where if index >= self.char_len() { return None; } - Some( - C::from_code(self.code_at(index)) - .expect("PackedChar rejected a code in a PackedStr invariant"), - ) + // SAFETY: `PackedStr` maintains a character-aligned BitStr start, so + // `self.bits.start()` is a multiple of BITS as required by the helper. + // `index < char_len()` implies + // `(index + 1) * BITS <= self.bits.bit_len()`, so the complete code lies + // inside this view. The BitStr invariant places + // `[self.bits.start(), self.bits.start() + self.bits.bit_len())` inside + // the source bit range. Therefore the code's first word and, when it + // crosses a word boundary, its successor are present in the source + // words, including when the view starts at a non-word offset. + let code = unsafe { + extract_code_unchecked::(self.bits.source().words(), self.bits.start(), index) + }; + Some(C::from_code(code).expect("PackedChar rejected a code in a PackedStr invariant")) } pub fn first(&self) -> Option { @@ -32,9 +41,7 @@ where .checked_sub(1) .and_then(|index| self.get(index)) } - - #[inline] - fn code_at(&self, index: usize) -> u8 { - extract_code::(self.bits.source().words(), self.bits.start(), index) - } } + +#[cfg(test)] +mod tests_for_access; diff --git a/src/packed_str/impls_for_access/tests_for_access.rs b/src/packed_str/impls_for_access/tests_for_access.rs new file mode 100644 index 0000000..54bb21c --- /dev/null +++ b/src/packed_str/impls_for_access/tests_for_access.rs @@ -0,0 +1,45 @@ +use crate::{PackedStr, PackedString, traits::PackedChar}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct TestChar(u8); + +impl PackedChar for TestChar { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} + +fn check_access() { + let mask = (1u16 << BITS) - 1; + let values: alloc::vec::Vec<_> = (0..96) + .map(|index| TestChar(((index * 29 + 7) as u16 & mask) as u8)) + .collect(); + let packed = PackedString::::from_chars(values.iter().copied()); + let view: PackedStr<'_, TestChar, BITS> = packed.as_packed_str().slice_from(1); + assert_eq!(view.bits.start() % usize::from(BITS), 0); + assert_ne!(view.bits.start() % 64, 0); + + for (index, expected) in values.iter().copied().skip(1).enumerate() { + assert_eq!(packed.get(index + 1), Some(expected)); + assert_eq!(view.get(index), Some(expected)); + } + + assert_eq!(view.get(view.char_len()), None); + assert_eq!(packed.get(packed.char_len()), None); +} + +#[test] +fn checked_get_supports_all_widths_and_offset_views() { + check_access::<1>(); + check_access::<2>(); + check_access::<3>(); + check_access::<4>(); + check_access::<5>(); + check_access::<6>(); + check_access::<7>(); + check_access::<8>(); +} diff --git a/src/packed_string/impls_for_access.rs b/src/packed_string/impls_for_access.rs index 9f6d241..64b7b93 100644 --- a/src/packed_string/impls_for_access.rs +++ b/src/packed_string/impls_for_access.rs @@ -1,5 +1,5 @@ use super::*; -use crate::extract_code; +use crate::extract_code_unchecked; impl PackedString where @@ -31,10 +31,12 @@ where if index >= self.char_len() { return None; } - Some( - C::from_code(self.code_at(index)) - .expect("PackedChar rejected a code it previously produced"), - ) + // SAFETY: `index < char_len()` implies + // `(index + 1) * BITS <= self.bits.bit_len()`, so this character's + // complete range lies in the owned packed bit range. Its first word, + // and the next word if it crosses a boundary, are therefore present. + let code = unsafe { extract_code_unchecked::(self.bits.words(), 0, index) }; + Some(C::from_code(code).expect("PackedChar rejected a code it previously produced")) } #[inline] @@ -46,11 +48,6 @@ where pub fn last(&self) -> Option { self.char_len().checked_sub(1).and_then(|i| self.get(i)) } - - #[inline] - fn code_at(&self, index: usize) -> u8 { - extract_code::(self.bits.words(), 0, index) - } } #[cfg(test)] From eeef026dbcd5409b84fbdc10a9feb7eba19611fe Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 7 Oct 2026 19:19:35 +0800 Subject: [PATCH 37/51] Document packed view alignment invariants --- src/packed_str.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/packed_str.rs b/src/packed_str.rs index 507a3af..aee5b5b 100644 --- a/src/packed_str.rs +++ b/src/packed_str.rs @@ -5,6 +5,13 @@ use core::marker::PhantomData; use crate::{BitStr, PackedString, traits::PackedChar}; /// A character-aligned borrowed view into a [`PackedString`]. +/// +/// The view may be unaligned to a `u64` word boundary, but never to a packed +/// character boundary. Its representation maintains these invariants: +/// +/// - `bits.start()` is a multiple of `BITS`; +/// - `bits.bit_len()` is a multiple of `BITS`; +/// - the bit range lies within its source [`BitString`](crate::BitString). #[derive(Clone, Copy)] pub struct PackedStr<'ps, C, const BITS: u8> where From 621718c0d319e9daec99ef1a25b41d96c8e0ef42 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 7 Oct 2026 19:33:30 +0800 Subject: [PATCH 38/51] Optimize packed view iteration with rolling cursor --- src/packed_str/impls_for_iter.rs | 79 ++++++++++++++-- .../impls_for_iter/tests_for_iter.rs | 90 +++++++++++++++++++ 2 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 src/packed_str/impls_for_iter/tests_for_iter.rs diff --git a/src/packed_str/impls_for_iter.rs b/src/packed_str/impls_for_iter.rs index 8eb20d6..10e0ecf 100644 --- a/src/packed_str/impls_for_iter.rs +++ b/src/packed_str/impls_for_iter.rs @@ -1,5 +1,7 @@ use core::iter::FusedIterator; +use crate::{WORD_BITS, code_mask}; + use super::*; impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> @@ -7,14 +9,46 @@ where C: PackedChar, { pub fn iter(&self) -> Iter<'ps, C, BITS> { + let char_len = self.char_len(); + let bit_start = self.bits.start(); + let front_cursor = if char_len == 0 { + FrontCursor { + pending_bits: 0, + pending_bit_count: 0, + next_word_index: 0, + } + } else { + let word_index = bit_start / WORD_BITS; + let bit_offset = bit_start % WORD_BITS; + let first_word = self.bits.source().words()[word_index]; + FrontCursor { + pending_bits: first_word >> bit_offset, + pending_bit_count: WORD_BITS - bit_offset, + next_word_index: word_index + 1, + } + }; + Iter { view: *self, front: 0, - back: self.char_len(), + back: char_len, + front_cursor, } } } +struct FrontCursor { + // Loaded but unconsumed bits; the next code starts at bit 0. + pending_bits: u64, + // Includes source bits past this view's end; front/back prevents yielding them. + pending_bit_count: usize, + // Index of the next backing word not yet loaded into `pending_bits`. + next_word_index: usize, + // For a non-empty iterator, with `Iter::front` and the view start: + // `view.bits.start() + front * BITS + pending_bit_count` + // equals `next_word_index * WORD_BITS`. +} + #[allow(dead_code)] pub struct Iter<'ps, C, const BITS: u8> where @@ -23,6 +57,7 @@ where view: PackedStr<'ps, C, BITS>, front: usize, back: usize, + front_cursor: FrontCursor, } impl Iterator for Iter<'_, C, BITS> @@ -33,12 +68,41 @@ where fn next(&mut self) -> Option { if self.front == self.back { - None - } else { - let c = self.view.get(self.front); - self.front += 1; - c + return None; } + + let width = usize::from(BITS); + let words = self.view.bits.source().words(); + let code = if self.front_cursor.pending_bit_count >= width { + let code = self.front_cursor.pending_bits; + self.front_cursor.pending_bits >>= width; + self.front_cursor.pending_bit_count -= width; + code + } else { + let old_fill = self.front_cursor.pending_bit_count; + // SAFETY: This refill branch requires `old_fill < width`, and + // `front < back`. The cursor invariant gives + // `front_cursor.next_word_index * WORD_BITS = view.bits.start() + // + front * BITS + old_fill`, which is less than + // `view.bits.start() + (front + 1) * BITS + // <= view.bits.start() + back * BITS + // <= view.bits.start() + view.bits.bit_len() + // <= words.len() * WORD_BITS`. Thus this word index is in bounds. + let next_word = unsafe { *words.get_unchecked(self.front_cursor.next_word_index) }; + self.front_cursor.next_word_index += 1; + + let code = self.front_cursor.pending_bits | (next_word << old_fill); + let used_from_next = width - old_fill; + self.front_cursor.pending_bits = next_word >> used_from_next; + self.front_cursor.pending_bit_count = WORD_BITS - used_from_next; + code + }; + + let character = C::from_code((code & u64::from(code_mask::())) as u8) + .expect("PackedChar rejected a code in a PackedStr invariant"); + self.front += 1; + + Some(character) } fn size_hint(&self) -> (usize, Option) { @@ -75,3 +139,6 @@ where self.iter() } } + +#[cfg(test)] +mod tests_for_iter; diff --git a/src/packed_str/impls_for_iter/tests_for_iter.rs b/src/packed_str/impls_for_iter/tests_for_iter.rs new file mode 100644 index 0000000..ee6bf33 --- /dev/null +++ b/src/packed_str/impls_for_iter/tests_for_iter.rs @@ -0,0 +1,90 @@ +use crate::{PackedStr, PackedString, WORD_BITS, traits::PackedChar}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} + +fn assert_front_window(iter: &super::Iter<'_, Code, BITS>, front: usize) { + assert_eq!( + iter.view.bits.start() + front * usize::from(BITS) + iter.front_cursor.pending_bit_count, + iter.front_cursor.next_word_index * WORD_BITS + ); + assert!(iter.front_cursor.pending_bit_count <= WORD_BITS); +} + +fn check_mixed_iteration() { + let width = usize::from(BITS); + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let codes_per_word = WORD_BITS / width; + let lengths = [ + 0, + 1, + codes_per_word.saturating_sub(1), + codes_per_word, + codes_per_word + 1, + codes_per_word * 2 - 1, + codes_per_word * 2, + codes_per_word * 2 + 1, + 95, + ]; + + for len in lengths { + let expected = (0..len) + .map(|index| Code(((index * 17 + 3) as u8) & mask)) + .collect::>(); + let mut source = alloc::vec::Vec::with_capacity(len + 1); + source.push(Code(0)); + source.extend_from_slice(&expected); + let owner = PackedString::::from_chars(source); + let view: PackedStr<'_, Code, BITS> = owner.as_packed_str().slice_from(1); + let mut iter = view.iter(); + let mut front = 0; + let mut back = len; + + if len > 0 { + assert_eq!(view.bits.start() % width, 0); + assert_ne!(view.bits.start() % WORD_BITS, 0); + assert_front_window(&iter, front); + } + + while front < back { + assert_eq!(iter.next(), Some(expected[front])); + front += 1; + assert_front_window(&iter, front); + if front < back { + back -= 1; + assert_eq!(iter.next_back(), Some(expected[back])); + assert_front_window(&iter, front); + } + } + + assert_eq!(iter.next(), None); + assert_eq!(iter.next_back(), None); + assert_eq!(iter.len(), 0); + } +} + +#[test] +fn rolling_window_handles_offset_views_and_mixed_iteration() { + check_mixed_iteration::<1>(); + check_mixed_iteration::<2>(); + check_mixed_iteration::<3>(); + check_mixed_iteration::<4>(); + check_mixed_iteration::<5>(); + check_mixed_iteration::<6>(); + check_mixed_iteration::<7>(); + check_mixed_iteration::<8>(); +} From d9296dd7306feda1f9028dc2e1e670f61fb5ad8c Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 7 Oct 2026 19:42:50 +0800 Subject: [PATCH 39/51] Cover aligned packed view iterator starts --- src/packed_str/impls_for_iter.rs | 1 - .../impls_for_iter/tests_for_iter.rs | 46 +++++++++++++------ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/packed_str/impls_for_iter.rs b/src/packed_str/impls_for_iter.rs index 10e0ecf..b928990 100644 --- a/src/packed_str/impls_for_iter.rs +++ b/src/packed_str/impls_for_iter.rs @@ -49,7 +49,6 @@ struct FrontCursor { // equals `next_word_index * WORD_BITS`. } -#[allow(dead_code)] pub struct Iter<'ps, C, const BITS: u8> where C: PackedChar, diff --git a/src/packed_str/impls_for_iter/tests_for_iter.rs b/src/packed_str/impls_for_iter/tests_for_iter.rs index ee6bf33..9b5b0a5 100644 --- a/src/packed_str/impls_for_iter/tests_for_iter.rs +++ b/src/packed_str/impls_for_iter/tests_for_iter.rs @@ -21,7 +21,7 @@ fn assert_front_window(iter: &super::Iter<'_, Code, BITS>, front assert!(iter.front_cursor.pending_bit_count <= WORD_BITS); } -fn check_mixed_iteration() { +fn check_mixed_iteration(word_unaligned: bool) { let width = usize::from(BITS); let mask = if BITS == 8 { u8::MAX @@ -45,18 +45,29 @@ fn check_mixed_iteration() { let expected = (0..len) .map(|index| Code(((index * 17 + 3) as u8) & mask)) .collect::>(); - let mut source = alloc::vec::Vec::with_capacity(len + 1); - source.push(Code(0)); + let mut source = alloc::vec::Vec::with_capacity(len + usize::from(word_unaligned)); + if word_unaligned { + source.push(Code(0)); + } source.extend_from_slice(&expected); let owner = PackedString::::from_chars(source); - let view: PackedStr<'_, Code, BITS> = owner.as_packed_str().slice_from(1); + let packed_str = owner.as_packed_str(); + let view: PackedStr<'_, Code, BITS> = if word_unaligned { + packed_str.slice_from(1) + } else { + packed_str + }; let mut iter = view.iter(); let mut front = 0; let mut back = len; - if len > 0 { + if !word_unaligned || len > 0 { assert_eq!(view.bits.start() % width, 0); - assert_ne!(view.bits.start() % WORD_BITS, 0); + if word_unaligned { + assert_ne!(view.bits.start() % WORD_BITS, 0); + } else { + assert_eq!(view.bits.start(), 0); + } assert_front_window(&iter, front); } @@ -78,13 +89,18 @@ fn check_mixed_iteration() { } #[test] -fn rolling_window_handles_offset_views_and_mixed_iteration() { - check_mixed_iteration::<1>(); - check_mixed_iteration::<2>(); - check_mixed_iteration::<3>(); - check_mixed_iteration::<4>(); - check_mixed_iteration::<5>(); - check_mixed_iteration::<6>(); - check_mixed_iteration::<7>(); - check_mixed_iteration::<8>(); +fn rolling_window_handles_aligned_and_offset_views_with_mixed_iteration() { + fn check_width() { + check_mixed_iteration::(false); + check_mixed_iteration::(true); + } + + check_width::<1>(); + check_width::<2>(); + check_width::<3>(); + check_width::<4>(); + check_width::<5>(); + check_width::<6>(); + check_width::<7>(); + check_width::<8>(); } From 70c5de7959745ea95d06a9f5b7ea83a5f895d694 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 7 Oct 2026 19:59:44 +0800 Subject: [PATCH 40/51] Use packed bit append for character push --- benches/packed_editing.rs | 133 ++++++++++++++++++++++++- src/packed_string/impls_for_editing.rs | 8 +- 2 files changed, 132 insertions(+), 9 deletions(-) diff --git a/benches/packed_editing.rs b/benches/packed_editing.rs index 6d739c6..2560de7 100644 --- a/benches/packed_editing.rs +++ b/benches/packed_editing.rs @@ -32,23 +32,83 @@ fn set_middle_vec(bencher: Bencher, len: usize) { } fn push_packed(bencher: Bencher, len: usize) { + push_packed_value::(bencher, len, 0); +} + +fn push_packed_value(bencher: Bencher, len: usize, code: u8) { let input = packed::(&codes(BITS, len)); bencher.with_inputs(|| input.clone()).bench_refs(|value| { - value.push(Code(0)); + value.push(Code(code)); black_box(&*value); black_box(value.char_len()) }); } fn push_amortized_packed(bencher: Bencher, len: usize) { + push_amortized_packed_value::(bencher, len, 0); +} + +fn push_amortized_packed_value(bencher: Bencher, len: usize, code: u8) { let input = packed::(&codes(BITS, len - 1)); bencher.with_inputs(|| input.clone()).bench_refs(|value| { - value.push(Code(0)); + value.push(Code(code)); black_box(&*value); black_box(value.char_len()) }); } +fn push_existing_word(bencher: Bencher, code: u8) { + push_packed_value::(bencher, 1, code); +} + +// Build the backing word first, then truncate so the timed push grows the +// logical word slice without triggering a heap reallocation. +fn push_word_growth(bencher: Bencher, code: u8) { + const PREFIX_LEN: usize = 64; + let input = packed::(&codes(BITS, PREFIX_LEN + 1)); + bencher + .with_inputs(|| { + let mut value = input.clone(); + value.truncate(PREFIX_LEN); + value + }) + .bench_refs(|value| { + value.push(Code(code)); + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn push_cross_word(bencher: Bencher, code: u8) { + let prefix_len = match BITS { + 3 => 21, + 5 => 12, + 6 => 10, + 7 => 9, + _ => unreachable!("cross-word character only exists for irregular widths"), + }; + let input = packed::(&codes(BITS, prefix_len + 1)); + bencher + .with_inputs(|| { + let mut value = input.clone(); + value.truncate(prefix_len); + value + }) + .bench_refs(|value| { + value.push(Code(code)); + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn max_code(bits: u8) -> u8 { + if bits == 8 { + u8::MAX + } else { + ((1u16 << bits) - 1) as u8 + } +} + fn push_vec(bencher: Bencher, len: usize) { let input = codes(BITS, len); bencher.with_inputs(|| input.clone()).bench_refs(|value| { @@ -189,6 +249,15 @@ macro_rules! define_case { pair!("set_position/aligned", set_middle_packed, set_middle_vec); pair!("push_boundary_probe", push_packed, push_vec); pair!("push_amortized", push_amortized_packed, push_amortized_vec); + #[divan::bench(name = concat!("packed_editing/push_boundary_probe_nonzero/", stringify!($case), "/ours_packed_string"))] + fn push_boundary_probe_nonzero(bencher: Bencher) { + crate::push_packed_value::<$bits>(bencher, $len, crate::max_code($bits)); + } + + #[divan::bench(name = concat!("packed_editing/push_amortized_nonzero/", stringify!($case), "/ours_packed_string"))] + fn push_amortized_nonzero(bencher: Bencher) { + crate::push_amortized_packed_value::<$bits>(bencher, $len, crate::max_code($bits)); + } pair!("pop", pop_packed, pop_vec); pair!("insert_position/unaligned", insert_middle_packed, insert_middle_vec); pair!("remove_position/word_boundary_or_fallback", remove_middle_packed, remove_middle_vec); @@ -198,3 +267,63 @@ macro_rules! define_case { } crate::for_each_packed_bench_case!(define_case); + +macro_rules! define_push_shape_case { + ($case:ident, $bits:literal) => { + mod $case { + use super::*; + + #[divan::bench(name = concat!("packed_editing/push_existing_word_zero/", stringify!($case), "/ours_packed_string"))] + fn existing_word_zero(bencher: Bencher) { + crate::push_existing_word::<$bits>(bencher, 0); + } + + #[divan::bench(name = concat!("packed_editing/push_existing_word_nonzero/", stringify!($case), "/ours_packed_string"))] + fn existing_word_nonzero(bencher: Bencher) { + crate::push_existing_word::<$bits>(bencher, crate::max_code($bits)); + } + + #[divan::bench(name = concat!("packed_editing/push_word_growth_zero/", stringify!($case), "/ours_packed_string"))] + fn word_growth_zero(bencher: Bencher) { + crate::push_word_growth::<$bits>(bencher, 0); + } + + #[divan::bench(name = concat!("packed_editing/push_word_growth_nonzero/", stringify!($case), "/ours_packed_string"))] + fn word_growth_nonzero(bencher: Bencher) { + crate::push_word_growth::<$bits>(bencher, crate::max_code($bits)); + } + } + }; +} + +macro_rules! define_cross_word_case { + ($case:ident, $bits:literal) => { + mod $case { + use super::*; + + #[divan::bench(name = concat!("packed_editing/push_cross_word_zero/", stringify!($case), "/ours_packed_string"))] + fn cross_word_zero(bencher: Bencher) { + crate::push_cross_word::<$bits>(bencher, 0); + } + + #[divan::bench(name = concat!("packed_editing/push_cross_word_nonzero/", stringify!($case), "/ours_packed_string"))] + fn cross_word_nonzero(bencher: Bencher) { + crate::push_cross_word::<$bits>(bencher, crate::max_code($bits)); + } + } + }; +} + +define_push_shape_case!(push_shape_bits_1, 1); +define_push_shape_case!(push_shape_bits_2, 2); +define_push_shape_case!(push_shape_bits_3, 3); +define_push_shape_case!(push_shape_bits_4, 4); +define_push_shape_case!(push_shape_bits_5, 5); +define_push_shape_case!(push_shape_bits_6, 6); +define_push_shape_case!(push_shape_bits_7, 7); +define_push_shape_case!(push_shape_bits_8, 8); + +define_cross_word_case!(cross_word_bits_3, 3); +define_cross_word_case!(cross_word_bits_5, 5); +define_cross_word_case!(cross_word_bits_6, 6); +define_cross_word_case!(cross_word_bits_7, 7); diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 2ed7592..b39eaba 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -6,14 +6,8 @@ where { /// Appends the character's code directly to the payload. pub fn push(&mut self, character: C) { - let code = character.code(); self.bits - .bit_len() - .checked_add(usize::from(BITS)) - .expect("packed string length overflow"); - for shift in 0..BITS { - self.bits.push((code >> shift) & 1 != 0); - } + .push_bits(u64::from(character.code()), usize::from(BITS)); } pub fn pop(&mut self) -> Option { From 51df6fa0527474fda6c48449d40f03ebbbf624e7 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 11 Oct 2026 19:24:20 +0800 Subject: [PATCH 41/51] Specialize byte-width packed push --- benches/packed_editing.rs | 53 +++++++++++++++++++ .../impls_for_editing/impls_for_push_pop.rs | 20 +++++++ .../impls_for_push_pop/tests_for_push.rs | 22 ++++++++ src/packed_string/impls_for_editing.rs | 8 ++- 4 files changed, 101 insertions(+), 2 deletions(-) diff --git a/benches/packed_editing.rs b/benches/packed_editing.rs index 2560de7..7d48564 100644 --- a/benches/packed_editing.rs +++ b/benches/packed_editing.rs @@ -118,6 +118,35 @@ fn push_vec(bencher: Bencher, len: usize) { }); } +fn push_existing_word_vec(bencher: Bencher, code: u8) { + bencher + .with_inputs(|| { + let mut value = vec![0u8]; + value.reserve(1); + value + }) + .bench_refs(|value| { + value.push(code); + black_box(&*value); + black_box(value.len()) + }); +} + +fn push_word_growth_vec(bencher: Bencher, code: u8) { + let input = vec![0u8; 65]; + bencher + .with_inputs(|| { + let mut value = input.clone(); + value.truncate(64); + value + }) + .bench_refs(|value| { + value.push(code); + black_box(&*value); + black_box(value.len()) + }); +} + fn push_amortized_vec(bencher: Bencher, len: usize) { let input = codes(BITS, len - 1); bencher @@ -323,6 +352,30 @@ define_push_shape_case!(push_shape_bits_6, 6); define_push_shape_case!(push_shape_bits_7, 7); define_push_shape_case!(push_shape_bits_8, 8); +mod push_vec_bits_8 { + use super::*; + + #[divan::bench(name = "packed_editing/push_existing_word_zero/push_shape_bits_8/vec_u8")] + fn existing_word_zero(bencher: Bencher) { + crate::push_existing_word_vec(bencher, 0); + } + + #[divan::bench(name = "packed_editing/push_existing_word_nonzero/push_shape_bits_8/vec_u8")] + fn existing_word_nonzero(bencher: Bencher) { + crate::push_existing_word_vec(bencher, u8::MAX); + } + + #[divan::bench(name = "packed_editing/push_word_growth_zero/push_shape_bits_8/vec_u8")] + fn word_growth_zero(bencher: Bencher) { + crate::push_word_growth_vec(bencher, 0); + } + + #[divan::bench(name = "packed_editing/push_word_growth_nonzero/push_shape_bits_8/vec_u8")] + fn word_growth_nonzero(bencher: Bencher) { + crate::push_word_growth_vec(bencher, u8::MAX); + } +} + define_cross_word_case!(cross_word_bits_3, 3); define_cross_word_case!(cross_word_bits_5, 5); define_cross_word_case!(cross_word_bits_6, 6); diff --git a/src/bit_string/impls_for_editing/impls_for_push_pop.rs b/src/bit_string/impls_for_editing/impls_for_push_pop.rs index 1db1061..522b17b 100644 --- a/src/bit_string/impls_for_editing/impls_for_push_pop.rs +++ b/src/bit_string/impls_for_editing/impls_for_push_pop.rs @@ -5,6 +5,26 @@ use crate::{WORD_BITS, low_mask}; use super::*; impl BitString { + /// Appends one byte to a byte-aligned bit string. + #[inline] + pub(crate) fn push_byte(&mut self, value: u8) { + debug_assert_eq!(self.bit_len % 8, 0); + + let new_bit_len = self + .bit_len + .checked_add(8) + .expect("bit string length overflow"); + let word_index = self.bit_len / WORD_BITS; + let bit_offset = self.bit_len % WORD_BITS; + + if word_index == self.words.len() { + self.words.push(0); + } + + self.words[word_index] |= u64::from(value) << bit_offset; + self.bit_len = new_bit_len; + } + /// Reserves backing storage for an additional number of bits. #[inline] pub(crate) fn reserve_bits(&mut self, additional_bits: usize) { diff --git a/src/bit_string/impls_for_editing/impls_for_push_pop/tests_for_push.rs b/src/bit_string/impls_for_editing/impls_for_push_pop/tests_for_push.rs index 682f8b7..2200a53 100644 --- a/src/bit_string/impls_for_editing/impls_for_push_pop/tests_for_push.rs +++ b/src/bit_string/impls_for_editing/impls_for_push_pop/tests_for_push.rs @@ -74,3 +74,25 @@ fn pushing_true_increments_count_ones() { assert_eq!(bits.count_zeros(), 1); assert_eq!(bits.to_string(), "10111"); } + +#[test] +fn push_byte_appends_at_byte_aligned_offsets() { + for prefix_len in [0, 8, 56, 64] { + for value in [0, 0xa5, u8::MAX] { + let mut bits = BitString::ones(prefix_len); + + bits.push_byte(value); + + assert_eq!(bits.bit_len(), prefix_len + 8); + for index in 0..prefix_len { + assert_eq!(bits.get(index), Some(true)); + } + for offset in 0..8 { + assert_eq!( + bits.get(prefix_len + offset), + Some(value & (1 << offset) != 0) + ); + } + } + } +} diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index b39eaba..989a51e 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -6,8 +6,12 @@ where { /// Appends the character's code directly to the payload. pub fn push(&mut self, character: C) { - self.bits - .push_bits(u64::from(character.code()), usize::from(BITS)); + let code = character.code(); + if BITS == 8 { + self.bits.push_byte(code); + } else { + self.bits.push_bits(u64::from(code), usize::from(BITS)); + } } pub fn pop(&mut self) -> Option { From c6cdbf090bf781bea0ff6a4a2aaf1032275719f9 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 14 Oct 2026 22:11:49 +0800 Subject: [PATCH 42/51] Optimize packed insertion and removal with fixed-width shifts Move the suffix once per packed character through internal BitString insert_bits/remove_bits primitives. Reuse WordsArith in-place shifts, preserve the boundary prefix, and keep padding and lazy shrinking intact. Add bit/character model tests and editing benchmarks at lengths 63/64/65/127/128/129 for BITS=1..8. Validation: default, native and release tests; fmt and staged diff checks; characterization bench check; read-only gpt-6-luna/high review. Native 1M middle-edit medians in microseconds (before -> after): BITS insert remove 1 16.38 -> 1.37 12.21 -> 1.16 2 53.08 -> 2.87 58.67 -> 2.51 3 113.0 -> 3.46 144.8 -> 3.21 4 194.4 -> 5.64 262.1 -> 5.48 5 307.0 -> 29.77 424.9 -> 7.69 6 423.8 -> 37.32 608.9 -> 8.93 7 614.1 -> 48.78 860.2 -> 10.64 8 806.8 -> 58.21 1106 -> 11.74 Compared with 51df6fa using identical benchmark sources, 100 samples and isolated Cargo target directories. Some short BITS=1 cases regress by approximately 2-9 ns; retain the uniform path and disclose this tradeoff. --- benches/packed_editing.rs | 28 ++++++ .../impls_for_insert_remove.rs | 76 +++++++++++++++ .../tests_for_edit_bits.rs | 95 +++++++++++++++++++ .../impls_for_insert_remove.rs | 21 ++-- .../tests_for_insert_remove.rs | 78 +++++++++++++++ 5 files changed, 283 insertions(+), 15 deletions(-) create mode 100644 src/bit_string/impls_for_editing/impls_for_insert_remove/tests_for_edit_bits.rs create mode 100644 src/packed_string/impls_for_editing/impls_for_insert_remove/tests_for_insert_remove.rs diff --git a/benches/packed_editing.rs b/benches/packed_editing.rs index 7d48564..5d50810 100644 --- a/benches/packed_editing.rs +++ b/benches/packed_editing.rs @@ -297,6 +297,34 @@ macro_rules! define_case { crate::for_each_packed_bench_case!(define_case); +// Adjacent lengths exercise backing-word growth/shrink and partial tails. +macro_rules! define_edit_boundary_case { + ($case:ident, $bits:literal) => { + mod $case { + use super::*; + + #[divan::bench(args = [63, 64, 65, 127, 128, 129])] + fn insert_middle(bencher: Bencher, len: usize) { + crate::insert_middle_packed::<$bits>(bencher, len); + } + + #[divan::bench(args = [63, 64, 65, 127, 128, 129])] + fn remove_middle(bencher: Bencher, len: usize) { + crate::remove_middle_packed::<$bits>(bencher, len); + } + } + }; +} + +define_edit_boundary_case!(edit_boundary_bits_1, 1); +define_edit_boundary_case!(edit_boundary_bits_2, 2); +define_edit_boundary_case!(edit_boundary_bits_3, 3); +define_edit_boundary_case!(edit_boundary_bits_4, 4); +define_edit_boundary_case!(edit_boundary_bits_5, 5); +define_edit_boundary_case!(edit_boundary_bits_6, 6); +define_edit_boundary_case!(edit_boundary_bits_7, 7); +define_edit_boundary_case!(edit_boundary_bits_8, 8); + macro_rules! define_push_shape_case { ($case:ident, $bits:literal) => { mod $case { diff --git a/src/bit_string/impls_for_editing/impls_for_insert_remove.rs b/src/bit_string/impls_for_editing/impls_for_insert_remove.rs index 01eddcb..8147999 100644 --- a/src/bit_string/impls_for_editing/impls_for_insert_remove.rs +++ b/src/bit_string/impls_for_editing/impls_for_insert_remove.rs @@ -4,6 +4,79 @@ use crate::traits::*; use super::*; impl BitString { + /// Inserts the low `width` bits of `value`, shifting the suffix once. + /// + /// `width` must be at most one word. The index is clamped to the end; + /// zero width is a no-op. Growth is checked before modifying the payload. + pub(crate) fn insert_bits(&mut self, index: usize, value: u64, width: usize) { + assert!( + width <= WORD_BITS, + "cannot insert more than one word of bits" + ); + let index = index.min(self.bit_len); + if index == self.bit_len { + self.push_bits(value, width); + return; + } + if width == 0 { + return; + } + + let new_len = self + .bit_len + .checked_add(width) + .expect("bit string length overflow"); + // Allocate before changing existing bits, retaining spare capacity. + self.words.resize(word_len(new_len), 0); + + let first_word = index / WORD_BITS; + let prefix_mask = crate::low_mask(index % WORD_BITS); + let prefix = self.words[first_word] & prefix_mask; + // Clear the boundary prefix before shifting so it cannot enter the gap. + // The word-level shift runs high to low, preserving overlapping input. + self.words[first_word] &= !prefix_mask; + self.words[first_word..].shl_assign(new_len - first_word * WORD_BITS, width); + self.words[first_word] |= prefix; + // The shift leaves [index, index + width) zero, as required by OR writes. + self.words.write_word_at::(index, value, width); + self.bit_len = new_len; + } + + /// Removes `width` bits at `index`, shifting the suffix once. + /// + /// `width` must be at most one word. Zero width is a no-op; otherwise + /// the entire range must exist. Invalid arguments panic before mutation. + pub(crate) fn remove_bits(&mut self, index: usize, width: usize) { + assert!( + width <= WORD_BITS, + "cannot remove more than one word of bits" + ); + if width == 0 { + return; + } + assert!( + index <= self.bit_len && width <= self.bit_len - index, + "bit string removal range out of bounds" + ); + + let first_word = index / WORD_BITS; + let prefix_mask = crate::low_mask(index % WORD_BITS); + let prefix = self.words[first_word] & prefix_mask; + // The word-level shift runs low to high, preserving overlapping input. + // At every bit position >= index, it reads the old position + width. + self.words[first_word..].shr_assign(self.bit_len - first_word * WORD_BITS, width); + // Removed bits may have shifted into the prefix: replace, rather than OR. + self.words[first_word] = (self.words[first_word] & !prefix_mask) | prefix; + + let new_len = self.bit_len - width; + let new_words = word_len(new_len); + if new_words < self.words.len() { + self.truncate_words(new_words); + } + self.words.mask_unused_bits(new_len); + self.bit_len = new_len; + } + /// Inserts a single bit at `index`, shifting the tail right. /// /// `index` is clamped to `[0, self.bit_len()]` — an out-of-bounds index @@ -86,3 +159,6 @@ mod tests_for_insert; #[cfg(test)] mod tests_for_remove; + +#[cfg(test)] +mod tests_for_edit_bits; diff --git a/src/bit_string/impls_for_editing/impls_for_insert_remove/tests_for_edit_bits.rs b/src/bit_string/impls_for_editing/impls_for_insert_remove/tests_for_edit_bits.rs new file mode 100644 index 0000000..4e4d7bd --- /dev/null +++ b/src/bit_string/impls_for_editing/impls_for_insert_remove/tests_for_edit_bits.rs @@ -0,0 +1,95 @@ +use crate::{BitString, low_mask, word_len}; +use alloc::vec::Vec; + +const LENGTHS: &[usize] = &[0, 1, 7, 8, 15, 31, 32, 63, 64, 65, 127, 128, 129, 193]; +const WIDTHS: &[usize] = &[0, 1, 2, 3, 5, 6, 7, 8, 31, 32, 63, 64]; + +fn input(len: usize) -> Vec { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state & 1 != 0 + }) + .collect() +} + +fn assert_bits(actual: &BitString, expected: &[bool]) { + assert_eq!(actual.bit_len(), expected.len()); + assert_eq!(actual.words().len(), word_len(expected.len())); + assert_eq!(actual.iter().collect::>(), expected); + if let Some(&last) = actual.words().last() { + let used = (expected.len() - 1) % 64 + 1; + assert_eq!(last & !low_mask(used), 0, "unused bits must remain zero"); + } +} + +#[test] +fn insert_bits_matches_boolean_model_at_every_position() { + for &len in LENGTHS { + let original = input(len); + let source = BitString::from_bool_iter(original.iter().copied()); + for &width in WIDTHS { + for index in 0..=len + 1 { + for code in [0, u64::MAX, 0xa53c_79e1_d24b_860fu64] { + let mut expected = original.clone(); + let bits = (0..width).map(|shift| (code >> shift) & 1 != 0); + expected.splice(index.min(len)..index.min(len), bits); + let mut actual = source.clone(); + actual.insert_bits(index, code, width); + assert_bits(&actual, &expected); + } + } + } + } +} + +#[test] +fn remove_bits_matches_boolean_model_at_every_position() { + for &len in LENGTHS { + let original = input(len); + let source = BitString::from_bool_iter(original.iter().copied()); + for &width in WIDTHS { + if width > len { + continue; + } + for index in 0..=len - width { + let mut expected = original.clone(); + expected.drain(index..index + width); + let mut actual = source.clone(); + actual.remove_bits(index, width); + assert_bits(&actual, &expected); + } + } + } +} + +#[test] +fn repeated_edits_preserve_zero_padding() { + let mut actual = BitString::from_bool_iter(input(193)); + let original = actual.clone(); + for &width in WIDTHS { + actual.insert_bits(63, u64::MAX, width); + actual.remove_bits(63, width); + assert_eq!(actual, original); + } + actual.insert_bits(usize::MAX, 0, 0); + actual.remove_bits(usize::MAX, 0); + assert_eq!(actual, original); +} + +#[test] +fn invalid_arguments_leave_payload_unchanged() { + extern crate std; + use std::panic::{AssertUnwindSafe, catch_unwind}; + let original = BitString::from_bool_iter(input(129)); + let mut actual = original.clone(); + assert!(catch_unwind(AssertUnwindSafe(|| actual.insert_bits(0, 1, 65))).is_err()); + assert_eq!(actual, original); + for (index, width) in [(0, 65), (129, 1), (128, 2), (usize::MAX, 1)] { + assert!(catch_unwind(AssertUnwindSafe(|| actual.remove_bits(index, width))).is_err()); + assert_eq!(actual, original); + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs index 03f4f02..8f80508 100644 --- a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs @@ -6,27 +6,18 @@ where { pub fn insert(&mut self, index: usize, character: C) { let index = index.min(self.char_len()); - self.bits - .bit_len() - .checked_add(usize::from(BITS)) - .expect("packed string length overflow"); let start = index * usize::from(BITS); - for shift in (0..BITS).rev() { - self.bits - .insert(start, (character.code() >> shift) & 1 != 0); - } + self.bits + .insert_bits(start, u64::from(character.code()), usize::from(BITS)); } pub fn remove(&mut self, index: usize) -> C { let character = self.get(index).expect("packed string index out of bounds"); let start = index * usize::from(BITS); - let mut code = 0; - for shift in 0..BITS { - if self.bits.remove(start) { - code |= 1 << shift; - } - } - debug_assert_eq!(code, character.code()); + self.bits.remove_bits(start, usize::from(BITS)); character } } + +#[cfg(test)] +mod tests_for_insert_remove; diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove/tests_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove/tests_for_insert_remove.rs new file mode 100644 index 0000000..e1a571c --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove/tests_for_insert_remove.rs @@ -0,0 +1,78 @@ +use crate::{PackedString, code_mask, traits::PackedChar}; +use alloc::vec::Vec; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + fn from_code(code: u8) -> Option { + (code <= code_mask::()).then_some(Self(code)) + } +} + +fn check() { + for len in [0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 65, 127, 128, 129] { + let mut state = 0x1234_5678_9abc_def0u64; + let expected = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + Code(state as u8 & code_mask::()) + }) + .collect::>(); + let source = PackedString::::from_chars(expected.iter().copied()); + for index in 0..=len + 1 { + for code in [Code(0), Code(code_mask::())] { + let mut actual = source.clone(); + let mut model = expected.clone(); + let at = index.min(len); + model.insert(at, code); + actual.insert(index, code); + assert_eq!(actual.iter().collect::>(), model); + assert_eq!( + actual.bits(), + PackedString::::from_chars(model).bits() + ); + assert_eq!(actual.remove(at), code); + assert_eq!(actual.bits(), source.bits()); + } + if index < len { + let mut actual = source.clone(); + let mut model = expected.clone(); + assert_eq!(actual.remove(index), model.remove(index)); + assert_eq!( + actual.bits(), + PackedString::::from_chars(model).bits() + ); + } + } + } +} + +#[test] +fn fixed_width_edits_match_character_model() { + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} + +#[test] +fn remove_out_of_bounds_preserves_payload() { + extern crate std; + use std::panic::{AssertUnwindSafe, catch_unwind}; + let mut actual = PackedString::::from_chars([Code(7), Code(31)]); + let original = actual.bits().clone(); + for index in [2, usize::MAX] { + assert!(catch_unwind(AssertUnwindSafe(|| actual.remove(index))).is_err()); + assert_eq!(actual.bits(), &original); + } +} From 390d58fd53cf1950646ff11478df8ec0ca9b67af Mon Sep 17 00:00:00 2001 From: juncheng Date: Thu, 15 Oct 2026 21:43:39 +0800 Subject: [PATCH 43/51] Optimize in-place word shifts with a carry stream For exact-overlap shifts of 1..63 bits, preserve the outgoing carry before writing each word and walk the input forward. This shape lets LLVM vectorize the carry stream in the AVX2 backend. Keep the existing implementation for independent destinations, zero, whole-word and mixed shifts. Document the overlap proof beside the unsafe loop, update traversal comments, and cover owned/in-place vector boundaries with backend-equivalence tests. Inspired by grit-bitvec's insertion carry stream. Add paired ours/grit insertion benchmarks for reserved capacity, repeated insertion with natural growth, and repeated insertion with reserved capacity. Each batch inserts min(initial_length, 256) values into the same object; input preparation and reservation are outside timing. Two CPU-0-pinned native A/B rounds at 1,048,576 initial characters show about 21-44% lower time for 256 consecutive insertions across BITS=1..8. Whole-batch median ranges in milliseconds: BITS baseline carry 1 0.423-0.426 0.245-0.246 2 0.845-0.878 0.492-0.500 3 1.310-1.312 0.762-0.835 4 1.683-1.752 0.984-0.993 5 2.171-2.209 1.266-1.409 6 2.593-2.655 2.046-2.072 7 2.995-3.100 2.320-2.401 8 3.390-3.511 2.515-2.729 Same-process rotating kernel comparisons also favor this form over the original loop, a 16-word unroll, and an indexed carry loop. Only this candidate is retained. Allocation-inclusive owned shift measurements remain noisy; isolated independent-destination comparisons show no consistent large regression. Frequency and background load were not controlled. Validation: cargo fmt --check; git diff --cached --check; default tests (653 library + 149 integration + 100 packed); native tests (677 + 149 + 100); 3 native release shl-core tests; cargo check --benches --features characterization; default/native insertion benchmarks and native shift checks. --- benches/packed_competitors.rs | 93 ++++++++++++++++++- .../impls_for_insert_remove.rs | 2 +- src/traits/words_arith/funcs_for_shl_core.rs | 4 +- .../words_arith/funcs_for_shl_core/avx2.rs | 26 ++++++ .../tests_for_backend_equivalence.rs | 34 +++++++ 5 files changed, 154 insertions(+), 5 deletions(-) diff --git a/benches/packed_competitors.rs b/benches/packed_competitors.rs index deb2d95..51613e7 100644 --- a/benches/packed_competitors.rs +++ b/benches/packed_competitors.rs @@ -5,8 +5,8 @@ use compressed_intvec::fixed::UFixedVec; use divan::{Bencher, black_box}; use grit_bitvec::{TypedBitElem, TypedBitVec}; use support::{ - aligned_index, aligned_index_for, codes, cross_word_index, cross_word_index_for, indices, - unaligned_index, unaligned_index_for, + Code, aligned_index, aligned_index_for, codes, cross_word_index, cross_word_index_for, indices, + packed, unaligned_index, unaligned_index_for, }; use sux::bits::BitFieldVec; use value_traits::slices::{SliceByValue, SliceByValueMut}; @@ -309,6 +309,89 @@ where }); } +// Batch timings include every insertion and any capacity growth it triggers. +// Keep the batch no larger than the initial length: truncating a cloned packed +// input then retains enough capacity even with BitString's lazy shrink policy. +const INSERT_BATCH_LEN: usize = 256; + +fn insert_reserved_packed(bencher: Bencher, len: usize) { + insert_sequence_packed::(bencher, len, 1, true); +} + +fn insert_repeated_packed(bencher: Bencher, len: usize) { + insert_sequence_packed::(bencher, len, len.min(INSERT_BATCH_LEN), false); +} + +fn insert_repeated_reserved_packed(bencher: Bencher, len: usize) { + insert_sequence_packed::(bencher, len, len.min(INSERT_BATCH_LEN), true); +} + +fn insert_sequence_packed( + bencher: Bencher, + len: usize, + count: usize, + reserve: bool, +) { + let input = packed::(&codes(BITS, len + if reserve { count } else { 0 })); + let index = unaligned_index::(len); + bencher + .with_inputs(|| { + let mut value = input.clone(); + // Clone before truncating: cloning the shorter value would discard + // the spare allocation. Setup is excluded from the timed region. + value.truncate(len); + value + }) + .bench_local_refs(|value| { + // A fixed insertion point keeps shifting the original suffix plus + // the characters inserted earlier in this batch. + for _ in 0..count { + value.insert(index, Code(0)); + } + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn insert_reserved_grit + 'static>(bencher: Bencher, len: usize) { + insert_sequence_grit::(bencher, len, 1, true); +} + +fn insert_repeated_grit + 'static>(bencher: Bencher, len: usize) { + insert_sequence_grit::(bencher, len, len.min(INSERT_BATCH_LEN), false); +} + +fn insert_repeated_reserved_grit + 'static>( + bencher: Bencher, + len: usize, +) { + insert_sequence_grit::(bencher, len, len.min(INSERT_BATCH_LEN), true); +} + +fn insert_sequence_grit + 'static>( + bencher: Bencher, + len: usize, + count: usize, + reserve: bool, +) { + let width = grit_width::(); + let input = codes(width, len); + let index = unaligned_index_for(usize::from(width), len); + bencher + .with_inputs(|| { + let mut value = TypedBitVec::::with_capacity(len + if reserve { count } else { 0 }); + value.append_iter(input.iter().copied()).unwrap(); + value + }) + .bench_local_refs(|value| { + for _ in 0..count { + value.insert(index, 0).unwrap(); + } + black_box(&*value); + black_box(value.len()) + }); +} + fn remove_grit(bencher: Bencher, len: usize) where T: TypedBitElem + 'static, @@ -376,6 +459,9 @@ macro_rules! define_packed_case { benchmark!("editing/pop", pop_fixed, "compressed_intvec"); benchmark!("editing/extend_bulk_25pct", extend_fixed, "compressed_intvec"); benchmark!("editing/insert_position/unaligned", insert_fixed, "compressed_intvec"); + benchmark!("editing/insert_reserved/unaligned", insert_reserved_packed, "ours_packed_string"); + benchmark!("editing/insert_repeated/unaligned", insert_repeated_packed, "ours_packed_string"); + benchmark!("editing/insert_repeated_reserved/unaligned", insert_repeated_reserved_packed, "ours_packed_string"); benchmark!("editing/remove_position/word_boundary_or_fallback", remove_fixed, "compressed_intvec"); benchmark!("construction/from_chars", construct_sux, "sux"); @@ -422,6 +508,9 @@ macro_rules! define_grit_case { benchmark!("editing/push_boundary_probe", push_grit); benchmark!("editing/pop", pop_grit); benchmark!("editing/insert_position/unaligned", insert_grit); + benchmark!("editing/insert_reserved/unaligned", insert_reserved_grit); + benchmark!("editing/insert_repeated/unaligned", insert_repeated_grit); + benchmark!("editing/insert_repeated_reserved/unaligned", insert_repeated_reserved_grit); benchmark!("editing/remove_position/word_boundary_or_fallback", remove_grit); benchmark!("editing/extend_bulk_25pct", append_grit); } diff --git a/src/bit_string/impls_for_editing/impls_for_insert_remove.rs b/src/bit_string/impls_for_editing/impls_for_insert_remove.rs index 8147999..6c614cc 100644 --- a/src/bit_string/impls_for_editing/impls_for_insert_remove.rs +++ b/src/bit_string/impls_for_editing/impls_for_insert_remove.rs @@ -33,7 +33,7 @@ impl BitString { let prefix_mask = crate::low_mask(index % WORD_BITS); let prefix = self.words[first_word] & prefix_mask; // Clear the boundary prefix before shifting so it cannot enter the gap. - // The word-level shift runs high to low, preserving overlapping input. + // The word-level shift preserves overlapping input. self.words[first_word] &= !prefix_mask; self.words[first_word..].shl_assign(new_len - first_word * WORD_BITS, width); self.words[first_word] |= prefix; diff --git a/src/traits/words_arith/funcs_for_shl_core.rs b/src/traits/words_arith/funcs_for_shl_core.rs index e933b47..8f21971 100644 --- a/src/traits/words_arith/funcs_for_shl_core.rs +++ b/src/traits/words_arith/funcs_for_shl_core.rs @@ -33,8 +33,8 @@ pub(super) fn assign(bits: &mut [u64], bit_len: usize, amount: usize) { // - `ptr` is valid for reads and writes of `word_len` u64 values. // - `dst == src` is explicitly allowed by `dispatch`. // - `dispatch` writes every slot in `0..word_len` exactly once. - // - The implementation writes from high word to low word, so in-place left shift - // does not overwrite source words before they are read. + // - Backends preserve overlapping input by descending writes or by saving + // each word's outgoing carry before overwriting that word. unsafe { dispatch(ptr, ptr.cast_const(), word_len, amount); } diff --git a/src/traits/words_arith/funcs_for_shl_core/avx2.rs b/src/traits/words_arith/funcs_for_shl_core/avx2.rs index b13adff..47380c0 100644 --- a/src/traits/words_arith/funcs_for_shl_core/avx2.rs +++ b/src/traits/words_arith/funcs_for_shl_core/avx2.rs @@ -39,6 +39,32 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + // In-place carry stream, inspired by grit-bitvec's insert. + // A sub-word shift needs only the current old word and its preceding carry. + // Save that carry before overwriting the word, so low-to-high is safe. + if dst.cast_const() == src && word_shift == 0 && bit_shift != 0 { + // SAFETY: dst spans word_len initialized words; end is one-past that + // same allocation. The cursor advances only while it is below end. + // Exact overlap means each old word is read before it is overwritten; + // its outgoing carry is retained before advancing to the next word. + // At word i, carry is old[i - 1] >> (64 - bit_shift), or zero for i=0. + // Thus the write produces exactly the shifted word without rereading + // overwritten input. This branch guarantees 1 <= bit_shift < 64. + unsafe { + let end = dst.add(word_len); + let mut cursor = dst; + let mut carry = 0; + while cursor < end { + let old = cursor.read(); + let next_carry = old >> (WORD_BITS - bit_shift); + cursor.write((old << bit_shift) | carry); + cursor = cursor.add(1); + carry = next_carry; + } + } + return; + } + let bulk_start = word_shift + usize::from(bit_shift != 0); let mut end = word_len; diff --git a/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs index a33a9b6..cba95ef 100644 --- a/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs @@ -99,6 +99,40 @@ fn avx2_matches_scalar() { assert_backend_matches_scalar(avx2::words); } +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_matches_scalar_across_vector_boundaries() { + for len in [0, 1, 3, 4, 5, 15, 16, 17, 18, 31, 32, 33, 63, 64, 65] { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let src: Vec<_> = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }) + .collect(); + for amount in [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 31, 63, 64, 65, 127, 128, 129, 191, 256, 4096, + ] { + let expected = run_backend_owned(scalar::words, &src, amount); + assert_eq!( + run_backend_owned(avx2::words, &src, amount), + expected, + "owned: len={len}, amount={amount}" + ); + assert_eq!( + run_backend_in_place(avx2::words, &src, amount), + expected, + "in-place: len={len}, amount={amount}" + ); + } + } +} + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] #[test] fn neon_matches_scalar() { From 3cf8fa61457c1866865838c7c855d1e10ba5027f Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 17 Oct 2026 07:52:37 +0800 Subject: [PATCH 44/51] Optimize AVX2 in-place right shifts with word lookahead For exact-overlap right shifts of 1..63 bits, cache the current word and read its successor before overwriting it. Continue low to high and shift the final word without a successor. Document the pointer bounds and old-word invariant. Keep the existing owned, zero, whole-word and mixed-shift paths. Extend the wide backend-equivalence checks to SSE2/AVX2/NEON for both shift operations. Register bit_ops_shr_assign with single assign, owned output and 256 shifts of the same allocation, at 4096/65536/1048576 bits and shifts 1/17/65. Input preparation is outside timing; output allocation is included for owned. Portability experiments, rustc 1.98.1 on x86-64: - SSE2-only (target-cpu=x86-64): shl forward carry showed no stable large-input win, and shr reverse carry/forward lookahead regressed. Retain both original SSE2 kernels and record the negative decisions beside them. - Native/AVX2: mirrored reverse carry regressed shr. Forward word lookahead improved large in-place shr kernels by about 20-32% in rotating same-process, CPU-0-pinned comparisons. Small inputs fluctuated; this is not a uniform win claim for every length. Independent destinations retain the old path. - NEON: no AArch64 target, toolchain or native runtime available. Algorithms remain unchanged; the expanded equivalence checks will apply on ARM builds. SSE2 functions compiled with target-cpu=native can inherit AVX2, so those measurements were not treated as evidence of SSE2-only performance. Native 1M-character single PackedString removal medians (microseconds): BITS before after 1 1.390 0.961 2 2.561 2.538 3 4.718 2.929 4 6.287 4.993 5 7.309 5.896 6 9.435 7.551 7 10.060 8.072 8 11.760 9.214 These end-to-end runs were separate and unpinned; treat small differences as noise. The pinned rotating kernel experiments separately reproduced the large sub-word-shift improvement. Whole-word/mixed controls were included. Validation: default tests (653 library + 149 integration + 100 packed), native tests (676 + 149 + 100), native release shift-core tests, fmt/diff checks, characterization bench compilation. Standalone native and SSE2-only comparisons checked lengths 0..80 and shifts 0..130/191/256/4096, owned/in-place and guard words. Only the beneficial AVX2 forward-lookahead prototype is retained. --- Cargo.toml | 4 ++ benches/bit_ops_shr_assign.rs | 63 +++++++++++++++++++ .../words_arith/funcs_for_shl_core/sse2.rs | 2 + .../tests_for_backend_equivalence.rs | 24 ++++--- .../words_arith/funcs_for_shr_core/avx2.rs | 22 +++++++ .../words_arith/funcs_for_shr_core/sse2.rs | 2 + .../tests_for_backend_equivalence.rs | 32 ++++++++++ 7 files changed, 136 insertions(+), 13 deletions(-) create mode 100644 benches/bit_ops_shr_assign.rs diff --git a/Cargo.toml b/Cargo.toml index ef5b665..c002e79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,6 +160,10 @@ harness = false name = "bit_ops_shl_assign" harness = false +[[bench]] +name = "bit_ops_shr_assign" +harness = false + [[bench]] name = "hash" harness = false diff --git a/benches/bit_ops_shr_assign.rs b/benches/bit_ops_shr_assign.rs new file mode 100644 index 0000000..c144651 --- /dev/null +++ b/benches/bit_ops_shr_assign.rs @@ -0,0 +1,63 @@ +use bit_string::BitString; +use divan::{Bencher, black_box}; + +fn main() { + divan::main(); +} + +// Preparation is outside timing. Repeated shifts reuse one allocation, so +// they measure steady-state throughput separately from a single operation. +fn assign(bencher: Bencher, len: usize, amount: usize, count: usize) { + let input = make_bit_string(len); + bencher.with_inputs(|| input.clone()).bench_refs(|bits| { + for _ in 0..count { + bits.shr_assign(black_box(amount)); + } + black_box(&*bits); + }); +} + +fn owned(bencher: Bencher, len: usize, amount: usize) { + let input = make_bit_string(len); + // Output allocation is part of the owned operation. + bencher.bench(|| black_box(black_box(&input).shr(black_box(amount)))); +} + +fn make_bit_string(len: usize) -> BitString { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state & 1 != 0 + }) + .collect() +} + +macro_rules! define_shift { + ($case:ident, $amount:literal) => { + mod $case { + use super::*; + + #[divan::bench(args = [4096, 65_536, 1_048_576])] + fn assign(bencher: Bencher, len: usize) { + super::assign(bencher, len, $amount, 1); + } + + #[divan::bench(args = [4096, 65_536, 1_048_576])] + fn repeated_256(bencher: Bencher, len: usize) { + super::assign(bencher, len, $amount, 256); + } + + #[divan::bench(args = [4096, 65_536, 1_048_576])] + fn owned(bencher: Bencher, len: usize) { + super::owned(bencher, len, $amount); + } + } + }; +} + +define_shift!(by_1, 1); +define_shift!(by_17, 17); +define_shift!(by_65, 65); diff --git a/src/traits/words_arith/funcs_for_shl_core/sse2.rs b/src/traits/words_arith/funcs_for_shl_core/sse2.rs index 7d74057..5fb1d76 100644 --- a/src/traits/words_arith/funcs_for_shl_core/sse2.rs +++ b/src/traits/words_arith/funcs_for_shl_core/sse2.rs @@ -39,6 +39,8 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + // A tested in-place carry stream did not improve SSE2-only throughput. + // Keep the overlapping-load kernel. let bulk_start = word_shift + usize::from(bit_shift != 0); let mut end = word_len; diff --git a/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs index cba95ef..4d993f6 100644 --- a/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs @@ -88,6 +88,7 @@ fn assert_backend_matches_scalar(backend: Backend) { #[test] fn sse2_matches_scalar() { assert_backend_matches_scalar(sse2::words); + assert_backend_matches_scalar_across_vector_boundaries(sse2::words); } #[cfg(all( @@ -97,14 +98,17 @@ fn sse2_matches_scalar() { #[test] fn avx2_matches_scalar() { assert_backend_matches_scalar(avx2::words); + assert_backend_matches_scalar_across_vector_boundaries(avx2::words); } -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" -))] +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] #[test] -fn avx2_matches_scalar_across_vector_boundaries() { +fn neon_matches_scalar() { + assert_backend_matches_scalar(neon::words); + assert_backend_matches_scalar_across_vector_boundaries(neon::words); +} + +fn assert_backend_matches_scalar_across_vector_boundaries(backend: Backend) { for len in [0, 1, 3, 4, 5, 15, 16, 17, 18, 31, 32, 33, 63, 64, 65] { let mut state = 0x9e37_79b9_7f4a_7c15u64; let src: Vec<_> = (0..len) @@ -120,21 +124,15 @@ fn avx2_matches_scalar_across_vector_boundaries() { ] { let expected = run_backend_owned(scalar::words, &src, amount); assert_eq!( - run_backend_owned(avx2::words, &src, amount), + run_backend_owned(backend, &src, amount), expected, "owned: len={len}, amount={amount}" ); assert_eq!( - run_backend_in_place(avx2::words, &src, amount), + run_backend_in_place(backend, &src, amount), expected, "in-place: len={len}, amount={amount}" ); } } } - -#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] -#[test] -fn neon_matches_scalar() { - assert_backend_matches_scalar(neon::words); -} diff --git a/src/traits/words_arith/funcs_for_shr_core/avx2.rs b/src/traits/words_arith/funcs_for_shr_core/avx2.rs index 07a8364..67d03f9 100644 --- a/src/traits/words_arith/funcs_for_shr_core/avx2.rs +++ b/src/traits/words_arith/funcs_for_shr_core/avx2.rs @@ -39,6 +39,28 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + // In-place forward right shift with one-word lookahead. + if dst.cast_const() == src && word_shift == 0 && bit_shift != 0 { + // SAFETY: word_len > 0 here, and 1 <= bit_shift < 64. At word i, + // current is old[i]. cursor < last guarantees that old[i + 1] exists + // and is unmodified by earlier writes, which only touch lower words. + // Read it before writing old[i] >> bit_shift | old[i + 1] << (64 - bit_shift). + // The last word has no following carry and is shifted on its own. + unsafe { + let last = dst.add(word_len - 1); + let mut cursor = dst; + let mut current = cursor.read(); + while cursor < last { + let next = cursor.add(1).read(); + cursor.write((current >> bit_shift) | (next << (WORD_BITS - bit_shift))); + current = next; + cursor = cursor.add(1); + } + cursor.write(current >> bit_shift); + } + return; + } + let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); let mut start = 0usize; diff --git a/src/traits/words_arith/funcs_for_shr_core/sse2.rs b/src/traits/words_arith/funcs_for_shr_core/sse2.rs index 6c2d8bd..d35fd93 100644 --- a/src/traits/words_arith/funcs_for_shr_core/sse2.rs +++ b/src/traits/words_arith/funcs_for_shr_core/sse2.rs @@ -39,6 +39,8 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + // Tested reverse-carry and forward-lookahead streams regressed SSE2-only + // throughput. Keep the overlapping-load kernel. let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); let mut start = 0usize; diff --git a/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs index a33a9b6..4d993f6 100644 --- a/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs @@ -88,6 +88,7 @@ fn assert_backend_matches_scalar(backend: Backend) { #[test] fn sse2_matches_scalar() { assert_backend_matches_scalar(sse2::words); + assert_backend_matches_scalar_across_vector_boundaries(sse2::words); } #[cfg(all( @@ -97,10 +98,41 @@ fn sse2_matches_scalar() { #[test] fn avx2_matches_scalar() { assert_backend_matches_scalar(avx2::words); + assert_backend_matches_scalar_across_vector_boundaries(avx2::words); } #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] #[test] fn neon_matches_scalar() { assert_backend_matches_scalar(neon::words); + assert_backend_matches_scalar_across_vector_boundaries(neon::words); +} + +fn assert_backend_matches_scalar_across_vector_boundaries(backend: Backend) { + for len in [0, 1, 3, 4, 5, 15, 16, 17, 18, 31, 32, 33, 63, 64, 65] { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let src: Vec<_> = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }) + .collect(); + for amount in [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 31, 63, 64, 65, 127, 128, 129, 191, 256, 4096, + ] { + let expected = run_backend_owned(scalar::words, &src, amount); + assert_eq!( + run_backend_owned(backend, &src, amount), + expected, + "owned: len={len}, amount={amount}" + ); + assert_eq!( + run_backend_in_place(backend, &src, amount), + expected, + "in-place: len={len}, amount={amount}" + ); + } + } } From 30591e9a27bc41c9cfe14a71e945f510cbc6b9c5 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 17 Oct 2026 08:16:14 +0800 Subject: [PATCH 45/51] Document SSE2 carry prototype codegen and negative results Record why the rustc 1.98.1 SSE2-only carry experiments were not retained: left shift traded saved loads for copies/shuffles without a throughput win; right shift's scalar lookahead compiled to one SHRD per word and lost the existing kernel's two-word SIMD parallelism. Scope these findings to the measured prototypes rather than claiming an inherent SSE2 limitation. Comments only. Validation: cargo fmt --check and git diff --cached --check. --- src/traits/words_arith/funcs_for_shl_core/sse2.rs | 7 +++++-- src/traits/words_arith/funcs_for_shr_core/sse2.rs | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/traits/words_arith/funcs_for_shl_core/sse2.rs b/src/traits/words_arith/funcs_for_shl_core/sse2.rs index 5fb1d76..36cc744 100644 --- a/src/traits/words_arith/funcs_for_shl_core/sse2.rs +++ b/src/traits/words_arith/funcs_for_shl_core/sse2.rs @@ -39,8 +39,11 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } - // A tested in-place carry stream did not improve SSE2-only throughput. - // Keep the overlapping-load kernel. + // With rustc 1.98.1 and an SSE2-only target, a tested forward carry stream + // saved loads but added register copies and lane shuffles; large-input + // throughput was flat or worse. Saving overlapping loads does not imply + // fewer main-memory reads; load count alone did not predict this result. + // Keep this kernel unless a separately benchmarked SSE2 candidate wins. let bulk_start = word_shift + usize::from(bit_shift != 0); let mut end = word_len; diff --git a/src/traits/words_arith/funcs_for_shr_core/sse2.rs b/src/traits/words_arith/funcs_for_shr_core/sse2.rs index d35fd93..df42b36 100644 --- a/src/traits/words_arith/funcs_for_shr_core/sse2.rs +++ b/src/traits/words_arith/funcs_for_shr_core/sse2.rs @@ -39,8 +39,11 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } - // Tested reverse-carry and forward-lookahead streams regressed SSE2-only - // throughput. Keep the overlapping-load kernel. + // With rustc 1.98.1 and an SSE2-only target, tested reverse-carry and scalar + // forward-lookahead streams regressed. The latter compiled to one scalar + // SHRD per word, losing this kernel's two-word SIMD parallelism. This is a + // codegen result for those prototypes, not a limitation of SSE2 itself. + // Keep this kernel unless a separately benchmarked vector candidate wins. let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); let mut start = 0usize; From 09ac2257fe47fd70d49c23c0d237fe26977d0bca Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 17 Oct 2026 08:22:36 +0800 Subject: [PATCH 46/51] Experiment with SSE2 vector lookahead for in-place right shifts Keep two original words in a cached SSE2 vector, load the next pair before writing, and use SHUFPD immediate 1 to select [current.high, next.low]. Shift both lanes and retain the prefetched pair for the next iteration. Handle the final two or three words explicitly without overreading. Document the overlap invariant and lane selection. Add the exact two-word boundary to equivalence coverage. Restrict the candidate to exact overlap, 1..63-bit shifts and at least two words. Preserve the existing independent-destination, zero, whole-word and mixed-shift paths. No public API, width-specialized policy or dispatch change. The preceding 30591e9 separately records why scalar carry prototypes failed. SSE2-only validation uses target-cpu=x86-64, not native: target_feature(sse2) functions in a native build may otherwise inherit AVX2. Same-process rotating kernel comparisons pinned to CPU 0 showed roughly 4-8% less time for large in-place shifts across two runs. A four-vector unroll did not add stable gains and was discarded. Two 200-sample project A/B rounds in before/after/after/before order, CPU 0: 1M-character single PackedString removal medians, microseconds: BITS before range candidate range 1 2.041-2.047 1.882-1.883 2 4.007-4.008 3.802-3.807 3 5.868-5.987 5.581-5.712 4 8.040-9.154 7.434-7.436 5 9.939-10.130 9.284-9.519 6 11.830-11.960 11.370-11.460 7 13.890-13.910 13.090-13.260 8 15.770-16.130 14.890-15.380 Most widths improve by about 3-8%. This is a modest gain, not an AVX2-sized speedup. Allocation-inclusive owned and some single-operation control cases fluctuated; isolated kernels and repeated control shifts did not show a consistent regression. No universal cross-platform/workload claim is made. The candidate stays on the experiment branch for independent review. Validation: full SSE2-only and native tests, SSE2 release shr-core test, characterization bench compilation, fmt and cached diff checks. Standalone harness checks lengths 0..80, shifts 0..130/191/256/4096, independent/exact-overlap buffers, guard words and a one-u64 vector offset. It passed before timing. --- .../words_arith/funcs_for_shr_core/sse2.rs | 60 +++++++++++++++++-- .../tests_for_backend_equivalence.rs | 2 +- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/traits/words_arith/funcs_for_shr_core/sse2.rs b/src/traits/words_arith/funcs_for_shr_core/sse2.rs index df42b36..878bcfa 100644 --- a/src/traits/words_arith/funcs_for_shr_core/sse2.rs +++ b/src/traits/words_arith/funcs_for_shr_core/sse2.rs @@ -3,14 +3,14 @@ use crate::WORD_BITS; #[cfg(target_arch = "x86")] use core::arch::x86::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, + __m128i, _mm_castpd_si128, _mm_castsi128_pd, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, + _mm_shuffle_pd, _mm_sll_epi64, _mm_srl_epi64, _mm_storeu_si128, }; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, + __m128i, _mm_castpd_si128, _mm_castsi128_pd, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, + _mm_shuffle_pd, _mm_sll_epi64, _mm_srl_epi64, _mm_storeu_si128, }; const LANES: usize = 2; @@ -39,6 +39,58 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + if dst.cast_const() == src && word_shift == 0 && bit_shift != 0 && word_len >= LANES { + // SAFETY: current holds the original words [start, start + 1]. + // Each next vector is loaded before either current word is overwritten; + // retaining next preserves that invariant when start advances by two. + // The bulk bound leaves room for both vectors inside 0..word_len. + // At loop exit, two or three initialized words remain. The optional + // last word is read before the final vector store and is shifted once. + // Exact overlap and 1 <= bit_shift < 64 make every shift valid. + unsafe { + let right_count = _mm_cvtsi64_si128(bit_shift as i64); + let left_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + let mut start = 0; + let mut current = _mm_loadu_si128(src.cast::<__m128i>()); + while word_len - start >= 2 * LANES { + let next = _mm_loadu_si128(src.add(start + LANES).cast::<__m128i>()); + // Bit casts do not convert values. SHUFPD immediate 1 selects + // [current.high, next.low], the neighbors of both current words. + let neighbor = _mm_castpd_si128(_mm_shuffle_pd::<1>( + _mm_castsi128_pd(current), + _mm_castsi128_pd(next), + )); + let out = _mm_or_si128( + _mm_srl_epi64(current, right_count), + _mm_sll_epi64(neighbor, left_count), + ); + _mm_storeu_si128(dst.add(start).cast::<__m128i>(), out); + current = next; + start += LANES; + } + // One cached vector remains, optionally followed by one word. + let has_tail = word_len - start > LANES; + let tail = if has_tail { + src.add(start + LANES).read() + } else { + 0 + }; + let neighbor = _mm_castpd_si128(_mm_shuffle_pd::<1>( + _mm_castsi128_pd(current), + _mm_castsi128_pd(_mm_cvtsi64_si128(tail as i64)), + )); + let out = _mm_or_si128( + _mm_srl_epi64(current, right_count), + _mm_sll_epi64(neighbor, left_count), + ); + _mm_storeu_si128(dst.add(start).cast::<__m128i>(), out); + if has_tail { + dst.add(start + LANES).write(tail >> bit_shift); + } + } + return; + } + // With rustc 1.98.1 and an SSE2-only target, tested reverse-carry and scalar // forward-lookahead streams regressed. The latter compiled to one scalar // SHRD per word, losing this kernel's two-word SIMD parallelism. This is a diff --git a/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs index 4d993f6..0d8e6ef 100644 --- a/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs @@ -109,7 +109,7 @@ fn neon_matches_scalar() { } fn assert_backend_matches_scalar_across_vector_boundaries(backend: Backend) { - for len in [0, 1, 3, 4, 5, 15, 16, 17, 18, 31, 32, 33, 63, 64, 65] { + for len in [0, 1, 2, 3, 4, 5, 15, 16, 17, 18, 31, 32, 33, 63, 64, 65] { let mut state = 0x9e37_79b9_7f4a_7c15u64; let src: Vec<_> = (0..len) .map(|_| { From 2dafe6a608f9ee7dfcabbadd82aed2543224dcda Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 17 Oct 2026 12:04:11 +0800 Subject: [PATCH 47/51] Reuse packed character push in Extend --- src/packed_string/impls_for_editing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 989a51e..ae3921f 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -51,7 +51,7 @@ where self.bits.reserve_bits(additional_bits); } for character in iter { - self.bits.push_bits(u64::from(character.code()), width); + self.push(character); } } } From 431179e9b95dcb31423a11e0fdd0796f7eb8df76 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 17 Oct 2026 12:27:09 +0800 Subject: [PATCH 48/51] Batch packed codes when extending PackedString --- benches/packed_editing.rs | 41 +++++++++- .../impls_for_editing/impls_for_concat.rs | 20 +++++ src/packed_string/funcs_for_code.rs | 47 +++++++---- src/packed_string/impls_for_editing.rs | 22 ++++- .../impls_for_editing/tests_for_editing.rs | 81 +++++++++++++++++++ 5 files changed, 194 insertions(+), 17 deletions(-) diff --git a/benches/packed_editing.rs b/benches/packed_editing.rs index 5d50810..1117b8a 100644 --- a/benches/packed_editing.rs +++ b/benches/packed_editing.rs @@ -221,7 +221,15 @@ fn remove_middle_vec(bencher: Bencher, len: usize) { } fn extend_packed(bencher: Bencher, len: usize) { - let input = packed::(&codes(BITS, len)); + extend_packed_at::(bencher, len, len); +} + +fn extend_packed_unaligned(bencher: Bencher, len: usize) { + extend_packed_at::(bencher, len, len + 1); +} + +fn extend_packed_at(bencher: Bencher, len: usize, prefix_len: usize) { + let input = packed::(&codes(BITS, prefix_len)); let extension = codes(BITS, len / 4); bencher.with_inputs(|| input.clone()).bench_refs(|value| { value.extend(extension.iter().copied().map(Code)); @@ -230,6 +238,25 @@ fn extend_packed(bencher: Bencher, len: usize) { }); } +fn extend_from_chars_concat(bencher: Bencher, len: usize) { + extend_from_chars_concat_at::(bencher, len, len); +} + +fn extend_from_chars_concat_unaligned(bencher: Bencher, len: usize) { + extend_from_chars_concat_at::(bencher, len, len + 1); +} + +fn extend_from_chars_concat_at(bencher: Bencher, len: usize, prefix_len: usize) { + let input = packed::(&codes(BITS, prefix_len)); + let extension = codes(BITS, len / 4); + bencher.with_inputs(|| input.clone()).bench_refs(|value| { + let suffix = packed::(&extension); + value.push_packed_string(&suffix); + black_box(&*value); + black_box(value.char_len()) + }); +} + fn extend_vec(bencher: Bencher, len: usize) { let input = codes(BITS, len); let extension = codes(BITS, len / 4); @@ -291,6 +318,18 @@ macro_rules! define_case { pair!("insert_position/unaligned", insert_middle_packed, insert_middle_vec); pair!("remove_position/word_boundary_or_fallback", remove_middle_packed, remove_middle_vec); pair!("extend_bulk_25pct", extend_packed, extend_vec); + #[divan::bench(name = concat!("packed_editing/extend_bulk_25pct_unaligned/", stringify!($case), "/ours_packed_string"))] + fn extend_bulk_25pct_unaligned(bencher: Bencher) { + crate::extend_packed_unaligned::<$bits>(bencher, $len); + } + #[divan::bench(name = concat!("packed_editing/extend_from_chars_concat/", stringify!($case), "/ours_packed_string"))] + fn extend_from_chars_concat(bencher: Bencher) { + crate::extend_from_chars_concat::<$bits>(bencher, $len); + } + #[divan::bench(name = concat!("packed_editing/extend_from_chars_concat_unaligned/", stringify!($case), "/ours_packed_string"))] + fn extend_from_chars_concat_unaligned(bencher: Bencher) { + crate::extend_from_chars_concat_unaligned::<$bits>(bencher, $len); + } } }; } diff --git a/src/bit_string/impls_for_editing/impls_for_concat.rs b/src/bit_string/impls_for_editing/impls_for_concat.rs index 92e0731..a229070 100644 --- a/src/bit_string/impls_for_editing/impls_for_concat.rs +++ b/src/bit_string/impls_for_editing/impls_for_concat.rs @@ -49,6 +49,26 @@ impl BitString { self.bit_len = new_len; } + /// Appends a word-aligned packed block whose final word may be partial. + pub(crate) fn push_packed_words(&mut self, rhs_words: &[u64], rhs_bit_len: usize) { + if rhs_bit_len == 0 { + return; + } + + assert_eq!(rhs_words.len(), word_len(rhs_bit_len)); + + let old_bit_len = self.bit_len; + let new_bit_len = old_bit_len + .checked_add(rhs_bit_len) + .expect("bit string length overflow"); + self.words.resize(word_len(new_bit_len), 0); + rhs_words + .copy_bits(0, rhs_bit_len) + .paste_to(&mut self.words, old_bit_len); + self.bit_len = new_bit_len; + self.words.mask_unused_bits(new_bit_len); + } + /// Inserts the bits of `rhs` at `index`, shifting the tail right. /// /// `index` is clamped to `[0, self.bit_len()]` — an out-of-bounds index diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 87662ab..2b649bf 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -5,7 +5,7 @@ use crate::{BitString, WORD_BITS, word_len}; use super::*; -const CODE_BATCH_LEN: usize = 64; +pub(super) const CODE_BATCH_LEN: usize = 64; pub(super) fn pack_codes(chars: I) -> BitString where @@ -20,8 +20,34 @@ where words.reserve(word_len(bit_len)); } + let (tail_codes, tail_len) = for_each_aligned_code_batch::(chars, |codes| { + let start = words.len(); + words.resize(start + codes.len() * width / WORD_BITS, 0); + words[start..].pack_codes::(codes); + }); + let mut bit_len = words.len() * WORD_BITS; + for &code in &tail_codes[..tail_len] { + append_code::(&mut words, &mut bit_len, u64::from(code)); + } + + BitString::from_owned_words(words, bit_len).expect("packed words have an invalid length") +} + +/// Passes full code batches and final complete layout blocks to `consume`. +/// Returns any remaining codes that cannot form a word-aligned layout block. +/// +/// Construction and extension share this batching so extension can pack a +/// bounded batch at a time instead of materializing the entire input first. +pub(super) fn for_each_aligned_code_batch( + chars: I, + mut consume: F, +) -> ([u8; CODE_BATCH_LEN], usize) +where + C: PackedChar, + I: IntoIterator, + F: FnMut(&[u8]), +{ let layout_len = layout_block_len::(); - // 64 is a multiple of every layout block for BITS=1..8. let mut batch = [0u8; CODE_BATCH_LEN]; let mut batch_len = 0; @@ -30,26 +56,19 @@ where batch_len += 1; if batch_len == CODE_BATCH_LEN { - let start = words.len(); - words.resize(start + CODE_BATCH_LEN * width / WORD_BITS, 0); - words[start..].pack_codes::(&batch); + consume(&batch); batch_len = 0; } } let aligned_count = batch_len / layout_len * layout_len; if aligned_count != 0 { - let start = words.len(); - words.resize(start + aligned_count * width / WORD_BITS, 0); - words[start..].pack_codes::(&batch[..aligned_count]); + consume(&batch[..aligned_count]); } - let mut bit_len = words.len() * WORD_BITS; - for &code in &batch[aligned_count..batch_len] { - append_code::(&mut words, &mut bit_len, u64::from(code)); - } - - BitString::from_owned_words(words, bit_len).expect("packed words have an invalid length") + let tail_len = batch_len - aligned_count; + batch.copy_within(aligned_count..batch_len, 0); + (batch, tail_len) } #[inline] diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index ae3921f..d410c70 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -1,4 +1,6 @@ +use super::funcs_for_code::{CODE_BATCH_LEN, for_each_aligned_code_batch}; use super::*; +use crate::traits::WordsPack; impl PackedString where @@ -50,8 +52,24 @@ where if let Some(additional_bits) = lower_bound.checked_mul(width) { self.bits.reserve_bits(additional_bits); } - for character in iter { - self.push(character); + + let mut packed_words = [0u64; CODE_BATCH_LEN / 8]; + let bits = &mut self.bits; + // Keep scratch space bounded; collecting the whole iterator first would + // require a temporary PackedString for the complete suffix. + let (tail_codes, tail_len) = for_each_aligned_code_batch::(iter, |codes| { + let packed_bit_len = codes.len() * width; + let packed_word_len = packed_bit_len / crate::WORD_BITS; + packed_words[..packed_word_len].pack_codes::(codes); + bits.push_packed_words(&packed_words[..packed_word_len], packed_bit_len); + }); + + for &code in &tail_codes[..tail_len] { + if BITS == 8 { + self.bits.push_byte(code); + } else { + self.bits.push_bits(u64::from(code), width); + } } } } diff --git a/src/packed_string/impls_for_editing/tests_for_editing.rs b/src/packed_string/impls_for_editing/tests_for_editing.rs index 06fad30..69e6a22 100644 --- a/src/packed_string/impls_for_editing/tests_for_editing.rs +++ b/src/packed_string/impls_for_editing/tests_for_editing.rs @@ -1,4 +1,5 @@ use crate::packed_string::tests_for_support::{Letter, LetterString}; +use crate::traits::PackedChar; #[test] fn push_set_pop_and_truncate_preserve_values() { @@ -14,3 +15,83 @@ fn push_set_pop_and_truncate_preserve_values() { value.clear(); assert!(value.is_empty()); } + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + (code <= crate::code_mask::()).then_some(Self(code)) + } +} + +fn assert_extend_preserves_codes() { + for prefix_len in [0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 65] { + for extension_len in [0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 65, 127, 128, 129, 193] { + let code_at = |index: usize| { + (index.wrapping_mul(37).wrapping_add(13) as u8) & crate::code_mask::() + }; + let prefix = (0..prefix_len).map(|index| Code::(code_at(index))); + let extension = + (prefix_len..prefix_len + extension_len).map(|index| Code::(code_at(index))); + let expected = prefix + .clone() + .chain(extension.clone()) + .collect::>(); + + let mut actual = super::super::PackedString::, BITS>::from_chars(prefix); + actual.extend(extension); + + assert_eq!(actual.iter().collect::>(), expected); + } + } +} + +#[test] +fn extend_batches_preserve_codes_at_unaligned_tails_for_every_width() { + assert_extend_preserves_codes::<1>(); + assert_extend_preserves_codes::<2>(); + assert_extend_preserves_codes::<3>(); + assert_extend_preserves_codes::<4>(); + assert_extend_preserves_codes::<5>(); + assert_extend_preserves_codes::<6>(); + assert_extend_preserves_codes::<7>(); + assert_extend_preserves_codes::<8>(); +} + +fn assert_extend_with_zero_size_hint() { + let prefix_len = 31; + let extension_len = 193; + let code_at = + |index: usize| (index.wrapping_mul(37).wrapping_add(13) as u8) & crate::code_mask::(); + let prefix = (0..prefix_len).map(|index| Code::(code_at(index))); + let extension = (prefix_len..prefix_len + extension_len) + .map(|index| Code::(code_at(index))) + .collect::>(); + let expected = prefix + .clone() + .chain(extension.iter().copied()) + .collect::>(); + let mut actual = super::super::PackedString::, BITS>::from_chars(prefix); + let filtered = extension.iter().copied().filter(|_| true); + assert_eq!(filtered.size_hint().0, 0); + actual.extend(filtered); + + assert_eq!(actual.iter().collect::>(), expected); +} + +#[test] +fn extend_batches_do_not_depend_on_size_hint_lower_bound() { + assert_extend_with_zero_size_hint::<1>(); + assert_extend_with_zero_size_hint::<2>(); + assert_extend_with_zero_size_hint::<3>(); + assert_extend_with_zero_size_hint::<4>(); + assert_extend_with_zero_size_hint::<5>(); + assert_extend_with_zero_size_hint::<6>(); + assert_extend_with_zero_size_hint::<7>(); + assert_extend_with_zero_size_hint::<8>(); +} From 6e3c688b661eed7b3ef6051167b5d91ba8f4c23d Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 17 Oct 2026 12:50:12 +0800 Subject: [PATCH 49/51] Document and benchmark batched packed extension --- benches/packed_editing.rs | 80 +++++++++++++++++++ src/packed_string/impls_for_editing.rs | 13 ++- .../impls_for_editing/tests_for_editing.rs | 28 +++++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/benches/packed_editing.rs b/benches/packed_editing.rs index 1117b8a..6fca073 100644 --- a/benches/packed_editing.rs +++ b/benches/packed_editing.rs @@ -1,6 +1,7 @@ #[path = "packed_support/mod.rs"] mod support; +use bit_string::PackedString; use divan::{Bencher, black_box}; use support::{Code, aligned_index, codes, cross_word_index, packed, unaligned_index}; @@ -257,6 +258,53 @@ fn extend_from_chars_concat_at(bencher: Bencher, len: usize, pre }); } +fn extend_with_length(bencher: Bencher, extension_len: usize, concat: bool) { + let input = extend_length_input::(extension_len); + let extension = codes(BITS, extension_len); + bencher + .with_inputs(|| extend_length_bench_input(&input)) + .bench_refs(|value| { + if concat { + let suffix = packed::(&extension); + value.push_packed_string(&suffix); + } else { + value.extend(extension.iter().copied().map(Code)); + } + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn extend_per_character_with_length(bencher: Bencher, extension_len: usize) { + let input = extend_length_input::(extension_len); + let extension = codes(BITS, extension_len); + bencher + .with_inputs(|| extend_length_bench_input(&input)) + .bench_refs(|value| { + for code in extension.iter().copied() { + value.push(Code(code)); + } + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn extend_length_input(extension_len: usize) -> PackedString { + const PREFIX_LEN: usize = 1_048_576; + packed::(&codes(BITS, PREFIX_LEN + extension_len)) +} + +fn extend_length_bench_input( + input: &PackedString, +) -> PackedString { + const PREFIX_LEN: usize = 1_048_576; + // The source was built with spare logical words. Clone before truncating so + // every candidate gets the same retained capacity outside the timed region. + let mut value = input.clone(); + value.truncate(PREFIX_LEN); + value +} + fn extend_vec(bencher: Bencher, len: usize) { let input = codes(BITS, len); let extension = codes(BITS, len / 4); @@ -336,6 +384,38 @@ macro_rules! define_case { crate::for_each_packed_bench_case!(define_case); +macro_rules! define_extend_length_case { + ($case:ident, $bits:literal) => { + mod $case { + use super::*; + + #[divan::bench(args = [1, 16, 63, 64, 262144])] + fn streaming_batches(bencher: Bencher, extension_len: usize) { + crate::extend_with_length::<$bits>(bencher, extension_len, false); + } + + #[divan::bench(args = [1, 16, 63, 64, 262144])] + fn suffix_concat(bencher: Bencher, extension_len: usize) { + crate::extend_with_length::<$bits>(bencher, extension_len, true); + } + + #[divan::bench(args = [1, 16, 63, 64, 262144])] + fn per_character_preallocated(bencher: Bencher, extension_len: usize) { + crate::extend_per_character_with_length::<$bits>(bencher, extension_len); + } + } + }; +} + +define_extend_length_case!(extend_length_bits_1, 1); +define_extend_length_case!(extend_length_bits_2, 2); +define_extend_length_case!(extend_length_bits_3, 3); +define_extend_length_case!(extend_length_bits_4, 4); +define_extend_length_case!(extend_length_bits_5, 5); +define_extend_length_case!(extend_length_bits_6, 6); +define_extend_length_case!(extend_length_bits_7, 7); +define_extend_length_case!(extend_length_bits_8, 8); + // Adjacent lengths exercise backing-word growth/shrink and partial tails. macro_rules! define_edit_boundary_case { ($case:ident, $bits:literal) => { diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index d410c70..8f632f2 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -48,15 +48,26 @@ where fn extend>(&mut self, iter: I) { let iter = iter.into_iter(); let width = usize::from(BITS); - let (lower_bound, _) = iter.size_hint(); + let (lower_bound, upper_bound) = iter.size_hint(); + if let Some(additional_bits) = lower_bound.checked_mul(width) { self.bits.reserve_bits(additional_bits); } + // Avoid batch setup when the iterator can yield fewer than one batch. + if matches!(upper_bound, Some(upper_bound) if upper_bound < CODE_BATCH_LEN) { + for character in iter { + self.push(character); + } + return; + } + let mut packed_words = [0u64; CODE_BATCH_LEN / 8]; let bits = &mut self.bits; // Keep scratch space bounded; collecting the whole iterator first would // require a temporary PackedString for the complete suffix. + // If iteration panics, completed batches remain appended and the partial + // batch is discarded during unwinding. let (tail_codes, tail_len) = for_each_aligned_code_batch::(iter, |codes| { let packed_bit_len = codes.len() * width; let packed_word_len = packed_bit_len / crate::WORD_BITS; diff --git a/src/packed_string/impls_for_editing/tests_for_editing.rs b/src/packed_string/impls_for_editing/tests_for_editing.rs index 69e6a22..4189d91 100644 --- a/src/packed_string/impls_for_editing/tests_for_editing.rs +++ b/src/packed_string/impls_for_editing/tests_for_editing.rs @@ -95,3 +95,31 @@ fn extend_batches_do_not_depend_on_size_hint_lower_bound() { assert_extend_with_zero_size_hint::<7>(); assert_extend_with_zero_size_hint::<8>(); } + +#[test] +fn extend_panic_keeps_only_completed_batches() { + extern crate std; + + let mut value = super::super::PackedString::, 5>::new(); + let mut index = 0; + let iter = core::iter::from_fn(|| { + if index == 70 { + panic!("test iterator panic"); + } + + let code = Code::<5>((index as u8) & crate::code_mask::<5>()); + index += 1; + Some(code) + }); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| value.extend(iter))); + assert!(result.is_err()); + assert_eq!(index, 70); + assert_eq!(value.char_len(), 64); + assert_eq!( + value.iter().collect::>(), + (0..64) + .map(|index| Code::<5>((index as u8) & crate::code_mask::<5>())) + .collect::>() + ); +} From dc8e55ac35547d07c8f1f2bc1029124727569b2f Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 20 Oct 2026 21:36:04 +0800 Subject: [PATCH 50/51] Bump workspace version to 0.6.1 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35d8034..8706156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,7 +147,7 @@ dependencies = [ [[package]] name = "bit-string" -version = "0.6.0" +version = "0.6.1" dependencies = [ "bit-string-derive", "bitvec_simd", @@ -163,7 +163,7 @@ dependencies = [ [[package]] name = "bit-string-derive" -version = "0.6.0" +version = "0.6.1" dependencies = [ "proc-macro-crate", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index c002e79..d015105 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ exclude = ["/benches", "/src/**/tests_for_*", "/tests", ".github/"] members = ["derive"] [workspace.package] -version = "0.6.0" +version = "0.6.1" [features] characterization = [] From a1809232ce4617c80be058de0fbba0f031d8cf8b Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 20 Oct 2026 21:45:50 +0800 Subject: [PATCH 51/51] Distinguish packing and unpacking batch constants --- src/packed_string/funcs_for_code.rs | 10 ++++++---- src/packed_string/impls_for_conversion.rs | 15 ++++++++------- src/packed_string/impls_for_editing.rs | 6 +++--- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 2b649bf..5bab330 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -5,7 +5,9 @@ use crate::{BitString, WORD_BITS, word_len}; use super::*; -pub(super) const CODE_BATCH_LEN: usize = 64; +// Encoding batch size shared by construction and extension. A batch contains +// complete layout blocks for BITS=1..=8; tune independently of decoding batches. +pub(super) const PACK_CODE_BATCH_LEN: usize = 64; pub(super) fn pack_codes(chars: I) -> BitString where @@ -41,21 +43,21 @@ where pub(super) fn for_each_aligned_code_batch( chars: I, mut consume: F, -) -> ([u8; CODE_BATCH_LEN], usize) +) -> ([u8; PACK_CODE_BATCH_LEN], usize) where C: PackedChar, I: IntoIterator, F: FnMut(&[u8]), { let layout_len = layout_block_len::(); - let mut batch = [0u8; CODE_BATCH_LEN]; + let mut batch = [0u8; PACK_CODE_BATCH_LEN]; let mut batch_len = 0; for character in chars { batch[batch_len] = character.code(); batch_len += 1; - if batch_len == CODE_BATCH_LEN { + if batch_len == PACK_CODE_BATCH_LEN { consume(&batch); batch_len = 0; } diff --git a/src/packed_string/impls_for_conversion.rs b/src/packed_string/impls_for_conversion.rs index e93b162..1ce4355 100644 --- a/src/packed_string/impls_for_conversion.rs +++ b/src/packed_string/impls_for_conversion.rs @@ -8,27 +8,28 @@ where { /// Collects the decoded characters into a vector. pub fn to_vec(&self) -> alloc::vec::Vec { + // Decoding batch size, tuned independently of construction and extension. // 64 codes is a multiple of every layout block for BITS=1..=8. - const CODE_BATCH_LEN: usize = 64; + const UNPACK_CODE_BATCH_LEN: usize = 64; if !has_accelerated_bulk_unpack::() { return self.iter().collect(); } - debug_assert!(CODE_BATCH_LEN.is_multiple_of(layout_block_len::())); + debug_assert!(UNPACK_CODE_BATCH_LEN.is_multiple_of(layout_block_len::())); let char_len = self.char_len(); - if char_len < CODE_BATCH_LEN { + if char_len < UNPACK_CODE_BATCH_LEN { return self.iter().collect(); } - let bulk_code_len = char_len / CODE_BATCH_LEN * CODE_BATCH_LEN; + let bulk_code_len = char_len / UNPACK_CODE_BATCH_LEN * UNPACK_CODE_BATCH_LEN; let words = self.bits.words(); - let words_per_batch = CODE_BATCH_LEN * usize::from(BITS) / crate::WORD_BITS; - let mut decoded = [0u8; CODE_BATCH_LEN]; + let words_per_batch = UNPACK_CODE_BATCH_LEN * usize::from(BITS) / crate::WORD_BITS; + let mut decoded = [0u8; UNPACK_CODE_BATCH_LEN]; let mut result = alloc::vec::Vec::with_capacity(char_len); - for batch_index in 0..bulk_code_len / CODE_BATCH_LEN { + for batch_index in 0..bulk_code_len / UNPACK_CODE_BATCH_LEN { let word_start = batch_index * words_per_batch; let word_end = word_start + words_per_batch; words[word_start..word_end].unpack_codes::(&mut decoded); diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 8f632f2..fe9c4a5 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -1,4 +1,4 @@ -use super::funcs_for_code::{CODE_BATCH_LEN, for_each_aligned_code_batch}; +use super::funcs_for_code::{PACK_CODE_BATCH_LEN, for_each_aligned_code_batch}; use super::*; use crate::traits::WordsPack; @@ -55,14 +55,14 @@ where } // Avoid batch setup when the iterator can yield fewer than one batch. - if matches!(upper_bound, Some(upper_bound) if upper_bound < CODE_BATCH_LEN) { + if matches!(upper_bound, Some(upper_bound) if upper_bound < PACK_CODE_BATCH_LEN) { for character in iter { self.push(character); } return; } - let mut packed_words = [0u64; CODE_BATCH_LEN / 8]; + let mut packed_words = [0u64; PACK_CODE_BATCH_LEN / 8]; let bits = &mut self.bits; // Keep scratch space bounded; collecting the whole iterator first would // require a temporary PackedString for the complete suffix.