From aab8b34c088a209044c9644fcc0054c1b9dc4ebb Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 2 Aug 2026 11:03:15 +0000 Subject: [PATCH 01/79] feat: add enum-backed packed string --- src/lib.rs | 4 +- src/packed_string.rs | 29 +++++++++ src/packed_string/funcs_for_code.rs | 43 +++++++++++++ .../funcs_for_code/tests_for_code.rs | 22 +++++++ src/packed_string/impls_for_access.rs | 57 +++++++++++++++++ .../impls_for_access/tests_for_access.rs | 18 ++++++ src/packed_string/impls_for_construction.rs | 38 +++++++++++ .../tests_for_construction.rs | 17 +++++ src/packed_string/impls_for_editing.rs | 59 +++++++++++++++++ .../impls_for_editing/tests_for_editing.rs | 17 +++++ src/packed_string/impls_for_eq.rs | 12 ++++ .../impls_for_eq/tests_for_eq.rs | 11 ++++ src/packed_string/impls_for_fmt.rs | 21 ++++++ .../impls_for_fmt/tests_for_fmt.rs | 15 +++++ src/packed_string/impls_for_iter.rs | 64 +++++++++++++++++++ .../impls_for_iter/tests_for_iter.rs | 13 ++++ src/packed_string/tests_for_support.rs | 56 ++++++++++++++++ src/traits.rs | 2 + src/traits/packed_char.rs | 22 +++++++ 19 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 src/packed_string.rs create mode 100644 src/packed_string/funcs_for_code.rs create mode 100644 src/packed_string/funcs_for_code/tests_for_code.rs create mode 100644 src/packed_string/impls_for_access.rs create mode 100644 src/packed_string/impls_for_access/tests_for_access.rs create mode 100644 src/packed_string/impls_for_construction.rs create mode 100644 src/packed_string/impls_for_construction/tests_for_construction.rs create mode 100644 src/packed_string/impls_for_editing.rs create mode 100644 src/packed_string/impls_for_editing/tests_for_editing.rs create mode 100644 src/packed_string/impls_for_eq.rs create mode 100644 src/packed_string/impls_for_eq/tests_for_eq.rs create mode 100644 src/packed_string/impls_for_fmt.rs create mode 100644 src/packed_string/impls_for_fmt/tests_for_fmt.rs create mode 100644 src/packed_string/impls_for_iter.rs create mode 100644 src/packed_string/impls_for_iter/tests_for_iter.rs create mode 100644 src/packed_string/tests_for_support.rs create mode 100644 src/traits/packed_char.rs diff --git a/src/lib.rs b/src/lib.rs index e2864ca..c6edb70 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,15 +5,17 @@ extern crate alloc; mod consts_for_bits; mod funcs_for_bits; -pub(crate) mod traits; +pub mod traits; pub(crate) use consts_for_bits::*; pub(crate) use funcs_for_bits::*; mod bit_str; mod bit_string; +mod packed_string; pub use bit_str::BitStr; pub use bit_string::BitString; +pub use packed_string::PackedString; pub use bit_str::errors::*; pub use bit_string::errors::*; diff --git a/src/packed_string.rs b/src/packed_string.rs new file mode 100644 index 0000000..9b22213 --- /dev/null +++ b/src/packed_string.rs @@ -0,0 +1,29 @@ +//! A packed sequence whose character type defines its own bit encoding. + +use core::marker::PhantomData; + +use crate::{BitString, traits::PackedChar}; + +/// An owned sequence of fixed-width [`PackedChar`] values. +/// +/// The character type is the alphabet and its enum discriminant is the stored +/// value, so no alphabet allocation or lookup table is required. +#[derive(Clone)] +pub struct PackedString { + bits: BitString, + char_len: usize, + marker: PhantomData C>, +} + +mod funcs_for_code; +mod impls_for_access; +mod impls_for_construction; +mod impls_for_editing; +mod impls_for_eq; +mod impls_for_fmt; +mod impls_for_iter; + +use funcs_for_code::{assert_valid_width, checked_code, code_mask, write_code}; + +#[cfg(test)] +mod tests_for_support; diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs new file mode 100644 index 0000000..f3c0627 --- /dev/null +++ b/src/packed_string/funcs_for_code.rs @@ -0,0 +1,43 @@ +use super::*; + +#[inline] +pub(super) fn assert_valid_width() { + assert!(C::BITS <= 8, "PackedChar::BITS must not exceed 8"); +} + +#[inline] +pub(super) fn code_mask() -> u8 { + match C::BITS { + 0 => 0, + 1..=7 => (1u8 << C::BITS) - 1, + 8 => u8::MAX, + _ => unreachable!("PackedChar::BITS must not exceed 8"), + } +} + +#[inline] +pub(super) fn checked_code(character: C) -> u8 { + assert_valid_width::(); + let code = character.code(); + assert_eq!( + code & !code_mask::(), + 0, + "PackedChar::code does not fit in PackedChar::BITS" + ); + assert!( + C::from_code(code) == Some(character), + "PackedChar::code and PackedChar::from_code disagree" + ); + code +} + +#[inline] +pub(super) fn write_code(bits: &mut BitString, position: usize, width: u8, code: u8) { + let start = position * usize::from(width); + for shift in 0..width { + let _ = bits.set(start + usize::from(shift), (code >> shift) & 1 != 0); + } +} + +#[cfg(test)] +mod tests_for_code; diff --git a/src/packed_string/funcs_for_code/tests_for_code.rs b/src/packed_string/funcs_for_code/tests_for_code.rs new file mode 100644 index 0000000..bbeb65a --- /dev/null +++ b/src/packed_string/funcs_for_code/tests_for_code.rs @@ -0,0 +1,22 @@ +use super::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Broken; + +impl PackedChar for Broken { + const BITS: u8 = 1; + + fn code(self) -> u8 { + 2 + } + + fn from_code(_: u8) -> Option { + Some(Self) + } +} + +#[test] +#[should_panic(expected = "does not fit")] +fn rejects_a_code_that_does_not_fit() { + let _ = checked_code(Broken); +} diff --git a/src/packed_string/impls_for_access.rs b/src/packed_string/impls_for_access.rs new file mode 100644 index 0000000..eed01bd --- /dev/null +++ b/src/packed_string/impls_for_access.rs @@ -0,0 +1,57 @@ +use super::*; + +impl PackedString { + /// Number of packed characters, not number of bits. + #[inline] + pub fn char_len(&self) -> usize { + self.char_len + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.char_len == 0 + } + + #[inline] + pub const fn bits_per_char(&self) -> usize { + C::BITS as usize + } + + #[inline] + pub fn bits(&self) -> &BitString { + &self.bits + } + + #[inline] + pub fn get(&self, index: usize) -> Option { + if index >= self.char_len { + return None; + } + Some( + C::from_code(self.code_at(index)) + .expect("PackedChar rejected a code it previously produced"), + ) + } + + #[inline] + pub fn first(&self) -> Option { + self.get(0) + } + + #[inline] + pub fn last(&self) -> Option { + self.char_len.checked_sub(1).and_then(|i| self.get(i)) + } + + #[inline] + fn code_at(&self, index: usize) -> u8 { + if C::BITS == 0 { + return 0; + } + let start = index * usize::from(C::BITS); + (self.bits.get_chunk(start) & u64::from(code_mask::())) as u8 + } +} + +#[cfg(test)] +mod tests_for_access; diff --git a/src/packed_string/impls_for_access/tests_for_access.rs b/src/packed_string/impls_for_access/tests_for_access.rs new file mode 100644 index 0000000..aebf078 --- /dev/null +++ b/src/packed_string/impls_for_access/tests_for_access.rs @@ -0,0 +1,18 @@ +use super::*; +use crate::packed_string::tests_for_support::{Letter, Only}; + +#[test] +fn get_first_and_last_decode_values() { + let value = PackedString::from_chars([Letter::A, Letter::B, Letter::C]); + assert_eq!(value.first(), Some(Letter::A)); + assert_eq!(value.last(), Some(Letter::C)); + assert_eq!(value.get(3), None); +} + +#[test] +fn one_value_type_needs_no_payload_bits() { + let value = PackedString::from_chars([Only; 100]); + assert_eq!(value.char_len(), 100); + assert_eq!(value.bits().bit_len(), 0); + assert_eq!(value.get(99), Some(Only)); +} diff --git a/src/packed_string/impls_for_construction.rs b/src/packed_string/impls_for_construction.rs new file mode 100644 index 0000000..a529191 --- /dev/null +++ b/src/packed_string/impls_for_construction.rs @@ -0,0 +1,38 @@ +use super::*; + +impl PackedString { + #[inline] + pub fn new() -> Self { + assert_valid_width::(); + Self { + bits: BitString::new(), + char_len: 0, + marker: PhantomData, + } + } + + /// Packs enum values directly, without an alphabet lookup. + pub fn from_chars(chars: I) -> Self + where + I: IntoIterator, + { + let mut result = Self::new(); + result.extend(chars); + result + } +} + +impl Default for PackedString { + fn default() -> Self { + Self::new() + } +} + +impl FromIterator for PackedString { + fn from_iter>(iter: I) -> Self { + Self::from_chars(iter) + } +} + +#[cfg(test)] +mod tests_for_construction; diff --git a/src/packed_string/impls_for_construction/tests_for_construction.rs b/src/packed_string/impls_for_construction/tests_for_construction.rs new file mode 100644 index 0000000..3f94ed5 --- /dev/null +++ b/src/packed_string/impls_for_construction/tests_for_construction.rs @@ -0,0 +1,17 @@ +use super::*; +use crate::packed_string::tests_for_support::Letter; + +#[test] +fn enum_discriminants_are_stored_directly() { + let value = PackedString::from_chars([Letter::A, Letter::B, Letter::C, Letter::D]); + assert_eq!(value.bits_per_char(), 2); + assert_eq!(value.bits().bit_len(), 8); + assert_eq!(value.bits().get_chunk(0), 0b11_10_01_00); +} + +#[test] +fn collect_constructs_a_packed_string() { + let value: PackedString = [Letter::A, Letter::C].into_iter().collect(); + assert_eq!(value.char_len(), 2); + assert_eq!(value.get(0), Some(Letter::A)); +} diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs new file mode 100644 index 0000000..ff84237 --- /dev/null +++ b/src/packed_string/impls_for_editing.rs @@ -0,0 +1,59 @@ +use super::*; + +impl PackedString { + /// Appends the character's code directly to the payload. + pub fn push(&mut self, character: C) { + let code = checked_code(character); + let new_len = self + .char_len + .checked_add(1) + .expect("packed string length overflow"); + for shift in 0..C::BITS { + self.bits.push((code >> shift) & 1 != 0); + } + self.char_len = new_len; + } + + pub fn pop(&mut self) -> Option { + let character = self.last()?; + self.char_len -= 1; + self.bits.truncate(self.char_len * usize::from(C::BITS)); + Some(character) + } + + pub fn set(&mut self, index: usize, character: C) -> Option { + let previous = self.get(index)?; + write_code(&mut self.bits, index, C::BITS, checked_code(character)); + Some(previous) + } + + pub fn truncate(&mut self, new_len: usize) { + if new_len >= self.char_len { + return; + } + self.char_len = new_len; + self.bits.truncate(new_len * usize::from(C::BITS)); + } + + pub fn clear(&mut self) { + self.char_len = 0; + self.bits.clear(); + } +} + +impl Extend for PackedString { + fn extend>(&mut self, iter: I) { + for character in iter { + self.push(character); + } + } +} + +impl<'a, C: PackedChar> Extend<&'a C> for PackedString { + fn extend>(&mut self, iter: I) { + self.extend(iter.into_iter().copied()); + } +} + +#[cfg(test)] +mod tests_for_editing; diff --git a/src/packed_string/impls_for_editing/tests_for_editing.rs b/src/packed_string/impls_for_editing/tests_for_editing.rs new file mode 100644 index 0000000..b773e6d --- /dev/null +++ b/src/packed_string/impls_for_editing/tests_for_editing.rs @@ -0,0 +1,17 @@ +use super::*; +use crate::packed_string::tests_for_support::Letter; + +#[test] +fn push_set_pop_and_truncate_preserve_values() { + let mut value = PackedString::from_chars([Letter::A, Letter::B, Letter::C]); + value.push(Letter::D); + assert_eq!(value.set(1, Letter::C), Some(Letter::B)); + assert_eq!(value.pop(), Some(Letter::D)); + value.truncate(2); + assert_eq!( + value.iter().collect::>(), + [Letter::A, Letter::C] + ); + value.clear(); + assert!(value.is_empty()); +} diff --git a/src/packed_string/impls_for_eq.rs b/src/packed_string/impls_for_eq.rs new file mode 100644 index 0000000..a6596ca --- /dev/null +++ b/src/packed_string/impls_for_eq.rs @@ -0,0 +1,12 @@ +use super::*; + +impl PartialEq for PackedString { + fn eq(&self, other: &Self) -> bool { + self.char_len == other.char_len && self.bits == other.bits + } +} + +impl Eq for PackedString {} + +#[cfg(test)] +mod tests_for_eq; diff --git a/src/packed_string/impls_for_eq/tests_for_eq.rs b/src/packed_string/impls_for_eq/tests_for_eq.rs new file mode 100644 index 0000000..ea6a0b9 --- /dev/null +++ b/src/packed_string/impls_for_eq/tests_for_eq.rs @@ -0,0 +1,11 @@ +use super::*; +use crate::packed_string::tests_for_support::Letter; + +#[test] +fn equality_uses_length_and_packed_values() { + let left = PackedString::from_chars([Letter::A, Letter::B]); + let same = PackedString::from_chars([Letter::A, Letter::B]); + let different = PackedString::from_chars([Letter::A, Letter::C]); + assert_eq!(left, same); + assert_ne!(left, different); +} diff --git a/src/packed_string/impls_for_fmt.rs b/src/packed_string/impls_for_fmt.rs new file mode 100644 index 0000000..f2854de --- /dev/null +++ b/src/packed_string/impls_for_fmt.rs @@ -0,0 +1,21 @@ +use core::fmt; + +use super::*; + +impl fmt::Debug for PackedString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl fmt::Display for PackedString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for character in self.iter() { + fmt::Display::fmt(&character, f)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests_for_fmt; diff --git a/src/packed_string/impls_for_fmt/tests_for_fmt.rs b/src/packed_string/impls_for_fmt/tests_for_fmt.rs new file mode 100644 index 0000000..14b428f --- /dev/null +++ b/src/packed_string/impls_for_fmt/tests_for_fmt.rs @@ -0,0 +1,15 @@ +use super::*; +use crate::packed_string::tests_for_support::Letter; +use alloc::string::ToString; + +#[test] +fn display_uses_the_character_display_implementation() { + let value = PackedString::from_chars([Letter::A, Letter::B, Letter::C, Letter::D]); + assert_eq!(value.to_string(), "ABCD"); +} + +#[test] +fn debug_lists_the_enum_values() { + let value = PackedString::from_chars([Letter::A, Letter::B]); + assert_eq!(alloc::format!("{value:?}"), "[A, B]"); +} diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs new file mode 100644 index 0000000..b23d1e1 --- /dev/null +++ b/src/packed_string/impls_for_iter.rs @@ -0,0 +1,64 @@ +use core::iter::FusedIterator; + +use super::*; + +impl PackedString { + #[inline] + pub fn iter(&self) -> Iter<'_, C> { + Iter { + string: self, + front: 0, + back: self.char_len, + } + } +} + +#[derive(Clone)] +pub struct Iter<'a, C: PackedChar> { + string: &'a PackedString, + front: usize, + back: usize, +} + +impl Iterator for Iter<'_, C> { + type Item = C; + + fn next(&mut self) -> Option { + if self.front == self.back { + return None; + } + let character = self.string.get(self.front); + self.front += 1; + character + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.back - self.front; + (len, Some(len)) + } +} + +impl DoubleEndedIterator for Iter<'_, C> { + fn next_back(&mut self) -> Option { + if self.front == self.back { + return None; + } + self.back -= 1; + self.string.get(self.back) + } +} + +impl ExactSizeIterator for Iter<'_, C> {} +impl FusedIterator for Iter<'_, C> {} + +impl<'a, C: PackedChar> IntoIterator for &'a PackedString { + type Item = C; + type IntoIter = Iter<'a, C>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +#[cfg(test)] +mod tests_for_iter; diff --git a/src/packed_string/impls_for_iter/tests_for_iter.rs b/src/packed_string/impls_for_iter/tests_for_iter.rs new file mode 100644 index 0000000..e8aebb9 --- /dev/null +++ b/src/packed_string/impls_for_iter/tests_for_iter.rs @@ -0,0 +1,13 @@ +use super::*; +use crate::packed_string::tests_for_support::Letter; + +#[test] +fn iter_is_double_ended_and_exact_size() { + let value = PackedString::from_chars([Letter::A, Letter::B, Letter::C]); + let mut iter = value.iter(); + assert_eq!(iter.len(), 3); + assert_eq!(iter.next(), Some(Letter::A)); + assert_eq!(iter.next_back(), Some(Letter::C)); + assert_eq!(iter.next(), Some(Letter::B)); + assert_eq!(iter.next(), None); +} diff --git a/src/packed_string/tests_for_support.rs b/src/packed_string/tests_for_support.rs new file mode 100644 index 0000000..df0af10 --- /dev/null +++ b/src/packed_string/tests_for_support.rs @@ -0,0 +1,56 @@ +use core::fmt; + +use super::PackedChar; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub(super) enum Letter { + A = 0b00, + B = 0b01, + C = 0b10, + D = 0b11, +} + +impl PackedChar for Letter { + const BITS: u8 = 2; + + fn code(self) -> u8 { + self as u8 + } + + fn from_code(code: u8) -> Option { + match code { + 0 => Some(Self::A), + 1 => Some(Self::B), + 2 => Some(Self::C), + 3 => Some(Self::D), + _ => None, + } + } +} + +impl fmt::Display for Letter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::A => "A", + Self::B => "B", + Self::C => "C", + Self::D => "D", + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Only; + +impl PackedChar for Only { + const BITS: u8 = 0; + + fn code(self) -> u8 { + 0 + } + + fn from_code(code: u8) -> Option { + (code == 0).then_some(Self) + } +} diff --git a/src/traits.rs b/src/traits.rs index 8723483..c682bf3 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,3 +1,4 @@ +mod packed_char; pub(crate) mod word_ord; pub(crate) mod words_arith; pub(crate) mod words_edit; @@ -6,6 +7,7 @@ pub(crate) mod words_find; pub(crate) mod words_ord; pub(crate) mod words_scan; +pub use packed_char::PackedChar; pub(crate) use word_ord::*; pub(crate) use words_arith::*; pub(crate) use words_edit::*; diff --git a/src/traits/packed_char.rs b/src/traits/packed_char.rs new file mode 100644 index 0000000..e8bc09e --- /dev/null +++ b/src/traits/packed_char.rs @@ -0,0 +1,22 @@ +/// A character with a fixed-width bit representation. +/// +/// This trait is intended for fieldless `#[repr(u8)]` enums. Implementations +/// must satisfy all of the following invariants: +/// +/// - [`BITS`](Self::BITS) does not exceed eight; +/// - [`code`](Self::code) fits in `BITS` bits; +/// - `from_code(code(value)) == Some(value)`; +/// - each valid code identifies at most one value. +/// +/// [`PackedString`](crate::PackedString) checks the first three invariants +/// whenever it writes a value. No `transmute` is used. +pub trait PackedChar: Copy + Eq { + /// Number of bits occupied by one character. + const BITS: u8; + + /// Returns the character's packed value. + fn code(self) -> u8; + + /// Decodes a packed value, rejecting unused bit patterns. + fn from_code(code: u8) -> Option; +} From 1630b5c02c698bd80e7ad322cebaf8026f7e6db7 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 2 Aug 2026 11:54:00 +0000 Subject: [PATCH 02/79] feat: scaffold packed string API --- src/lib.rs | 2 + src/packed_str.rs | 23 ++++++++++ src/packed_str/impls_for_access.rs | 31 +++++++++++++ src/packed_str/impls_for_conversion.rs | 13 ++++++ src/packed_str/impls_for_eq.rs | 9 ++++ src/packed_str/impls_for_iter.rs | 46 +++++++++++++++++++ src/packed_str/impls_for_matching.rs | 35 ++++++++++++++ src/packed_str/impls_for_ord.rs | 15 ++++++ src/packed_str/impls_for_slice.rs | 17 +++++++ src/packed_string.rs | 3 ++ src/packed_string/impls_for_construction.rs | 4 ++ .../impls_for_from_bits.rs | 16 +++++++ .../impls_for_from_slice.rs | 13 ++++++ .../impls_for_repeat.rs | 7 +++ src/packed_string/impls_for_editing.rs | 8 ++++ .../impls_for_editing/impls_for_concat.rs | 15 ++++++ .../impls_for_editing/impls_for_drain.rs | 13 ++++++ .../impls_for_insert_remove.rs | 11 +++++ .../impls_for_editing/impls_for_replace.rs | 21 +++++++++ .../impls_for_editing/impls_for_retain.rs | 10 ++++ .../impls_for_editing/impls_for_reverse.rs | 11 +++++ .../impls_for_editing/impls_for_slice.rs | 17 +++++++ src/packed_string/impls_for_hash.rs | 9 ++++ src/packed_string/impls_for_iter.rs | 5 ++ src/packed_string/impls_for_matching.rs | 35 ++++++++++++++ src/packed_string/impls_for_ord.rs | 22 +++++++++ 26 files changed, 411 insertions(+) create mode 100644 src/packed_str.rs create mode 100644 src/packed_str/impls_for_access.rs create mode 100644 src/packed_str/impls_for_conversion.rs create mode 100644 src/packed_str/impls_for_eq.rs create mode 100644 src/packed_str/impls_for_iter.rs create mode 100644 src/packed_str/impls_for_matching.rs create mode 100644 src/packed_str/impls_for_ord.rs create mode 100644 src/packed_str/impls_for_slice.rs create mode 100644 src/packed_string/impls_for_construction/impls_for_from_bits.rs create mode 100644 src/packed_string/impls_for_construction/impls_for_from_slice.rs create mode 100644 src/packed_string/impls_for_construction/impls_for_repeat.rs create mode 100644 src/packed_string/impls_for_editing/impls_for_concat.rs create mode 100644 src/packed_string/impls_for_editing/impls_for_drain.rs create mode 100644 src/packed_string/impls_for_editing/impls_for_insert_remove.rs create mode 100644 src/packed_string/impls_for_editing/impls_for_replace.rs create mode 100644 src/packed_string/impls_for_editing/impls_for_retain.rs create mode 100644 src/packed_string/impls_for_editing/impls_for_reverse.rs create mode 100644 src/packed_string/impls_for_editing/impls_for_slice.rs create mode 100644 src/packed_string/impls_for_hash.rs create mode 100644 src/packed_string/impls_for_matching.rs create mode 100644 src/packed_string/impls_for_ord.rs diff --git a/src/lib.rs b/src/lib.rs index c6edb70..ec1a492 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,10 +11,12 @@ pub(crate) use funcs_for_bits::*; mod bit_str; mod bit_string; +mod packed_str; mod packed_string; pub use bit_str::BitStr; pub use bit_string::BitString; +pub use packed_str::PackedStr; pub use packed_string::PackedString; pub use bit_str::errors::*; diff --git a/src/packed_str.rs b/src/packed_str.rs new file mode 100644 index 0000000..ee03ff9 --- /dev/null +++ b/src/packed_str.rs @@ -0,0 +1,23 @@ +//! A zero-copy borrowed view into a [`PackedString`](crate::PackedString). + +use core::marker::PhantomData; + +use crate::{PackedString, traits::PackedChar}; + +/// A character-aligned borrowed view into a [`PackedString`]. +#[derive(Clone, Copy)] +#[allow(dead_code)] +pub struct PackedStr<'ps, C: PackedChar> { + source: &'ps PackedString, + char_start: usize, + char_len: usize, + marker: PhantomData C>, +} + +mod impls_for_access; +mod impls_for_conversion; +mod impls_for_eq; +mod impls_for_iter; +mod impls_for_matching; +mod impls_for_ord; +mod impls_for_slice; diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs new file mode 100644 index 0000000..037d545 --- /dev/null +++ b/src/packed_str/impls_for_access.rs @@ -0,0 +1,31 @@ +use super::*; + +impl<'ps, C: PackedChar> PackedStr<'ps, C> { + pub fn source(&self) -> &'ps PackedString { + unimplemented!("PackedStr::source") + } + + pub fn char_start(&self) -> usize { + unimplemented!("PackedStr::char_start") + } + + pub fn char_len(&self) -> usize { + unimplemented!("PackedStr::char_len") + } + + pub fn is_empty(&self) -> bool { + unimplemented!("PackedStr::is_empty") + } + + pub fn get(&self, _index: usize) -> Option { + unimplemented!("PackedStr::get") + } + + pub fn first(&self) -> Option { + unimplemented!("PackedStr::first") + } + + pub fn last(&self) -> Option { + unimplemented!("PackedStr::last") + } +} diff --git a/src/packed_str/impls_for_conversion.rs b/src/packed_str/impls_for_conversion.rs new file mode 100644 index 0000000..b9d61de --- /dev/null +++ b/src/packed_str/impls_for_conversion.rs @@ -0,0 +1,13 @@ +use super::*; + +impl<'ps, C: PackedChar> PackedStr<'ps, C> { + pub fn to_packed_string(&self) -> PackedString { + unimplemented!("PackedStr::to_packed_string") + } +} + +impl PackedString { + pub fn as_packed_str(&self) -> PackedStr<'_, C> { + unimplemented!("PackedString::as_packed_str") + } +} diff --git a/src/packed_str/impls_for_eq.rs b/src/packed_str/impls_for_eq.rs new file mode 100644 index 0000000..9ee3182 --- /dev/null +++ b/src/packed_str/impls_for_eq.rs @@ -0,0 +1,9 @@ +use super::*; + +impl PartialEq for PackedStr<'_, C> { + fn eq(&self, _other: &Self) -> bool { + unimplemented!("PackedStr::eq") + } +} + +impl Eq for PackedStr<'_, C> {} diff --git a/src/packed_str/impls_for_iter.rs b/src/packed_str/impls_for_iter.rs new file mode 100644 index 0000000..08c4d34 --- /dev/null +++ b/src/packed_str/impls_for_iter.rs @@ -0,0 +1,46 @@ +use core::iter::FusedIterator; + +use super::*; + +impl<'ps, C: PackedChar> PackedStr<'ps, C> { + pub fn iter(&self) -> Iter<'ps, C> { + unimplemented!("PackedStr::iter") + } +} + +#[allow(dead_code)] +pub struct Iter<'ps, C: PackedChar> { + view: PackedStr<'ps, C>, + front: usize, + back: usize, +} + +impl Iterator for Iter<'_, C> { + type Item = C; + + fn next(&mut self) -> Option { + unimplemented!("PackedStr::Iter::next") + } + + fn size_hint(&self) -> (usize, Option) { + unimplemented!("PackedStr::Iter::size_hint") + } +} + +impl DoubleEndedIterator for Iter<'_, C> { + fn next_back(&mut self) -> Option { + unimplemented!("PackedStr::Iter::next_back") + } +} + +impl ExactSizeIterator for Iter<'_, C> {} +impl FusedIterator for Iter<'_, C> {} + +impl<'ps, C: PackedChar> IntoIterator for &'ps PackedStr<'ps, C> { + type Item = C; + type IntoIter = Iter<'ps, C>; + + fn into_iter(self) -> Self::IntoIter { + unimplemented!("PackedStr::into_iter") + } +} diff --git a/src/packed_str/impls_for_matching.rs b/src/packed_str/impls_for_matching.rs new file mode 100644 index 0000000..5e488d5 --- /dev/null +++ b/src/packed_str/impls_for_matching.rs @@ -0,0 +1,35 @@ +use super::*; + +impl PackedStr<'_, C> { + pub fn matches_at(&self, _index: usize, _pattern: Self) -> bool { + unimplemented!("PackedStr::matches_at") + } + + pub fn starts_with(&self, _prefix: Self) -> bool { + unimplemented!("PackedStr::starts_with") + } + + pub fn ends_with(&self, _suffix: Self) -> bool { + unimplemented!("PackedStr::ends_with") + } + + pub fn contains(&self, _needle: Self) -> bool { + unimplemented!("PackedStr::contains") + } + + pub fn find(&self, _needle: Self) -> Option { + unimplemented!("PackedStr::find") + } + + pub fn rfind(&self, _needle: Self) -> Option { + unimplemented!("PackedStr::rfind") + } + + pub fn strip_prefix(&self, _prefix: Self) -> Option { + unimplemented!("PackedStr::strip_prefix") + } + + pub fn strip_suffix(&self, _suffix: Self) -> Option { + unimplemented!("PackedStr::strip_suffix") + } +} diff --git a/src/packed_str/impls_for_ord.rs b/src/packed_str/impls_for_ord.rs new file mode 100644 index 0000000..a9327e3 --- /dev/null +++ b/src/packed_str/impls_for_ord.rs @@ -0,0 +1,15 @@ +use core::cmp::Ordering; + +use super::*; + +impl PartialOrd for PackedStr<'_, C> { + fn partial_cmp(&self, _other: &Self) -> Option { + unimplemented!("PackedStr::partial_cmp") + } +} + +impl Ord for PackedStr<'_, C> { + fn cmp(&self, _other: &Self) -> Ordering { + unimplemented!("PackedStr::cmp") + } +} diff --git a/src/packed_str/impls_for_slice.rs b/src/packed_str/impls_for_slice.rs new file mode 100644 index 0000000..8bd5227 --- /dev/null +++ b/src/packed_str/impls_for_slice.rs @@ -0,0 +1,17 @@ +use int_interval::UsizeCO; + +use super::*; + +impl<'ps, C: PackedChar> PackedStr<'ps, C> { + pub fn slice(&self, _interval: UsizeCO) -> Self { + unimplemented!("PackedStr::slice") + } + + pub fn slice_from(&self, _start: usize) -> Self { + unimplemented!("PackedStr::slice_from") + } + + pub fn slice_until(&self, _end: usize) -> Self { + unimplemented!("PackedStr::slice_until") + } +} diff --git a/src/packed_string.rs b/src/packed_string.rs index 9b22213..8c3fbd6 100644 --- a/src/packed_string.rs +++ b/src/packed_string.rs @@ -21,7 +21,10 @@ mod impls_for_construction; mod impls_for_editing; mod impls_for_eq; mod impls_for_fmt; +mod impls_for_hash; mod impls_for_iter; +mod impls_for_matching; +mod impls_for_ord; use funcs_for_code::{assert_valid_width, checked_code, code_mask, write_code}; diff --git a/src/packed_string/impls_for_construction.rs b/src/packed_string/impls_for_construction.rs index a529191..7f6c0e1 100644 --- a/src/packed_string/impls_for_construction.rs +++ b/src/packed_string/impls_for_construction.rs @@ -36,3 +36,7 @@ impl FromIterator for PackedString { #[cfg(test)] mod tests_for_construction; + +mod impls_for_from_bits; +mod impls_for_from_slice; +mod impls_for_repeat; diff --git a/src/packed_string/impls_for_construction/impls_for_from_bits.rs b/src/packed_string/impls_for_construction/impls_for_from_bits.rs new file mode 100644 index 0000000..f5377a7 --- /dev/null +++ b/src/packed_string/impls_for_construction/impls_for_from_bits.rs @@ -0,0 +1,16 @@ +use super::*; + +impl PackedString { + /// Validates and adopts an already packed bit payload. + pub fn from_bits(_bits: BitString, _char_len: usize) -> Option { + unimplemented!("PackedString::from_bits") + } + + pub fn into_bits(self) -> BitString { + unimplemented!("PackedString::into_bits") + } + + pub fn into_parts(self) -> (BitString, usize) { + unimplemented!("PackedString::into_parts") + } +} diff --git a/src/packed_string/impls_for_construction/impls_for_from_slice.rs b/src/packed_string/impls_for_construction/impls_for_from_slice.rs new file mode 100644 index 0000000..6dc32b0 --- /dev/null +++ b/src/packed_string/impls_for_construction/impls_for_from_slice.rs @@ -0,0 +1,13 @@ +use super::*; + +impl From<&[C]> for PackedString { + fn from(_characters: &[C]) -> Self { + unimplemented!("PackedString::from(&[C])") + } +} + +impl From<[C; N]> for PackedString { + fn from(_characters: [C; N]) -> Self { + unimplemented!("PackedString::from([C; N])") + } +} diff --git a/src/packed_string/impls_for_construction/impls_for_repeat.rs b/src/packed_string/impls_for_construction/impls_for_repeat.rs new file mode 100644 index 0000000..62e59ef --- /dev/null +++ b/src/packed_string/impls_for_construction/impls_for_repeat.rs @@ -0,0 +1,7 @@ +use super::*; + +impl PackedString { + pub fn repeat(_character: C, _char_len: usize) -> Self { + unimplemented!("PackedString::repeat") + } +} diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index ff84237..188641c 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -57,3 +57,11 @@ impl<'a, C: PackedChar> Extend<&'a C> for PackedString { #[cfg(test)] mod tests_for_editing; + +mod impls_for_concat; +mod impls_for_drain; +mod impls_for_insert_remove; +mod impls_for_replace; +mod impls_for_retain; +mod impls_for_reverse; +mod impls_for_slice; diff --git a/src/packed_string/impls_for_editing/impls_for_concat.rs b/src/packed_string/impls_for_editing/impls_for_concat.rs new file mode 100644 index 0000000..39b4ebc --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_concat.rs @@ -0,0 +1,15 @@ +use super::*; + +impl PackedString { + pub fn push_packed_string(&mut self, _other: &Self) { + unimplemented!("PackedString::push_packed_string") + } + + pub fn insert_packed_string(&mut self, _index: usize, _other: &Self) { + unimplemented!("PackedString::insert_packed_string") + } + + pub fn split_off(&mut self, _at: usize) -> Self { + unimplemented!("PackedString::split_off") + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_drain.rs b/src/packed_string/impls_for_editing/impls_for_drain.rs new file mode 100644 index 0000000..775bff7 --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_drain.rs @@ -0,0 +1,13 @@ +use int_interval::UsizeCO; + +use super::*; + +impl PackedString { + pub fn drain_interval(&self, _interval: UsizeCO) -> Self { + unimplemented!("PackedString::drain_interval") + } + + pub fn drain_interval_assign(&mut self, _interval: UsizeCO) { + unimplemented!("PackedString::drain_interval_assign") + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs new file mode 100644 index 0000000..f5c71c8 --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs @@ -0,0 +1,11 @@ +use super::*; + +impl PackedString { + pub fn insert(&mut self, _index: usize, _character: C) { + unimplemented!("PackedString::insert") + } + + pub fn remove(&mut self, _index: usize) -> C { + unimplemented!("PackedString::remove") + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_replace.rs b/src/packed_string/impls_for_editing/impls_for_replace.rs new file mode 100644 index 0000000..53217f2 --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_replace.rs @@ -0,0 +1,21 @@ +use int_interval::UsizeCO; + +use super::*; + +impl PackedString { + pub fn replace_interval(&self, _interval: UsizeCO, _replacement: &Self) -> Self { + unimplemented!("PackedString::replace_interval") + } + + pub fn replace_interval_assign(&mut self, _interval: UsizeCO, _replacement: &Self) { + unimplemented!("PackedString::replace_interval_assign") + } + + pub fn replace(&self, _start: usize, _replacement: &Self) -> Self { + unimplemented!("PackedString::replace") + } + + pub fn replace_assign(&mut self, _start: usize, _replacement: &Self) { + unimplemented!("PackedString::replace_assign") + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_retain.rs b/src/packed_string/impls_for_editing/impls_for_retain.rs new file mode 100644 index 0000000..0fa821f --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_retain.rs @@ -0,0 +1,10 @@ +use super::*; + +impl PackedString { + pub fn retain(&mut self, _predicate: F) + where + F: FnMut(C) -> bool, + { + unimplemented!("PackedString::retain") + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_reverse.rs b/src/packed_string/impls_for_editing/impls_for_reverse.rs new file mode 100644 index 0000000..1ece027 --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_reverse.rs @@ -0,0 +1,11 @@ +use super::*; + +impl PackedString { + pub fn reverse(&self) -> Self { + unimplemented!("PackedString::reverse") + } + + pub fn reverse_assign(&mut self) { + unimplemented!("PackedString::reverse_assign") + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_slice.rs b/src/packed_string/impls_for_editing/impls_for_slice.rs new file mode 100644 index 0000000..d728c3d --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_slice.rs @@ -0,0 +1,17 @@ +use int_interval::UsizeCO; + +use super::*; + +impl PackedString { + pub fn slice(&self, _interval: UsizeCO) -> Self { + unimplemented!("PackedString::slice") + } + + pub fn slice_from(&self, _start: usize) -> Self { + unimplemented!("PackedString::slice_from") + } + + pub fn slice_until(&self, _end: usize) -> Self { + unimplemented!("PackedString::slice_until") + } +} diff --git a/src/packed_string/impls_for_hash.rs b/src/packed_string/impls_for_hash.rs new file mode 100644 index 0000000..a973ec0 --- /dev/null +++ b/src/packed_string/impls_for_hash.rs @@ -0,0 +1,9 @@ +use core::hash::{Hash, Hasher}; + +use super::*; + +impl Hash for PackedString { + fn hash(&self, _state: &mut H) { + unimplemented!("PackedString::hash") + } +} diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index b23d1e1..7c18876 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -11,6 +11,11 @@ impl PackedString { back: self.char_len, } } + + /// Collects the decoded characters into a vector. + pub fn to_vec(&self) -> alloc::vec::Vec { + unimplemented!("PackedString::to_vec") + } } #[derive(Clone)] diff --git a/src/packed_string/impls_for_matching.rs b/src/packed_string/impls_for_matching.rs new file mode 100644 index 0000000..88c34b8 --- /dev/null +++ b/src/packed_string/impls_for_matching.rs @@ -0,0 +1,35 @@ +use super::*; + +impl PackedString { + pub fn matches_at(&self, _index: usize, _pattern: &Self) -> bool { + unimplemented!("PackedString::matches_at") + } + + pub fn starts_with(&self, _prefix: &Self) -> bool { + unimplemented!("PackedString::starts_with") + } + + pub fn ends_with(&self, _suffix: &Self) -> bool { + unimplemented!("PackedString::ends_with") + } + + pub fn contains(&self, _needle: &Self) -> bool { + unimplemented!("PackedString::contains") + } + + pub fn find(&self, _needle: &Self) -> Option { + unimplemented!("PackedString::find") + } + + pub fn rfind(&self, _needle: &Self) -> Option { + unimplemented!("PackedString::rfind") + } + + pub fn strip_prefix(&self, _prefix: &Self) -> Option { + unimplemented!("PackedString::strip_prefix") + } + + pub fn strip_suffix(&self, _suffix: &Self) -> Option { + unimplemented!("PackedString::strip_suffix") + } +} diff --git a/src/packed_string/impls_for_ord.rs b/src/packed_string/impls_for_ord.rs new file mode 100644 index 0000000..de380ad --- /dev/null +++ b/src/packed_string/impls_for_ord.rs @@ -0,0 +1,22 @@ +use core::cmp::Ordering; + +use super::*; + +impl PackedString { + /// Compares characters lexicographically by their packed code values. + pub fn cmp_string(&self, _other: &Self) -> Ordering { + unimplemented!("PackedString::cmp_string") + } +} + +impl PartialOrd for PackedString { + fn partial_cmp(&self, _other: &Self) -> Option { + unimplemented!("PackedString::partial_cmp") + } +} + +impl Ord for PackedString { + fn cmp(&self, _other: &Self) -> Ordering { + unimplemented!("PackedString::cmp") + } +} From 00329f01f3717645bc9378dae0478cd31cf085ae Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 2 Aug 2026 13:26:42 +0000 Subject: [PATCH 03/79] fix: make set_chunk overwrite existing bits --- .../impls_for_editing/impls_for_set.rs | 24 +++++++------------ .../impls_for_set/tests_for_set_chunk.rs | 22 +++++++++++++---- src/packed_string/funcs_for_code.rs | 4 +--- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/bit_string/impls_for_editing/impls_for_set.rs b/src/bit_string/impls_for_editing/impls_for_set.rs index 883c5fc..5c4df00 100644 --- a/src/bit_string/impls_for_editing/impls_for_set.rs +++ b/src/bit_string/impls_for_editing/impls_for_set.rs @@ -1,4 +1,3 @@ -use crate::funcs_for_bits::*; use crate::traits::*; use super::*; @@ -17,27 +16,22 @@ impl BitString { Some(old) } - /// Writes `len` bits of `value` starting at `bit_start`, OR-ing them - /// with the existing bits. Bits beyond `self.len()` are ignored. + /// Overwrites `len` bits starting at `bit_start` with the low bits of + /// `value`. Bits beyond `self.len()` are ignored. /// /// Only the low `len` bits of `value` are used; higher bits are /// masked out. #[inline] pub fn set_chunk(&mut self, bit_start: usize, value: u64, len: usize) { - let value = value & low_mask(len); - let word = bit_start / WORD_BITS; - let shift = bit_start % WORD_BITS; - - if let Some(w) = self.words.get_mut(word) { - *w |= value << shift; - } - - if shift != 0 { - if let Some(w) = self.words.get_mut(word + 1) { - *w |= value >> (WORD_BITS - shift); - } + let len = len + .min(WORD_BITS) + .min(self.bit_len.saturating_sub(bit_start)); + if len == 0 { + return; } + self.words.clear_bits_at(bit_start, len); + self.words.write_word_at::(bit_start, value, len); self.words.mask_unused_bits(self.bit_len); } } diff --git a/src/bit_string/impls_for_editing/impls_for_set/tests_for_set_chunk.rs b/src/bit_string/impls_for_editing/impls_for_set/tests_for_set_chunk.rs index b9bc9fd..e86d535 100644 --- a/src/bit_string/impls_for_editing/impls_for_set/tests_for_set_chunk.rs +++ b/src/bit_string/impls_for_editing/impls_for_set/tests_for_set_chunk.rs @@ -20,13 +20,25 @@ fn sets_unaligned_chunk() { } #[test] -fn set_chunk_ors_with_existing() { - let mut bits = BitString::zeros(128); - bits.set_chunk(0, 0b11, 2); - bits.set_chunk(1, 0b10, 2); +fn set_chunk_overwrites_existing_bits() { + let mut bits = BitString::ones(4); + bits.set_chunk(1, 0b01, 2); assert_eq!(bits.get(0), Some(true)); assert_eq!(bits.get(1), Some(true)); - assert_eq!(bits.get(2), Some(true)); + assert_eq!(bits.get(2), Some(false)); + assert_eq!(bits.get(3), Some(true)); +} + +#[test] +fn overwrite_crosses_a_word_boundary() { + let mut bits = BitString::ones(130); + bits.set_chunk(62, 0b001, 3); + + assert_eq!(bits.get(61), Some(true)); + assert_eq!(bits.get(62), Some(true)); + assert_eq!(bits.get(63), Some(false)); + assert_eq!(bits.get(64), Some(false)); + assert_eq!(bits.get(65), Some(true)); } #[test] diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index f3c0627..f4fbc20 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -34,9 +34,7 @@ pub(super) fn checked_code(character: C) -> u8 { #[inline] pub(super) fn write_code(bits: &mut BitString, position: usize, width: u8, code: u8) { let start = position * usize::from(width); - for shift in 0..width { - let _ = bits.set(start + usize::from(shift), (code >> shift) & 1 != 0); - } + bits.set_chunk(start, u64::from(code), usize::from(width)); } #[cfg(test)] From 85531ecd57e225ef35f19636fcf60e1695b45641 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 2 Aug 2026 13:42:55 +0000 Subject: [PATCH 04/79] refactor: encode packed width as const generic --- src/packed_str.rs | 7 ++-- src/packed_str/impls_for_access.rs | 7 ++-- src/packed_str/impls_for_conversion.rs | 14 +++++--- src/packed_str/impls_for_eq.rs | 7 ++-- src/packed_str/impls_for_iter.rs | 35 +++++++++++++------ src/packed_str/impls_for_matching.rs | 5 ++- src/packed_str/impls_for_ord.rs | 10 ++++-- src/packed_str/impls_for_slice.rs | 5 ++- src/packed_string.rs | 10 ++++-- src/packed_string/funcs_for_code.rs | 29 +++++++-------- .../funcs_for_code/tests_for_code.rs | 6 ++-- src/packed_string/impls_for_access.rs | 13 ++++--- .../impls_for_access/tests_for_access.rs | 7 ++-- src/packed_string/impls_for_construction.rs | 19 +++++++--- .../impls_for_from_bits.rs | 5 ++- .../impls_for_from_slice.rs | 10 ++++-- .../impls_for_repeat.rs | 5 ++- .../tests_for_construction.rs | 7 ++-- src/packed_string/impls_for_editing.rs | 25 ++++++++----- .../impls_for_editing/impls_for_concat.rs | 5 ++- .../impls_for_editing/impls_for_drain.rs | 5 ++- .../impls_for_insert_remove.rs | 5 ++- .../impls_for_editing/impls_for_replace.rs | 5 ++- .../impls_for_editing/impls_for_retain.rs | 5 ++- .../impls_for_editing/impls_for_reverse.rs | 5 ++- .../impls_for_editing/impls_for_slice.rs | 5 ++- .../impls_for_editing/tests_for_editing.rs | 5 ++- src/packed_string/impls_for_eq.rs | 7 ++-- .../impls_for_eq/tests_for_eq.rs | 9 +++-- src/packed_string/impls_for_fmt.rs | 10 ++++-- .../impls_for_fmt/tests_for_fmt.rs | 7 ++-- src/packed_string/impls_for_hash.rs | 5 ++- src/packed_string/impls_for_iter.rs | 35 +++++++++++++------ .../impls_for_iter/tests_for_iter.rs | 5 ++- src/packed_string/impls_for_matching.rs | 5 ++- src/packed_string/impls_for_ord.rs | 15 ++++++-- src/packed_string/tests_for_support.rs | 11 +++--- src/traits/packed_char.rs | 12 +++---- 38 files changed, 257 insertions(+), 130 deletions(-) diff --git a/src/packed_str.rs b/src/packed_str.rs index ee03ff9..5efc08a 100644 --- a/src/packed_str.rs +++ b/src/packed_str.rs @@ -7,8 +7,11 @@ use crate::{PackedString, traits::PackedChar}; /// A character-aligned borrowed view into a [`PackedString`]. #[derive(Clone, Copy)] #[allow(dead_code)] -pub struct PackedStr<'ps, C: PackedChar> { - source: &'ps PackedString, +pub struct PackedStr<'ps, C, const BITS: u8> +where + C: PackedChar, +{ + source: &'ps PackedString, char_start: usize, char_len: usize, marker: PhantomData C>, diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs index 037d545..10a0d60 100644 --- a/src/packed_str/impls_for_access.rs +++ b/src/packed_str/impls_for_access.rs @@ -1,7 +1,10 @@ use super::*; -impl<'ps, C: PackedChar> PackedStr<'ps, C> { - pub fn source(&self) -> &'ps PackedString { +impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ + pub fn source(&self) -> &'ps PackedString { unimplemented!("PackedStr::source") } diff --git a/src/packed_str/impls_for_conversion.rs b/src/packed_str/impls_for_conversion.rs index b9d61de..b6651ba 100644 --- a/src/packed_str/impls_for_conversion.rs +++ b/src/packed_str/impls_for_conversion.rs @@ -1,13 +1,19 @@ use super::*; -impl<'ps, C: PackedChar> PackedStr<'ps, C> { - pub fn to_packed_string(&self) -> PackedString { +impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ + pub fn to_packed_string(&self) -> PackedString { unimplemented!("PackedStr::to_packed_string") } } -impl PackedString { - pub fn as_packed_str(&self) -> PackedStr<'_, C> { +impl PackedString +where + C: PackedChar, +{ + pub fn as_packed_str(&self) -> PackedStr<'_, C, BITS> { unimplemented!("PackedString::as_packed_str") } } diff --git a/src/packed_str/impls_for_eq.rs b/src/packed_str/impls_for_eq.rs index 9ee3182..d90b37a 100644 --- a/src/packed_str/impls_for_eq.rs +++ b/src/packed_str/impls_for_eq.rs @@ -1,9 +1,12 @@ use super::*; -impl PartialEq for PackedStr<'_, C> { +impl PartialEq for PackedStr<'_, C, BITS> +where + C: PackedChar, +{ fn eq(&self, _other: &Self) -> bool { unimplemented!("PackedStr::eq") } } -impl Eq for PackedStr<'_, C> {} +impl Eq for PackedStr<'_, C, BITS> where C: PackedChar {} diff --git a/src/packed_str/impls_for_iter.rs b/src/packed_str/impls_for_iter.rs index 08c4d34..4183586 100644 --- a/src/packed_str/impls_for_iter.rs +++ b/src/packed_str/impls_for_iter.rs @@ -2,20 +2,29 @@ use core::iter::FusedIterator; use super::*; -impl<'ps, C: PackedChar> PackedStr<'ps, C> { - pub fn iter(&self) -> Iter<'ps, C> { +impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ + pub fn iter(&self) -> Iter<'ps, C, BITS> { unimplemented!("PackedStr::iter") } } #[allow(dead_code)] -pub struct Iter<'ps, C: PackedChar> { - view: PackedStr<'ps, C>, +pub struct Iter<'ps, C, const BITS: u8> +where + C: PackedChar, +{ + view: PackedStr<'ps, C, BITS>, front: usize, back: usize, } -impl Iterator for Iter<'_, C> { +impl Iterator for Iter<'_, C, BITS> +where + C: PackedChar, +{ type Item = C; fn next(&mut self) -> Option { @@ -27,18 +36,24 @@ impl Iterator for Iter<'_, C> { } } -impl DoubleEndedIterator for Iter<'_, C> { +impl DoubleEndedIterator for Iter<'_, C, BITS> +where + C: PackedChar, +{ fn next_back(&mut self) -> Option { unimplemented!("PackedStr::Iter::next_back") } } -impl ExactSizeIterator for Iter<'_, C> {} -impl FusedIterator for Iter<'_, C> {} +impl ExactSizeIterator for Iter<'_, C, BITS> where C: PackedChar {} +impl FusedIterator for Iter<'_, C, BITS> where C: PackedChar {} -impl<'ps, C: PackedChar> IntoIterator for &'ps PackedStr<'ps, C> { +impl<'ps, C, const BITS: u8> IntoIterator for &'ps PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ type Item = C; - type IntoIter = Iter<'ps, C>; + type IntoIter = Iter<'ps, C, BITS>; fn into_iter(self) -> Self::IntoIter { unimplemented!("PackedStr::into_iter") diff --git a/src/packed_str/impls_for_matching.rs b/src/packed_str/impls_for_matching.rs index 5e488d5..eef34ef 100644 --- a/src/packed_str/impls_for_matching.rs +++ b/src/packed_str/impls_for_matching.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedStr<'_, C> { +impl PackedStr<'_, C, BITS> +where + C: PackedChar, +{ pub fn matches_at(&self, _index: usize, _pattern: Self) -> bool { unimplemented!("PackedStr::matches_at") } diff --git a/src/packed_str/impls_for_ord.rs b/src/packed_str/impls_for_ord.rs index a9327e3..a1d491f 100644 --- a/src/packed_str/impls_for_ord.rs +++ b/src/packed_str/impls_for_ord.rs @@ -2,13 +2,19 @@ use core::cmp::Ordering; use super::*; -impl PartialOrd for PackedStr<'_, C> { +impl PartialOrd for PackedStr<'_, C, BITS> +where + C: PackedChar, +{ fn partial_cmp(&self, _other: &Self) -> Option { unimplemented!("PackedStr::partial_cmp") } } -impl Ord for PackedStr<'_, C> { +impl Ord for PackedStr<'_, C, BITS> +where + C: PackedChar, +{ fn cmp(&self, _other: &Self) -> Ordering { unimplemented!("PackedStr::cmp") } diff --git a/src/packed_str/impls_for_slice.rs b/src/packed_str/impls_for_slice.rs index 8bd5227..2c623d8 100644 --- a/src/packed_str/impls_for_slice.rs +++ b/src/packed_str/impls_for_slice.rs @@ -2,7 +2,10 @@ use int_interval::UsizeCO; use super::*; -impl<'ps, C: PackedChar> PackedStr<'ps, C> { +impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ pub fn slice(&self, _interval: UsizeCO) -> Self { unimplemented!("PackedStr::slice") } diff --git a/src/packed_string.rs b/src/packed_string.rs index 8c3fbd6..3c9eb92 100644 --- a/src/packed_string.rs +++ b/src/packed_string.rs @@ -4,12 +4,16 @@ use core::marker::PhantomData; use crate::{BitString, traits::PackedChar}; -/// An owned sequence of fixed-width [`PackedChar`] values. +/// An owned sequence of [`PackedChar`] values stored in exactly `BITS` bits +/// per character. /// /// The character type is the alphabet and its enum discriminant is the stored /// value, so no alphabet allocation or lookup table is required. #[derive(Clone)] -pub struct PackedString { +pub struct PackedString +where + C: PackedChar, +{ bits: BitString, char_len: usize, marker: PhantomData C>, @@ -26,7 +30,7 @@ mod impls_for_iter; mod impls_for_matching; mod impls_for_ord; -use funcs_for_code::{assert_valid_width, checked_code, code_mask, write_code}; +use funcs_for_code::{checked_code, code_mask, write_code}; #[cfg(test)] mod tests_for_support; diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index f4fbc20..1d3e330 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -1,28 +1,25 @@ use super::*; #[inline] -pub(super) fn assert_valid_width() { - assert!(C::BITS <= 8, "PackedChar::BITS must not exceed 8"); -} - -#[inline] -pub(super) fn code_mask() -> u8 { - match C::BITS { +pub(super) const fn code_mask() -> u8 { + match BITS { 0 => 0, - 1..=7 => (1u8 << C::BITS) - 1, + 1..=7 => (1u8 << BITS) - 1, 8 => u8::MAX, - _ => unreachable!("PackedChar::BITS must not exceed 8"), + _ => panic!("packed character width must not exceed 8"), } } #[inline] -pub(super) fn checked_code(character: C) -> u8 { - assert_valid_width::(); +pub(super) fn checked_code(character: C) -> u8 +where + C: PackedChar, +{ let code = character.code(); assert_eq!( - code & !code_mask::(), + code & !code_mask::(), 0, - "PackedChar::code does not fit in PackedChar::BITS" + "PackedChar::code does not fit in BITS" ); assert!( C::from_code(code) == Some(character), @@ -32,9 +29,9 @@ pub(super) fn checked_code(character: C) -> u8 { } #[inline] -pub(super) fn write_code(bits: &mut BitString, position: usize, width: u8, code: u8) { - let start = position * usize::from(width); - bits.set_chunk(start, u64::from(code), usize::from(width)); +pub(super) fn write_code(bits: &mut BitString, position: usize, code: u8) { + let start = position * usize::from(BITS); + bits.set_chunk(start, u64::from(code), usize::from(BITS)); } #[cfg(test)] diff --git a/src/packed_string/funcs_for_code/tests_for_code.rs b/src/packed_string/funcs_for_code/tests_for_code.rs index bbeb65a..a69100d 100644 --- a/src/packed_string/funcs_for_code/tests_for_code.rs +++ b/src/packed_string/funcs_for_code/tests_for_code.rs @@ -3,9 +3,7 @@ use super::*; #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct Broken; -impl PackedChar for Broken { - const BITS: u8 = 1; - +impl PackedChar<1> for Broken { fn code(self) -> u8 { 2 } @@ -18,5 +16,5 @@ impl PackedChar for Broken { #[test] #[should_panic(expected = "does not fit")] fn rejects_a_code_that_does_not_fit() { - let _ = checked_code(Broken); + let _ = checked_code::(Broken); } diff --git a/src/packed_string/impls_for_access.rs b/src/packed_string/impls_for_access.rs index eed01bd..d855edc 100644 --- a/src/packed_string/impls_for_access.rs +++ b/src/packed_string/impls_for_access.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ /// Number of packed characters, not number of bits. #[inline] pub fn char_len(&self) -> usize { @@ -14,7 +17,7 @@ impl PackedString { #[inline] pub const fn bits_per_char(&self) -> usize { - C::BITS as usize + BITS as usize } #[inline] @@ -45,11 +48,11 @@ impl PackedString { #[inline] fn code_at(&self, index: usize) -> u8 { - if C::BITS == 0 { + if BITS == 0 { return 0; } - let start = index * usize::from(C::BITS); - (self.bits.get_chunk(start) & u64::from(code_mask::())) as u8 + let start = index * usize::from(BITS); + (self.bits.get_chunk(start) & u64::from(code_mask::())) as u8 } } diff --git a/src/packed_string/impls_for_access/tests_for_access.rs b/src/packed_string/impls_for_access/tests_for_access.rs index aebf078..9897e56 100644 --- a/src/packed_string/impls_for_access/tests_for_access.rs +++ b/src/packed_string/impls_for_access/tests_for_access.rs @@ -1,9 +1,8 @@ -use super::*; -use crate::packed_string::tests_for_support::{Letter, Only}; +use crate::packed_string::tests_for_support::{Letter, LetterString, Only, OnlyString}; #[test] fn get_first_and_last_decode_values() { - let value = PackedString::from_chars([Letter::A, Letter::B, Letter::C]); + let value = LetterString::from_chars([Letter::A, Letter::B, Letter::C]); assert_eq!(value.first(), Some(Letter::A)); assert_eq!(value.last(), Some(Letter::C)); assert_eq!(value.get(3), None); @@ -11,7 +10,7 @@ fn get_first_and_last_decode_values() { #[test] fn one_value_type_needs_no_payload_bits() { - let value = PackedString::from_chars([Only; 100]); + let value = OnlyString::from_chars([Only; 100]); assert_eq!(value.char_len(), 100); assert_eq!(value.bits().bit_len(), 0); assert_eq!(value.get(99), Some(Only)); diff --git a/src/packed_string/impls_for_construction.rs b/src/packed_string/impls_for_construction.rs index 7f6c0e1..c81ba1f 100644 --- a/src/packed_string/impls_for_construction.rs +++ b/src/packed_string/impls_for_construction.rs @@ -1,9 +1,14 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ + const VALID_WIDTH: () = assert!(BITS <= 8, "packed character width must not exceed 8"); + #[inline] pub fn new() -> Self { - assert_valid_width::(); + let () = Self::VALID_WIDTH; Self { bits: BitString::new(), char_len: 0, @@ -22,13 +27,19 @@ impl PackedString { } } -impl Default for PackedString { +impl Default for PackedString +where + C: PackedChar, +{ fn default() -> Self { Self::new() } } -impl FromIterator for PackedString { +impl FromIterator for PackedString +where + C: PackedChar, +{ fn from_iter>(iter: I) -> Self { Self::from_chars(iter) } diff --git a/src/packed_string/impls_for_construction/impls_for_from_bits.rs b/src/packed_string/impls_for_construction/impls_for_from_bits.rs index f5377a7..9c3aae1 100644 --- a/src/packed_string/impls_for_construction/impls_for_from_bits.rs +++ b/src/packed_string/impls_for_construction/impls_for_from_bits.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ /// Validates and adopts an already packed bit payload. pub fn from_bits(_bits: BitString, _char_len: usize) -> Option { unimplemented!("PackedString::from_bits") diff --git a/src/packed_string/impls_for_construction/impls_for_from_slice.rs b/src/packed_string/impls_for_construction/impls_for_from_slice.rs index 6dc32b0..91e3140 100644 --- a/src/packed_string/impls_for_construction/impls_for_from_slice.rs +++ b/src/packed_string/impls_for_construction/impls_for_from_slice.rs @@ -1,12 +1,18 @@ use super::*; -impl From<&[C]> for PackedString { +impl From<&[C]> for PackedString +where + C: PackedChar, +{ fn from(_characters: &[C]) -> Self { unimplemented!("PackedString::from(&[C])") } } -impl From<[C; N]> for PackedString { +impl From<[C; N]> for PackedString +where + C: PackedChar, +{ fn from(_characters: [C; N]) -> Self { unimplemented!("PackedString::from([C; N])") } diff --git a/src/packed_string/impls_for_construction/impls_for_repeat.rs b/src/packed_string/impls_for_construction/impls_for_repeat.rs index 62e59ef..fcbc7a2 100644 --- a/src/packed_string/impls_for_construction/impls_for_repeat.rs +++ b/src/packed_string/impls_for_construction/impls_for_repeat.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn repeat(_character: C, _char_len: usize) -> Self { unimplemented!("PackedString::repeat") } diff --git a/src/packed_string/impls_for_construction/tests_for_construction.rs b/src/packed_string/impls_for_construction/tests_for_construction.rs index 3f94ed5..cbe7a9d 100644 --- a/src/packed_string/impls_for_construction/tests_for_construction.rs +++ b/src/packed_string/impls_for_construction/tests_for_construction.rs @@ -1,9 +1,8 @@ -use super::*; -use crate::packed_string::tests_for_support::Letter; +use crate::packed_string::tests_for_support::{Letter, LetterString}; #[test] fn enum_discriminants_are_stored_directly() { - let value = PackedString::from_chars([Letter::A, Letter::B, Letter::C, Letter::D]); + let value = LetterString::from_chars([Letter::A, Letter::B, Letter::C, Letter::D]); assert_eq!(value.bits_per_char(), 2); assert_eq!(value.bits().bit_len(), 8); assert_eq!(value.bits().get_chunk(0), 0b11_10_01_00); @@ -11,7 +10,7 @@ fn enum_discriminants_are_stored_directly() { #[test] fn collect_constructs_a_packed_string() { - let value: PackedString = [Letter::A, Letter::C].into_iter().collect(); + let value: LetterString = [Letter::A, Letter::C].into_iter().collect(); assert_eq!(value.char_len(), 2); assert_eq!(value.get(0), Some(Letter::A)); } diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 188641c..6029566 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -1,14 +1,17 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ /// Appends the character's code directly to the payload. pub fn push(&mut self, character: C) { - let code = checked_code(character); + let code = checked_code::(character); let new_len = self .char_len .checked_add(1) .expect("packed string length overflow"); - for shift in 0..C::BITS { + for shift in 0..BITS { self.bits.push((code >> shift) & 1 != 0); } self.char_len = new_len; @@ -17,13 +20,13 @@ impl PackedString { pub fn pop(&mut self) -> Option { let character = self.last()?; self.char_len -= 1; - self.bits.truncate(self.char_len * usize::from(C::BITS)); + self.bits.truncate(self.char_len * usize::from(BITS)); Some(character) } pub fn set(&mut self, index: usize, character: C) -> Option { let previous = self.get(index)?; - write_code(&mut self.bits, index, C::BITS, checked_code(character)); + write_code::(&mut self.bits, index, checked_code::(character)); Some(previous) } @@ -32,7 +35,7 @@ impl PackedString { return; } self.char_len = new_len; - self.bits.truncate(new_len * usize::from(C::BITS)); + self.bits.truncate(new_len * usize::from(BITS)); } pub fn clear(&mut self) { @@ -41,7 +44,10 @@ impl PackedString { } } -impl Extend for PackedString { +impl Extend for PackedString +where + C: PackedChar, +{ fn extend>(&mut self, iter: I) { for character in iter { self.push(character); @@ -49,7 +55,10 @@ impl Extend for PackedString { } } -impl<'a, C: PackedChar> Extend<&'a C> for PackedString { +impl<'a, C, const BITS: u8> Extend<&'a C> for PackedString +where + C: PackedChar, +{ fn extend>(&mut self, iter: I) { self.extend(iter.into_iter().copied()); } diff --git a/src/packed_string/impls_for_editing/impls_for_concat.rs b/src/packed_string/impls_for_editing/impls_for_concat.rs index 39b4ebc..60f74b2 100644 --- a/src/packed_string/impls_for_editing/impls_for_concat.rs +++ b/src/packed_string/impls_for_editing/impls_for_concat.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn push_packed_string(&mut self, _other: &Self) { unimplemented!("PackedString::push_packed_string") } diff --git a/src/packed_string/impls_for_editing/impls_for_drain.rs b/src/packed_string/impls_for_editing/impls_for_drain.rs index 775bff7..c0cfedf 100644 --- a/src/packed_string/impls_for_editing/impls_for_drain.rs +++ b/src/packed_string/impls_for_editing/impls_for_drain.rs @@ -2,7 +2,10 @@ use int_interval::UsizeCO; use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn drain_interval(&self, _interval: UsizeCO) -> Self { unimplemented!("PackedString::drain_interval") } diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs index f5c71c8..36156ab 100644 --- a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn insert(&mut self, _index: usize, _character: C) { unimplemented!("PackedString::insert") } diff --git a/src/packed_string/impls_for_editing/impls_for_replace.rs b/src/packed_string/impls_for_editing/impls_for_replace.rs index 53217f2..ca90eec 100644 --- a/src/packed_string/impls_for_editing/impls_for_replace.rs +++ b/src/packed_string/impls_for_editing/impls_for_replace.rs @@ -2,7 +2,10 @@ use int_interval::UsizeCO; use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn replace_interval(&self, _interval: UsizeCO, _replacement: &Self) -> Self { unimplemented!("PackedString::replace_interval") } diff --git a/src/packed_string/impls_for_editing/impls_for_retain.rs b/src/packed_string/impls_for_editing/impls_for_retain.rs index 0fa821f..2379321 100644 --- a/src/packed_string/impls_for_editing/impls_for_retain.rs +++ b/src/packed_string/impls_for_editing/impls_for_retain.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn retain(&mut self, _predicate: F) where F: FnMut(C) -> bool, diff --git a/src/packed_string/impls_for_editing/impls_for_reverse.rs b/src/packed_string/impls_for_editing/impls_for_reverse.rs index 1ece027..b85a9b7 100644 --- a/src/packed_string/impls_for_editing/impls_for_reverse.rs +++ b/src/packed_string/impls_for_editing/impls_for_reverse.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn reverse(&self) -> Self { unimplemented!("PackedString::reverse") } diff --git a/src/packed_string/impls_for_editing/impls_for_slice.rs b/src/packed_string/impls_for_editing/impls_for_slice.rs index d728c3d..d73eab6 100644 --- a/src/packed_string/impls_for_editing/impls_for_slice.rs +++ b/src/packed_string/impls_for_editing/impls_for_slice.rs @@ -2,7 +2,10 @@ use int_interval::UsizeCO; use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn slice(&self, _interval: UsizeCO) -> Self { unimplemented!("PackedString::slice") } diff --git a/src/packed_string/impls_for_editing/tests_for_editing.rs b/src/packed_string/impls_for_editing/tests_for_editing.rs index b773e6d..06fad30 100644 --- a/src/packed_string/impls_for_editing/tests_for_editing.rs +++ b/src/packed_string/impls_for_editing/tests_for_editing.rs @@ -1,9 +1,8 @@ -use super::*; -use crate::packed_string::tests_for_support::Letter; +use crate::packed_string::tests_for_support::{Letter, LetterString}; #[test] fn push_set_pop_and_truncate_preserve_values() { - let mut value = PackedString::from_chars([Letter::A, Letter::B, Letter::C]); + let mut value = LetterString::from_chars([Letter::A, Letter::B, Letter::C]); value.push(Letter::D); assert_eq!(value.set(1, Letter::C), Some(Letter::B)); assert_eq!(value.pop(), Some(Letter::D)); diff --git a/src/packed_string/impls_for_eq.rs b/src/packed_string/impls_for_eq.rs index a6596ca..6340e13 100644 --- a/src/packed_string/impls_for_eq.rs +++ b/src/packed_string/impls_for_eq.rs @@ -1,12 +1,15 @@ use super::*; -impl PartialEq for PackedString { +impl PartialEq for PackedString +where + C: PackedChar, +{ fn eq(&self, other: &Self) -> bool { self.char_len == other.char_len && self.bits == other.bits } } -impl Eq for PackedString {} +impl Eq for PackedString where C: PackedChar {} #[cfg(test)] mod tests_for_eq; diff --git a/src/packed_string/impls_for_eq/tests_for_eq.rs b/src/packed_string/impls_for_eq/tests_for_eq.rs index ea6a0b9..461176e 100644 --- a/src/packed_string/impls_for_eq/tests_for_eq.rs +++ b/src/packed_string/impls_for_eq/tests_for_eq.rs @@ -1,11 +1,10 @@ -use super::*; -use crate::packed_string::tests_for_support::Letter; +use crate::packed_string::tests_for_support::{Letter, LetterString}; #[test] fn equality_uses_length_and_packed_values() { - let left = PackedString::from_chars([Letter::A, Letter::B]); - let same = PackedString::from_chars([Letter::A, Letter::B]); - let different = PackedString::from_chars([Letter::A, Letter::C]); + let left = LetterString::from_chars([Letter::A, Letter::B]); + let same = LetterString::from_chars([Letter::A, Letter::B]); + let different = LetterString::from_chars([Letter::A, Letter::C]); assert_eq!(left, same); assert_ne!(left, different); } diff --git a/src/packed_string/impls_for_fmt.rs b/src/packed_string/impls_for_fmt.rs index f2854de..7b80e89 100644 --- a/src/packed_string/impls_for_fmt.rs +++ b/src/packed_string/impls_for_fmt.rs @@ -2,13 +2,19 @@ use core::fmt; use super::*; -impl fmt::Debug for PackedString { +impl fmt::Debug for PackedString +where + C: PackedChar + fmt::Debug, +{ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(self.iter()).finish() } } -impl fmt::Display for PackedString { +impl fmt::Display for PackedString +where + C: PackedChar + fmt::Display, +{ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for character in self.iter() { fmt::Display::fmt(&character, f)?; diff --git a/src/packed_string/impls_for_fmt/tests_for_fmt.rs b/src/packed_string/impls_for_fmt/tests_for_fmt.rs index 14b428f..a9626f8 100644 --- a/src/packed_string/impls_for_fmt/tests_for_fmt.rs +++ b/src/packed_string/impls_for_fmt/tests_for_fmt.rs @@ -1,15 +1,14 @@ -use super::*; -use crate::packed_string::tests_for_support::Letter; +use crate::packed_string::tests_for_support::{Letter, LetterString}; use alloc::string::ToString; #[test] fn display_uses_the_character_display_implementation() { - let value = PackedString::from_chars([Letter::A, Letter::B, Letter::C, Letter::D]); + let value = LetterString::from_chars([Letter::A, Letter::B, Letter::C, Letter::D]); assert_eq!(value.to_string(), "ABCD"); } #[test] fn debug_lists_the_enum_values() { - let value = PackedString::from_chars([Letter::A, Letter::B]); + let value = LetterString::from_chars([Letter::A, Letter::B]); assert_eq!(alloc::format!("{value:?}"), "[A, B]"); } diff --git a/src/packed_string/impls_for_hash.rs b/src/packed_string/impls_for_hash.rs index a973ec0..6027697 100644 --- a/src/packed_string/impls_for_hash.rs +++ b/src/packed_string/impls_for_hash.rs @@ -2,7 +2,10 @@ use core::hash::{Hash, Hasher}; use super::*; -impl Hash for PackedString { +impl Hash for PackedString +where + C: PackedChar, +{ fn hash(&self, _state: &mut H) { unimplemented!("PackedString::hash") } diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index 7c18876..317ff7d 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -2,9 +2,12 @@ use core::iter::FusedIterator; use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ #[inline] - pub fn iter(&self) -> Iter<'_, C> { + pub fn iter(&self) -> Iter<'_, C, BITS> { Iter { string: self, front: 0, @@ -19,13 +22,19 @@ impl PackedString { } #[derive(Clone)] -pub struct Iter<'a, C: PackedChar> { - string: &'a PackedString, +pub struct Iter<'a, C, const BITS: u8> +where + C: PackedChar, +{ + string: &'a PackedString, front: usize, back: usize, } -impl Iterator for Iter<'_, C> { +impl Iterator for Iter<'_, C, BITS> +where + C: PackedChar, +{ type Item = C; fn next(&mut self) -> Option { @@ -43,7 +52,10 @@ impl Iterator for Iter<'_, C> { } } -impl DoubleEndedIterator for Iter<'_, C> { +impl DoubleEndedIterator for Iter<'_, C, BITS> +where + C: PackedChar, +{ fn next_back(&mut self) -> Option { if self.front == self.back { return None; @@ -53,12 +65,15 @@ impl DoubleEndedIterator for Iter<'_, C> { } } -impl ExactSizeIterator for Iter<'_, C> {} -impl FusedIterator for Iter<'_, C> {} +impl ExactSizeIterator for Iter<'_, C, BITS> where C: PackedChar {} +impl FusedIterator for Iter<'_, C, BITS> where C: PackedChar {} -impl<'a, C: PackedChar> IntoIterator for &'a PackedString { +impl<'a, C, const BITS: u8> IntoIterator for &'a PackedString +where + C: PackedChar, +{ type Item = C; - type IntoIter = Iter<'a, C>; + type IntoIter = Iter<'a, C, BITS>; fn into_iter(self) -> Self::IntoIter { self.iter() diff --git a/src/packed_string/impls_for_iter/tests_for_iter.rs b/src/packed_string/impls_for_iter/tests_for_iter.rs index e8aebb9..22f755b 100644 --- a/src/packed_string/impls_for_iter/tests_for_iter.rs +++ b/src/packed_string/impls_for_iter/tests_for_iter.rs @@ -1,9 +1,8 @@ -use super::*; -use crate::packed_string::tests_for_support::Letter; +use crate::packed_string::tests_for_support::{Letter, LetterString}; #[test] fn iter_is_double_ended_and_exact_size() { - let value = PackedString::from_chars([Letter::A, Letter::B, Letter::C]); + let value = LetterString::from_chars([Letter::A, Letter::B, Letter::C]); let mut iter = value.iter(); assert_eq!(iter.len(), 3); assert_eq!(iter.next(), Some(Letter::A)); diff --git a/src/packed_string/impls_for_matching.rs b/src/packed_string/impls_for_matching.rs index 88c34b8..afe461c 100644 --- a/src/packed_string/impls_for_matching.rs +++ b/src/packed_string/impls_for_matching.rs @@ -1,6 +1,9 @@ use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ pub fn matches_at(&self, _index: usize, _pattern: &Self) -> bool { unimplemented!("PackedString::matches_at") } diff --git a/src/packed_string/impls_for_ord.rs b/src/packed_string/impls_for_ord.rs index de380ad..18880e6 100644 --- a/src/packed_string/impls_for_ord.rs +++ b/src/packed_string/impls_for_ord.rs @@ -2,20 +2,29 @@ use core::cmp::Ordering; use super::*; -impl PackedString { +impl PackedString +where + C: PackedChar, +{ /// Compares characters lexicographically by their packed code values. pub fn cmp_string(&self, _other: &Self) -> Ordering { unimplemented!("PackedString::cmp_string") } } -impl PartialOrd for PackedString { +impl PartialOrd for PackedString +where + C: PackedChar, +{ fn partial_cmp(&self, _other: &Self) -> Option { unimplemented!("PackedString::partial_cmp") } } -impl Ord for PackedString { +impl Ord for PackedString +where + C: PackedChar, +{ fn cmp(&self, _other: &Self) -> Ordering { unimplemented!("PackedString::cmp") } diff --git a/src/packed_string/tests_for_support.rs b/src/packed_string/tests_for_support.rs index df0af10..a01a1fb 100644 --- a/src/packed_string/tests_for_support.rs +++ b/src/packed_string/tests_for_support.rs @@ -11,9 +11,7 @@ pub(super) enum Letter { D = 0b11, } -impl PackedChar for Letter { - const BITS: u8 = 2; - +impl PackedChar<2> for Letter { fn code(self) -> u8 { self as u8 } @@ -43,9 +41,7 @@ impl fmt::Display for Letter { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct Only; -impl PackedChar for Only { - const BITS: u8 = 0; - +impl PackedChar<0> for Only { fn code(self) -> u8 { 0 } @@ -54,3 +50,6 @@ impl PackedChar for Only { (code == 0).then_some(Self) } } + +pub(super) type LetterString = super::PackedString; +pub(super) type OnlyString = super::PackedString; diff --git a/src/traits/packed_char.rs b/src/traits/packed_char.rs index e8bc09e..00fb029 100644 --- a/src/traits/packed_char.rs +++ b/src/traits/packed_char.rs @@ -3,17 +3,15 @@ /// This trait is intended for fieldless `#[repr(u8)]` enums. Implementations /// must satisfy all of the following invariants: /// -/// - [`BITS`](Self::BITS) does not exceed eight; +/// - `BITS` does not exceed eight; /// - [`code`](Self::code) fits in `BITS` bits; /// - `from_code(code(value)) == Some(value)`; /// - each valid code identifies at most one value. /// -/// [`PackedString`](crate::PackedString) checks the first three invariants -/// whenever it writes a value. No `transmute` is used. -pub trait PackedChar: Copy + Eq { - /// Number of bits occupied by one character. - const BITS: u8; - +/// [`PackedString`](crate::PackedString) validates `BITS` when it is +/// constructed, and checks the `code`/`from_code` agreement whenever it writes +/// a value. No `transmute` is used. +pub trait PackedChar: Copy + Eq { /// Returns the character's packed value. fn code(self) -> u8; From c15fa1c254ea5b2bbabfe5f008d41f91e8297b89 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 2 Aug 2026 14:15:21 +0000 Subject: [PATCH 05/79] feat: derive packed character encodings --- Cargo.lock | 11 ++ Cargo.toml | 3 + derive/Cargo.toml | 16 ++ derive/src/funcs_for_expand.rs | 144 ++++++++++++++++++ .../src/funcs_for_expand/tests_for_expand.rs | 103 +++++++++++++ derive/src/lib.rs | 19 +++ src/lib.rs | 1 + src/packed_string.rs | 2 +- src/packed_string/funcs_for_code.rs | 21 --- .../funcs_for_code/tests_for_code.rs | 20 --- .../impls_for_access/tests_for_access.rs | 4 +- src/packed_string/impls_for_editing.rs | 4 +- src/packed_string/tests_for_support.rs | 35 +---- src/traits.rs | 1 + src/traits/packed_char.rs | 11 +- tests/packed_char_derive.rs | 18 +++ 16 files changed, 334 insertions(+), 79 deletions(-) create mode 100644 derive/Cargo.toml create mode 100644 derive/src/funcs_for_expand.rs create mode 100644 derive/src/funcs_for_expand/tests_for_expand.rs create mode 100644 derive/src/lib.rs delete mode 100644 src/packed_string/funcs_for_code/tests_for_code.rs create mode 100644 tests/packed_char_derive.rs diff --git a/Cargo.lock b/Cargo.lock index 038a9fe..7a3e6ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,6 +51,7 @@ dependencies = [ name = "bit-string" version = "0.5.0" dependencies = [ + "bit-string-derive", "bitvec_simd", "codspeed-divan-compat", "int-interval", @@ -58,6 +59,16 @@ dependencies = [ "witnessed", ] +[[package]] +name = "bit-string-derive" +version = "0.5.0" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "bit-vec" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 87207e8..e187312 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,11 @@ keywords = ["bitstring", "bits", "bitset", "sequence"] categories = ["data-structures", "no-std"] exclude = ["/benches", "/src/**/tests_for_*", "/tests", ".github/"] +[workspace] +members = ["derive"] [dependencies] +bit-string-derive = { version = "0.5.0", path = "derive" } int-interval = "0.9.6" witnessed = "0.8.0" diff --git a/derive/Cargo.toml b/derive/Cargo.toml new file mode 100644 index 0000000..81ac59e --- /dev/null +++ b/derive/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "bit-string-derive" +version = "0.5.0" +edition = "2024" +description = "Derive macros for bit-string" +repository = "https://github.com/jcfangc/bit-string" +license = "MIT OR Apache-2.0" + +[lib] +proc-macro = true + +[dependencies] +proc-macro-crate = "3.3.0" +proc-macro2 = "1.0" +quote = "1.0" +syn = "2.0" diff --git a/derive/src/funcs_for_expand.rs b/derive/src/funcs_for_expand.rs new file mode 100644 index 0000000..7674b05 --- /dev/null +++ b/derive/src/funcs_for_expand.rs @@ -0,0 +1,144 @@ +use std::collections::BTreeMap; + +use proc_macro_crate::{FoundCrate, crate_name}; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::quote; +use syn::{Data, DeriveInput, Error, Expr, Fields, Lit, LitInt, Result}; + +pub(super) fn expand(input: DeriveInput) -> Result { + require_repr_u8(&input)?; + let (bits, bits_span) = parse_bits(&input)?; + if bits > 8 { + return Err(Error::new(bits_span, "packed bit width must not exceed 8")); + } + + let Data::Enum(data) = &input.data else { + return Err(Error::new_spanned( + &input.ident, + "PackedChar can only be derived for enums", + )); + }; + + let limit = 1u16 << bits; + let mut codes = BTreeMap::new(); + let mut variants = Vec::with_capacity(data.variants.len()); + + for variant in &data.variants { + if !matches!(variant.fields, Fields::Unit) { + return Err(Error::new_spanned( + &variant.fields, + "PackedChar variants must not contain fields", + )); + } + + let Some((_, Expr::Lit(expression))) = &variant.discriminant else { + return Err(Error::new_spanned( + &variant.ident, + "PackedChar variants require an explicit integer discriminant", + )); + }; + let Lit::Int(discriminant) = &expression.lit else { + return Err(Error::new_spanned( + &expression.lit, + "PackedChar discriminants must be integer literals", + )); + }; + let code = discriminant.base10_parse::()?; + if code >= limit { + return Err(Error::new_spanned( + discriminant, + format!("discriminant {code} does not fit in {bits} bits"), + )); + } + if let Some(previous) = codes.insert(code, discriminant.span()) { + let mut error = Error::new_spanned(discriminant, "duplicate packed discriminant"); + error.combine(Error::new(previous, "first used here")); + return Err(error); + } + + variants.push((&variant.ident, code)); + } + + let crate_path = bit_string_path(); + let name = &input.ident; + let (impl_generics, type_generics, where_clause) = input.generics.split_for_impl(); + let bits = LitInt::new(&bits.to_string(), bits_span); + let variant_names = variants.iter().map(|(variant, _)| variant); + let discriminants = variants + .iter() + .map(|(_, code)| LitInt::new(&code.to_string(), Span::call_site())); + + Ok(quote! { + impl #impl_generics #crate_path::traits::PackedChar<#bits> + for #name #type_generics #where_clause + { + #[inline] + fn code(self) -> u8 { + self as u8 + } + + #[inline] + fn from_code(code: u8) -> ::core::option::Option { + match code { + #(#discriminants => ::core::option::Option::Some(Self::#variant_names),)* + _ => ::core::option::Option::None, + } + } + } + }) +} + +fn require_repr_u8(input: &DeriveInput) -> Result<()> { + let mut found = false; + for attribute in &input.attrs { + if attribute.path().is_ident("repr") { + attribute.parse_nested_meta(|meta| { + found |= meta.path.is_ident("u8"); + Ok(()) + })?; + } + } + if found { + Ok(()) + } else { + Err(Error::new_spanned( + &input.ident, + "PackedChar requires #[repr(u8)]", + )) + } +} + +fn parse_bits(input: &DeriveInput) -> Result<(u8, Span)> { + let mut bits = None; + for attribute in &input.attrs { + if !attribute.path().is_ident("packed") { + continue; + } + attribute.parse_nested_meta(|meta| { + if !meta.path.is_ident("bits") { + return Err(meta.error("expected `bits = N`")); + } + if bits.is_some() { + return Err(meta.error("duplicate packed bit width")); + } + let literal: LitInt = meta.value()?.parse()?; + bits = Some((literal.base10_parse::()?, literal.span())); + Ok(()) + })?; + } + bits.ok_or_else(|| Error::new_spanned(&input.ident, "PackedChar requires #[packed(bits = N)]")) +} + +fn bit_string_path() -> TokenStream2 { + match crate_name("bit-string") { + Ok(FoundCrate::Itself) => quote!(crate), + Ok(FoundCrate::Name(name)) => { + let name = syn::Ident::new(&name, Span::call_site()); + quote!(::#name) + } + Err(_) => quote!(::bit_string), + } +} + +#[cfg(test)] +mod tests_for_expand; diff --git a/derive/src/funcs_for_expand/tests_for_expand.rs b/derive/src/funcs_for_expand/tests_for_expand.rs new file mode 100644 index 0000000..47fb61a --- /dev/null +++ b/derive/src/funcs_for_expand/tests_for_expand.rs @@ -0,0 +1,103 @@ +use super::*; +use quote::quote; +use syn::parse2; + +fn error(input: TokenStream2) -> String { + expand(parse2(input).unwrap()).unwrap_err().to_string() +} + +#[test] +fn accepts_strict_enum_encoding() { + let result = expand( + parse2(quote! { + #[repr(u8)] + #[packed(bits = 2)] + enum Letter { A = 0, B = 1, C = 2, D = 3 } + }) + .unwrap(), + ); + assert!(result.is_ok()); +} + +#[test] +fn requires_explicit_discriminants() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 1)] + enum Letter { A = 0, B } + }) + .contains("explicit integer discriminant") + ); +} + +#[test] +fn rejects_codes_outside_the_width() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 2)] + enum Letter { A = 4 } + }) + .contains("does not fit in 2 bits") + ); +} + +#[test] +fn rejects_widths_above_u8_capacity() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 9)] + enum Letter { A = 0 } + }) + .contains("must not exceed 8") + ); +} + +#[test] +fn rejects_variants_with_fields() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 1)] + enum Letter { A(u8) = 0 } + }) + .contains("must not contain fields") + ); +} + +#[test] +fn rejects_duplicate_codes() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 1)] + enum Letter { A = 0, B = 0 } + }) + .contains("duplicate packed discriminant") + ); +} + +#[test] +fn rejects_non_literal_discriminants() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 2)] + enum Letter { A = 1 + 1 } + }) + .contains("explicit integer discriminant") + ); +} + +#[test] +fn requires_repr_u8() { + assert!( + error(quote! { + #[packed(bits = 1)] + enum Letter { A = 0 } + }) + .contains("requires #[repr(u8)]") + ); +} diff --git a/derive/src/lib.rs b/derive/src/lib.rs new file mode 100644 index 0000000..ee3e10b --- /dev/null +++ b/derive/src/lib.rs @@ -0,0 +1,19 @@ +use proc_macro::TokenStream; +use syn::{DeriveInput, Error, parse_macro_input}; + +mod funcs_for_expand; + +use funcs_for_expand::expand; + +/// Derives a fixed-width packed encoding from explicit `#[repr(u8)]` enum +/// discriminants. +/// +/// The enum must use `#[packed(bits = N)]`, contain only fieldless variants, +/// and give every variant an integer-literal discriminant that fits in `N` +/// bits. +#[proc_macro_derive(PackedChar, attributes(packed))] +pub fn derive_packed_char(input: TokenStream) -> TokenStream { + expand(parse_macro_input!(input as DeriveInput)) + .unwrap_or_else(Error::into_compile_error) + .into() +} diff --git a/src/lib.rs b/src/lib.rs index ec1a492..58223ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ mod packed_string; pub use bit_str::BitStr; pub use bit_string::BitString; +pub use bit_string_derive::PackedChar; pub use packed_str::PackedStr; pub use packed_string::PackedString; diff --git a/src/packed_string.rs b/src/packed_string.rs index 3c9eb92..9223639 100644 --- a/src/packed_string.rs +++ b/src/packed_string.rs @@ -30,7 +30,7 @@ mod impls_for_iter; mod impls_for_matching; mod impls_for_ord; -use funcs_for_code::{checked_code, code_mask, write_code}; +use funcs_for_code::{code_mask, write_code}; #[cfg(test)] mod tests_for_support; diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 1d3e330..2c85441 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -10,29 +10,8 @@ pub(super) const fn code_mask() -> u8 { } } -#[inline] -pub(super) fn checked_code(character: C) -> u8 -where - C: PackedChar, -{ - let code = character.code(); - assert_eq!( - code & !code_mask::(), - 0, - "PackedChar::code does not fit in BITS" - ); - assert!( - C::from_code(code) == Some(character), - "PackedChar::code and PackedChar::from_code disagree" - ); - code -} - #[inline] pub(super) fn write_code(bits: &mut BitString, position: usize, code: u8) { let start = position * usize::from(BITS); bits.set_chunk(start, u64::from(code), usize::from(BITS)); } - -#[cfg(test)] -mod tests_for_code; diff --git a/src/packed_string/funcs_for_code/tests_for_code.rs b/src/packed_string/funcs_for_code/tests_for_code.rs deleted file mode 100644 index a69100d..0000000 --- a/src/packed_string/funcs_for_code/tests_for_code.rs +++ /dev/null @@ -1,20 +0,0 @@ -use super::*; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Broken; - -impl PackedChar<1> for Broken { - fn code(self) -> u8 { - 2 - } - - fn from_code(_: u8) -> Option { - Some(Self) - } -} - -#[test] -#[should_panic(expected = "does not fit")] -fn rejects_a_code_that_does_not_fit() { - let _ = checked_code::(Broken); -} diff --git a/src/packed_string/impls_for_access/tests_for_access.rs b/src/packed_string/impls_for_access/tests_for_access.rs index 9897e56..232261b 100644 --- a/src/packed_string/impls_for_access/tests_for_access.rs +++ b/src/packed_string/impls_for_access/tests_for_access.rs @@ -10,8 +10,8 @@ fn get_first_and_last_decode_values() { #[test] fn one_value_type_needs_no_payload_bits() { - let value = OnlyString::from_chars([Only; 100]); + let value = OnlyString::from_chars([Only::Value; 100]); assert_eq!(value.char_len(), 100); assert_eq!(value.bits().bit_len(), 0); - assert_eq!(value.get(99), Some(Only)); + assert_eq!(value.get(99), Some(Only::Value)); } diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 6029566..194de85 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -6,7 +6,7 @@ where { /// Appends the character's code directly to the payload. pub fn push(&mut self, character: C) { - let code = checked_code::(character); + let code = character.code(); let new_len = self .char_len .checked_add(1) @@ -26,7 +26,7 @@ where pub fn set(&mut self, index: usize, character: C) -> Option { let previous = self.get(index)?; - write_code::(&mut self.bits, index, checked_code::(character)); + write_code::(&mut self.bits, index, character.code()); Some(previous) } diff --git a/src/packed_string/tests_for_support.rs b/src/packed_string/tests_for_support.rs index a01a1fb..251aba1 100644 --- a/src/packed_string/tests_for_support.rs +++ b/src/packed_string/tests_for_support.rs @@ -2,8 +2,9 @@ use core::fmt; use super::PackedChar; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PackedChar)] #[repr(u8)] +#[packed(bits = 2)] pub(super) enum Letter { A = 0b00, B = 0b01, @@ -11,22 +12,6 @@ pub(super) enum Letter { D = 0b11, } -impl PackedChar<2> for Letter { - fn code(self) -> u8 { - self as u8 - } - - fn from_code(code: u8) -> Option { - match code { - 0 => Some(Self::A), - 1 => Some(Self::B), - 2 => Some(Self::C), - 3 => Some(Self::D), - _ => None, - } - } -} - impl fmt::Display for Letter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(match self { @@ -38,17 +23,11 @@ impl fmt::Display for Letter { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) struct Only; - -impl PackedChar<0> for Only { - fn code(self) -> u8 { - 0 - } - - fn from_code(code: u8) -> Option { - (code == 0).then_some(Self) - } +#[derive(Clone, Copy, Debug, PartialEq, Eq, PackedChar)] +#[repr(u8)] +#[packed(bits = 0)] +pub(super) enum Only { + Value = 0, } pub(super) type LetterString = super::PackedString; diff --git a/src/traits.rs b/src/traits.rs index c682bf3..0b48cb0 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -7,6 +7,7 @@ pub(crate) mod words_find; pub(crate) mod words_ord; pub(crate) mod words_scan; +pub use bit_string_derive::PackedChar; pub use packed_char::PackedChar; pub(crate) use word_ord::*; pub(crate) use words_arith::*; diff --git a/src/traits/packed_char.rs b/src/traits/packed_char.rs index 00fb029..547574a 100644 --- a/src/traits/packed_char.rs +++ b/src/traits/packed_char.rs @@ -1,16 +1,17 @@ /// A character with a fixed-width bit representation. /// -/// This trait is intended for fieldless `#[repr(u8)]` enums. Implementations -/// must satisfy all of the following invariants: +/// This trait is intended for fieldless `#[repr(u8)]` enums. Prefer deriving +/// it with `#[derive(PackedChar)]` and `#[packed(bits = N)]`; manual +/// implementations must satisfy all of the following invariants: /// /// - `BITS` does not exceed eight; /// - [`code`](Self::code) fits in `BITS` bits; /// - `from_code(code(value)) == Some(value)`; /// - each valid code identifies at most one value. /// -/// [`PackedString`](crate::PackedString) validates `BITS` when it is -/// constructed, and checks the `code`/`from_code` agreement whenever it writes -/// a value. No `transmute` is used. +/// [`PackedString`](crate::PackedString) relies on these invariants. The derive +/// macro validates them at compile time and generates both directions from the +/// same discriminants. No `transmute` is used. pub trait PackedChar: Copy + Eq { /// Returns the character's packed value. fn code(self) -> u8; diff --git a/tests/packed_char_derive.rs b/tests/packed_char_derive.rs new file mode 100644 index 0000000..e8c5dbf --- /dev/null +++ b/tests/packed_char_derive.rs @@ -0,0 +1,18 @@ +use bit_string::{PackedString, traits::PackedChar}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PackedChar)] +#[repr(u8)] +#[packed(bits = 2)] +enum Symbol { + Zero = 0, + One = 1, + Two = 2, +} + +#[test] +fn derived_encoding_round_trips_through_packed_string() { + let string = PackedString::::from_chars([Symbol::Two, Symbol::Zero, Symbol::One]); + assert_eq!(string.get(0), Some(Symbol::Two)); + assert_eq!(string.get(1), Some(Symbol::Zero)); + assert_eq!(string.get(2), Some(Symbol::One)); +} From 600ef8abdf18a0b383ac6b4f63ae3975b15348bc Mon Sep 17 00:00:00 2001 From: juncheng Date: Thu, 27 Aug 2026 14:21:33 +0000 Subject: [PATCH 06/79] chore: add reproducible codegen comparison tooling --- .gitignore | 7 + scripts/bench_config.py | 4 + scripts/codegen_roots.txt | 18 ++ scripts/compare_bench.py | 99 +++++++++ scripts/compare_codegen.py | 130 ++++++++++++ scripts/run_bench_comparison.py | 62 ++++++ scripts/run_codegen_comparison.py | 46 +++++ src/bit_string/impls_for_bit_arith.rs | 1 + .../impls_for_leading_zeros.rs | 193 +++--------------- .../impls_for_trailing_zeros.rs | 134 ++++++++++++ 10 files changed, 526 insertions(+), 168 deletions(-) create mode 100644 scripts/bench_config.py create mode 100644 scripts/codegen_roots.txt create mode 100755 scripts/compare_bench.py create mode 100755 scripts/compare_codegen.py create mode 100755 scripts/run_bench_comparison.py create mode 100755 scripts/run_codegen_comparison.py create mode 100644 src/bit_string/impls_for_bit_arith/impls_for_trailing_zeros.rs diff --git a/.gitignore b/.gitignore index b7cd4ac..3743c0e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,10 @@ /.venv /proptest-regressions .cargo/config.toml + +# Python tooling caches +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ diff --git a/scripts/bench_config.py b/scripts/bench_config.py new file mode 100644 index 0000000..39a6a9b --- /dev/null +++ b/scripts/bench_config.py @@ -0,0 +1,4 @@ +"""Shared constants for reproducible baseline/codegen benchmark runs.""" + +BASELINE_SHA = "c15fa1c254ea5b2bbabfe5f008d41f91e8297b89" +RUSTFLAGS = "-C target-cpu=native" diff --git a/scripts/codegen_roots.txt b/scripts/codegen_roots.txt new file mode 100644 index 0000000..7d4d1b0 --- /dev/null +++ b/scripts/codegen_roots.txt @@ -0,0 +1,18 @@ +# Stable backend roots. Public wrappers may inline these, so backend symbols are +# used where they are emitted as standalone functions in the release rlib. +bit_string::traits::words_scan::funcs_for_count_ones::scalar::count_words +bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words +bit_string::traits::words_scan::funcs_for_count_ones::ssse3::count_words +bit_string::traits::words_scan::funcs_for_count_ones::neon::count_words +bit_string::traits::words_arith::funcs_for_binary_core::scalar::words +bit_string::traits::words_arith::funcs_for_binary_core::sse2::words +bit_string::traits::words_arith::funcs_for_binary_core::avx2::words +bit_string::traits::words_arith::funcs_for_binary_core::neon::words +bit_string::traits::words_arith::funcs_for_not_core::scalar::words +bit_string::traits::words_arith::funcs_for_not_core::sse2::words +bit_string::traits::words_arith::funcs_for_not_core::avx2::words +bit_string::traits::words_arith::funcs_for_not_core::neon::words +bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::scalar::eq_words +bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::avx2::eq_words +bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::sse41::eq_words +bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::neon::eq_words diff --git a/scripts/compare_bench.py b/scripts/compare_bench.py new file mode 100755 index 0000000..a07a78d --- /dev/null +++ b/scripts/compare_bench.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Compare Divan text reports using each benchmark's median time.""" + +from __future__ import annotations + +import argparse +import re +import statistics +import sys +from pathlib import Path + +TIME = re.compile(r"(?P[0-9]+(?:\.[0-9]+)?)\s*(?Pns|µs|us|ms|s)") +NAME = re.compile(r"^[├╰]─\s*(?P.+?)(?:\s{2,}|\s+Timer precision:)") +UNITS_TO_NS = {"ns": 1.0, "µs": 1_000.0, "us": 1_000.0, "ms": 1_000_000.0, "s": 1_000_000_000.0} + + +def parse_report(path: Path) -> dict[str, float]: + """Parse Divan's columnar text report, returning median values in ns.""" + result: dict[str, float] = {} + pending: str | None = None + for raw in path.read_text(encoding="utf-8").splitlines(): + match = NAME.match(raw) + if match: + pending = match.group("name").strip() + if pending is None: + continue + values = list(TIME.finditer(raw)) + # Data rows contain fastest, slowest, median, and mean in that order. + if len(values) >= 4: + median = values[2] + result[pending] = float(median.group("value")) * UNITS_TO_NS[median.group("unit")] + pending = None + elif raw.strip() and not raw.startswith((" ", "\t")): + pending = None + if not result: + raise ValueError(f"no Divan benchmark medians found in {path}") + return result + + +def fmt_ns(value: float) -> str: + if value >= 1_000_000_000: + return f"{value / 1_000_000_000:.3f} s" + if value >= 1_000_000: + return f"{value / 1_000_000:.3f} ms" + if value >= 1_000: + return f"{value / 1_000:.3f} µs" + return f"{value:.3f} ns" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("baseline", type=Path, nargs="+", help="baseline Divan text reports") + parser.add_argument("--current", type=Path, nargs="+", required=True, help="current Divan text reports") + parser.add_argument("--warning-threshold", type=float, default=5.0) + args = parser.parse_args() + + def aggregate(paths: list[Path]) -> dict[str, float]: + samples: dict[str, list[float]] = {} + for path in paths: + for name, value in parse_report(path).items(): + samples.setdefault(name, []).append(value) + return {name: statistics.median(values) for name, values in samples.items()} + + baseline, current = aggregate(args.baseline), aggregate(args.current) + common = sorted(baseline.keys() & current.keys()) + only_baseline = sorted(baseline.keys() - current.keys()) + only_current = sorted(current.keys() - baseline.keys()) + improved = unchanged = regressed = warnings = 0 + print(f"{'benchmark':<50} {'baseline':>12} {'current':>12} {'delta':>10} status") + print("-" * 96) + for name in common: + before, after = baseline[name], current[name] + delta = (after - before) / before * 100 + if delta < 0: + improved += 1 + elif delta > 0: + regressed += 1 + else: + unchanged += 1 + status = "WARNING" if delta > args.warning_threshold else "OK" + if status == "WARNING": + warnings += 1 + print(f"WARNING: {name} regressed by {delta:+.2f}%", file=sys.stderr) + print(f"{name:<50.50} {fmt_ns(before):>12} {fmt_ns(after):>12} {delta:>+9.2f}% {status}") + for name in only_baseline: + print(f"missing in current: {name}", file=sys.stderr) + for name in only_current: + print(f"missing in baseline: {name}", file=sys.stderr) + print("\nSUMMARY") + print(f"matched benchmarks : {len(common)}") + print(f"improved : {improved}") + print(f"unchanged : {unchanged}") + print(f"regressed : {regressed}") + print(f"warnings : {warnings}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py new file mode 100755 index 0000000..ebf5e39 --- /dev/null +++ b/scripts/compare_codegen.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Compare normalized release-rlib disassembly for selected codegen roots.""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +FUNCTION = re.compile(r"^\s*[0-9a-f]+ <(.+)>:\s*$") +INSTRUCTION = re.compile(r"^\s*[0-9a-f]+:\s+(?:[0-9a-f]{2}(?:\s+[0-9a-f]{2})*\s+)?(.+?)\s*$") +TARGET_INSTRUCTION = re.compile(r"^(?:j[a-z]+|call|loop[a-z]*)(?:\s|$)") +CONDITIONAL_BRANCH = re.compile(r"^(?:ja|jae|jb|jbe|jc|je|jg|jge|jl|jle|jna|jnae|jnb|jnbe|jnc|jne|jng|jnge|jnl|jnle|jno|jnp|jns|jnz|jo|jp|jpe|jpo|js|jz)(?:\s|$)") +TARGET = re.compile(r"(?:0x[0-9a-f]+|[0-9a-f]+)\s+<[^>]+>") + + +@dataclass(frozen=True) +class Function: + instructions: tuple[str, ...] + bytes: int + calls: int + branches: int + + @property + def digest(self) -> str: + return hashlib.sha256("\n".join(self.instructions).encode()).hexdigest()[:16] + + +def normalized_instruction(text: str) -> str: + text = text.split("#", 1)[0].strip() + if not text: + return "" + if TARGET_INSTRUCTION.match(text): + text = TARGET.sub("", text) + return " ".join(text.split()) + + +def load_functions(artifact: Path, objdump: str) -> dict[str, Function]: + output = subprocess.check_output([objdump, "-drwC", str(artifact)], text=True, errors="replace") + found: dict[str, list[str]] = {} + sizes: dict[str, int] = {} + current: str | None = None + for line in output.splitlines(): + label = FUNCTION.match(line) + if label: + current = label.group(1) + found.setdefault(current, []) + sizes.setdefault(current, 0) + continue + if current is None: + continue + match = INSTRUCTION.match(line) + if not match: + continue + raw = match.group(1) + normalized = normalized_instruction(raw) + if not normalized: + continue + found[current].append(normalized) + # The byte column is the sequence before the mnemonic. Count bytes from + # the source line rather than estimating from the mnemonic. + prefix = line.split(":", 1)[1].split(raw, 1)[0] + sizes[current] += len(re.findall(r"\b[0-9a-f]{2}\b", prefix)) + result: dict[str, Function] = {} + for name, instructions in found.items(): + if not instructions: + continue + calls = sum(1 for item in instructions if item.startswith("call ")) + branches = sum(1 for item in instructions if CONDITIONAL_BRANCH.match(item)) + result[name] = Function(tuple(instructions), sizes[name], calls, branches) + return result + + +def symbols(path: Path) -> list[str]: + return [line.strip() for line in path.read_text().splitlines() if line.strip() and not line.lstrip().startswith("#")] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("baseline", type=Path) + parser.add_argument("current", type=Path) + parser.add_argument("--symbols", type=Path, required=True) + parser.add_argument("--objdump", default="objdump") + args = parser.parse_args() + before, after = load_functions(args.baseline, args.objdump), load_functions(args.current, args.objdump) + changed = missing = unavailable = 0 + names = symbols(args.symbols) + print(f"{'symbol':<78} {'bytes':>13} {'insns':>13} {'calls':>11} {'branches':>14} status") + print("-" * 140) + for name in names: + old, new = before.get(name), after.get(name) + if old is None or new is None: + if old is None and new is None: + unavailable += 1 + status = "UNAVAILABLE" + else: + missing += 1 + status = "MISSING" + print(f"{name:<78} {'-':>6} -> {'-':<6} {'-':>6} -> {'-':<6} {'-':>5} -> {'-':<5} {'-':>7} -> {'-':<7} {status}") + continue + identical = old.instructions == new.instructions + if not identical: + changed += 1 + status = "CODEGEN IDENTICAL" if identical else "CODEGEN CHANGED" + print(f"{name:<78} {old.bytes:>6} -> {new.bytes:<6} {len(old.instructions):>6} -> {len(new.instructions):<6} {old.calls:>5} -> {new.calls:<5} {old.branches:>7} -> {new.branches:<7} {old.digest} -> {new.digest} {status}") + if new.calls > old.calls: + print(f"WARNING: {name} call count increased from {old.calls} to {new.calls}", file=sys.stderr) + if not identical: + print(f"--- baseline {name}\n+++ current {name}") + for left, right in zip(old.instructions, new.instructions): + if left != right: + print(f"- {left}\n+ {right}") + break + if len(old.instructions) != len(new.instructions): + print(f"instruction count: {len(old.instructions)} -> {len(new.instructions)}") + print("\nCODEGEN SUMMARY") + print(f"compared : {len(names) - missing - unavailable}") + print(f"identical : {len(names) - missing - unavailable - changed}") + print(f"changed : {changed}") + print(f"missing : {missing}") + print(f"unavailable : {unavailable}") + return 0 if not changed and not missing else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_bench_comparison.py b/scripts/run_bench_comparison.py new file mode 100755 index 0000000..19e3774 --- /dev/null +++ b/scripts/run_bench_comparison.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Run interleaved Divan benchmarks for a fixed baseline and this worktree.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from bench_config import BASELINE_SHA, RUSTFLAGS + +ORDER = ("baseline", "current", "current", "baseline", "baseline", "current", "current", "baseline", "baseline", "current") + + +def run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None, output: Path | None = None) -> None: + completed = subprocess.run(command, cwd=cwd, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + if output is not None: + output.write_text(completed.stdout, encoding="utf-8") + if completed.returncode: + sys.stdout.write(completed.stdout) + raise SystemExit(f"command failed ({completed.returncode}): {' '.join(command)}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--runs", type=int, default=5, help="runs per revision; default is 5") + parser.add_argument("--warning-threshold", type=float, default=5.0) + parser.add_argument("--baseline", default=BASELINE_SHA) + args = parser.parse_args() + if args.runs != 5: + raise SystemExit("this runner currently fixes --runs at 5 to preserve the documented interleaving") + + root = Path(__file__).resolve().parents[1] + output_dir = root / "target" / "bench-comparison" + baseline_dir = output_dir / "baseline-worktree" + output_dir.mkdir(parents=True, exist_ok=True) + if not baseline_dir.exists(): + run(["git", "worktree", "add", "--detach", str(baseline_dir), args.baseline], cwd=root) + else: + run(["git", "-C", str(baseline_dir), "checkout", "--detach", args.baseline], cwd=root) + + env = os.environ.copy() + env["RUSTFLAGS"] = RUSTFLAGS + command = ["cargo", "bench", "--", "--color", "never"] + counts = {"baseline": 0, "current": 0} + for revision in ORDER: + counts[revision] += 1 + report = output_dir / f"{revision}-{counts[revision]}.txt" + print(f"[{revision} {counts[revision]}/5] {' '.join(command)}") + run(command, cwd=baseline_dir if revision == "baseline" else root, env=env, output=report) + + compare = [sys.executable, str(root / "scripts" / "compare_bench.py")] + compare += [str(output_dir / f"baseline-{i}.txt") for i in range(1, 6)] + compare += ["--current"] + [str(output_dir / f"current-{i}.txt") for i in range(1, 6)] + compare += ["--warning-threshold", str(args.warning_threshold)] + return subprocess.run(compare, cwd=root, check=False).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py new file mode 100755 index 0000000..9c4c0f0 --- /dev/null +++ b/scripts/run_codegen_comparison.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Build the fixed baseline and current worktree, then compare release codegen.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from bench_config import BASELINE_SHA, RUSTFLAGS + + +def run(command: list[str], cwd: Path, env: dict[str, str]) -> None: + print(f"[{cwd}] {' '.join(command)}", flush=True) + completed = subprocess.run(command, cwd=cwd, env=env, check=False) + if completed.returncode: + raise SystemExit(completed.returncode) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", default=BASELINE_SHA) + parser.add_argument("--symbols", type=Path, default=None) + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + output = root / "target" / "codegen-comparison" + baseline = output / "baseline-worktree" + output.mkdir(parents=True, exist_ok=True) + if not baseline.exists(): + run(["git", "worktree", "add", "--detach", str(baseline), args.baseline], root, os.environ.copy()) + else: + run(["git", "-C", str(baseline), "checkout", "--detach", args.baseline], root, os.environ.copy()) + env = os.environ.copy() + env["RUSTFLAGS"] = RUSTFLAGS + command = ["cargo", "rustc", "--release", "--lib", "--", "--emit=obj"] + run(command, baseline, env) + run(command, root, env) + symbols = args.symbols or (root / "scripts" / "codegen_roots.txt") + compare = [sys.executable, str(root / "scripts" / "compare_codegen.py"), str(baseline / "target" / "release" / "libbit_string.rlib"), str(root / "target" / "release" / "libbit_string.rlib"), "--symbols", str(symbols)] + return subprocess.run(compare, cwd=root, env=env, check=False).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/bit_string/impls_for_bit_arith.rs b/src/bit_string/impls_for_bit_arith.rs index 5e26ab7..b207815 100644 --- a/src/bit_string/impls_for_bit_arith.rs +++ b/src/bit_string/impls_for_bit_arith.rs @@ -5,6 +5,7 @@ mod impls_for_not; mod impls_for_or; mod impls_for_shl; mod impls_for_shr; +mod impls_for_trailing_zeros; mod impls_for_xor; use crate::bit_string::errors::BitStringLenMismatch; diff --git a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs index 4a17438..fdce4db 100644 --- a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs +++ b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs @@ -11,47 +11,37 @@ impl BitString { if bit_len == 0 { return 0; } - // SAFETY: `words` is always non-empty when bit_len > 0 - // (BitString invariants guarantee at least one word). let words_ptr = self.words.as_ptr(); - - // ── First-word fast path ────────────────────────────────── - // SAFETY: `bit_len > 0` (guarded above), so `words` has at least - // one element per the BitString invariant. + // SAFETY: `bit_len > 0`, so the first backing word exists. let w0 = unsafe { *words_ptr }; if w0 != 0 { return (w0.trailing_zeros() as usize).min(bit_len); } - - // One- and two-word strings are common enough to avoid the generic scan. if bit_len <= WORD_BITS { return bit_len; } if bit_len <= WORD_BITS * 2 { let used = bit_len - WORD_BITS; - // SAFETY: bit_len > WORD_BITS, so the second backing word exists. - let w1 = unsafe { *words_ptr.add(1) }; + // SAFETY: bit_len > WORD_BITS, so a second backing word exists. let mask = if used == WORD_BITS { u64::MAX } else { (1u64 << used) - 1 }; - let w1 = w1 & mask; - if w1 == 0 { - return bit_len; - } - return WORD_BITS + w1.trailing_zeros() as usize; + let w1 = unsafe { *words_ptr.add(1) } & mask; + return if w1 == 0 { + bit_len + } else { + WORD_BITS + w1.trailing_zeros() as usize + }; } - - // ── Tiny inputs ─────────────────────────────────── let last_wi = (bit_len - 1) / WORD_BITS; let end_rem = bit_len % WORD_BITS; let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; if mid_end < SMALL_WORDS { - let mut scanned = WORD_BITS; // word 0 already checked above - // SAFETY: i ranges in 1..mid_end. words contains every - // backing word covered by bit_len. + let mut scanned = WORD_BITS; for i in 1..mid_end { + // SAFETY: `i < mid_end`, which is covered by backing storage. let w = unsafe { *words_ptr.add(i) }; if w != 0 { return (scanned + w.trailing_zeros() as usize).min(bit_len); @@ -59,17 +49,16 @@ impl BitString { scanned += WORD_BITS; } if end_rem != 0 { - // SAFETY: a partial final word exists at mid_end. + // SAFETY: a partial final word exists at `mid_end`. let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); - if last == 0 { - return bit_len; - } - return (scanned + last.trailing_zeros() as usize).min(bit_len); + return if last == 0 { + bit_len + } else { + (scanned + last.trailing_zeros() as usize).min(bit_len) + }; } return bit_len; } - - // ── SIMD via trait ──────────────────────────────────────── self.words() .leading_value_bits::(0, bit_len) } @@ -82,22 +71,18 @@ impl BitString { return 0; } let words_ptr = self.words.as_ptr(); - - // SAFETY: `bit_len > 0` (guarded above), so `words` has at least - // one element per the BitString invariant. + // SAFETY: `bit_len > 0`, so the first backing word exists. let w0 = unsafe { *words_ptr }; if w0 != u64::MAX { return ((!w0).trailing_zeros() as usize).min(bit_len); } - let last_wi = (bit_len - 1) / WORD_BITS; let end_rem = bit_len % WORD_BITS; let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; if mid_end < SMALL_WORDS { let mut scanned = WORD_BITS; - // SAFETY: `i < mid_end`. `words` has at least - // `mid_end + (end_rem != 0) as usize` elements. for i in 1..mid_end { + // SAFETY: `i < mid_end`, which is covered by backing storage. let w = unsafe { *words_ptr.add(i) }; if w != u64::MAX { return (scanned + (!w).trailing_zeros() as usize).min(bit_len); @@ -105,146 +90,18 @@ impl BitString { scanned += WORD_BITS; } if end_rem != 0 { - // SAFETY: `mid_end` is the index of the last partial word - // and is within bounds (backing storage covers all bits). + // SAFETY: a partial final word exists at `mid_end`. let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); - if last == ((1u64 << end_rem).wrapping_sub(1)) { - return bit_len; - } - return (scanned + (!last).trailing_zeros() as usize).min(bit_len); + let fill = (1u64 << end_rem).wrapping_sub(1); + return if last == fill { + bit_len + } else { + (scanned + (!last).trailing_zeros() as usize).min(bit_len) + }; } return bit_len; } - self.words() .leading_value_bits::(0, bit_len) } - - /// Returns the number of consecutive `false` bits from the end. - #[inline] - pub fn trailing_zeros(&self) -> usize { - let bit_len = self.bit_len; - if bit_len == 0 { - return 0; - } - let words_ptr = self.words.as_ptr(); - - if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { - return count; - } - - // ── Last partial word ──────────────────────────────────── - let end_rem = bit_len % WORD_BITS; - if end_rem != 0 { - let last_wi = (bit_len - 1) / WORD_BITS; - // SAFETY: `last_wi` is a valid index — `bit_len > 0` and - // `words` always has ≥ (bit_len + 63) / 64 elements. - let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); - if last != 0 { - let shifted = last << (WORD_BITS - end_rem); - return (shifted.leading_zeros() as usize).min(bit_len); - } - } - - // ── Rightmost full word ────────────────────────────────── - if bit_len > WORD_BITS { - let last_full = if end_rem != 0 { - (bit_len - 1) / WORD_BITS - 1 - } else { - (bit_len - 1) / WORD_BITS - }; - // SAFETY: `last_full < (bit_len + 63) / 64`, so it is a valid - // index into `words` (backing storage covers all bits). - let w = unsafe { *words_ptr.add(last_full) }; - if w != 0 { - let tail = if end_rem != 0 { end_rem } else { 0 }; - return (tail + w.leading_zeros() as usize).min(bit_len); - } - } - - self.words() - .trailing_value_bits::(0, bit_len) - } - - /// Returns the number of consecutive `true` bits from the end. - #[inline] - pub fn trailing_ones(&self) -> usize { - let bit_len = self.bit_len; - if bit_len == 0 { - return 0; - } - let words_ptr = self.words.as_ptr(); - - if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { - return count; - } - - let end_rem = bit_len % WORD_BITS; - if end_rem != 0 { - let last_wi = (bit_len - 1) / WORD_BITS; - // SAFETY: `last_wi` is a valid index per the same invariant - // as trailing_zeros (BitString backing storage covers all bits). - let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); - if last != ((1u64 << end_rem).wrapping_sub(1)) { - let shifted = (!last) << (WORD_BITS - end_rem); - return (shifted.leading_zeros() as usize).min(bit_len); - } - } - - if bit_len > WORD_BITS { - let last_full = if end_rem != 0 { - (bit_len - 1) / WORD_BITS - 1 - } else { - (bit_len - 1) / WORD_BITS - }; - // SAFETY: `last_full` is a valid index into `words` per the - // BitString backing-storage invariant. - let w = unsafe { *words_ptr.add(last_full) }; - if w != u64::MAX { - let tail = if end_rem != 0 { end_rem } else { 0 }; - return (tail + (!w).leading_zeros() as usize).min(bit_len); - } - } - - self.words() - .trailing_value_bits::(0, bit_len) - } -} - -/// Handles owned strings that fit in at most two words without entering the -/// generic reverse scanner. -/// -/// # Safety -/// -/// `words_ptr` must point to the backing storage for `bit_len` bits. -#[inline] -unsafe fn trailing_small(words_ptr: *const u64, bit_len: usize) -> Option { - if bit_len > WORD_BITS * 2 { - return None; - } - - let last_wi = (bit_len - 1) / WORD_BITS; - let used = bit_len - last_wi * WORD_BITS; - let mask = if used == WORD_BITS { - u64::MAX - } else { - (1u64 << used) - 1 - }; - // SAFETY: the caller guarantees backing storage for every covered word. - let last = unsafe { *words_ptr.add(last_wi) }; - let mismatch = (last ^ FILL) & mask; - if mismatch != 0 { - return Some((mismatch << (WORD_BITS - used)).leading_zeros() as usize); - } - if last_wi == 0 { - return Some(bit_len); - } - - // SAFETY: last_wi == 1, so the first backing word also exists. - let first_mismatch = unsafe { *words_ptr } ^ FILL; - if first_mismatch == 0 { - Some(bit_len) - } else { - Some(used + first_mismatch.leading_zeros() as usize) - } } diff --git a/src/bit_string/impls_for_bit_arith/impls_for_trailing_zeros.rs b/src/bit_string/impls_for_bit_arith/impls_for_trailing_zeros.rs new file mode 100644 index 0000000..fa9d4c1 --- /dev/null +++ b/src/bit_string/impls_for_bit_arith/impls_for_trailing_zeros.rs @@ -0,0 +1,134 @@ +use crate::traits::WordsScan; +use crate::{FILL_ONES, FILL_ZEROS, WORD_BITS}; + +use super::BitString; + +impl BitString { + /// Returns the number of consecutive `false` bits from the end. + #[inline] + pub fn trailing_zeros(&self) -> usize { + let bit_len = self.bit_len; + if bit_len == 0 { + return 0; + } + let words_ptr = self.words.as_ptr(); + + if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { + return count; + } + + // ── Last partial word ──────────────────────────────────── + let end_rem = bit_len % WORD_BITS; + if end_rem != 0 { + let last_wi = (bit_len - 1) / WORD_BITS; + // SAFETY: `last_wi` is a valid index — `bit_len > 0` and + // `words` always has ≥ (bit_len + 63) / 64 elements. + let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); + if last != 0 { + let shifted = last << (WORD_BITS - end_rem); + return (shifted.leading_zeros() as usize).min(bit_len); + } + } + + // ── Rightmost full word ────────────────────────────────── + if bit_len > WORD_BITS { + let last_full = if end_rem != 0 { + (bit_len - 1) / WORD_BITS - 1 + } else { + (bit_len - 1) / WORD_BITS + }; + // SAFETY: `last_full < (bit_len + 63) / 64`, so it is a valid + // index into `words` (backing storage covers all bits). + let w = unsafe { *words_ptr.add(last_full) }; + if w != 0 { + let tail = if end_rem != 0 { end_rem } else { 0 }; + return (tail + w.leading_zeros() as usize).min(bit_len); + } + } + + self.words() + .trailing_value_bits::(0, bit_len) + } + + /// Returns the number of consecutive `true` bits from the end. + #[inline] + pub fn trailing_ones(&self) -> usize { + let bit_len = self.bit_len; + if bit_len == 0 { + return 0; + } + let words_ptr = self.words.as_ptr(); + + if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { + return count; + } + + let end_rem = bit_len % WORD_BITS; + if end_rem != 0 { + let last_wi = (bit_len - 1) / WORD_BITS; + // SAFETY: `last_wi` is a valid index per the same invariant + // as trailing_zeros (BitString backing storage covers all bits). + let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); + if last != ((1u64 << end_rem).wrapping_sub(1)) { + let shifted = (!last) << (WORD_BITS - end_rem); + return (shifted.leading_zeros() as usize).min(bit_len); + } + } + + if bit_len > WORD_BITS { + let last_full = if end_rem != 0 { + (bit_len - 1) / WORD_BITS - 1 + } else { + (bit_len - 1) / WORD_BITS + }; + // SAFETY: `last_full` is a valid index into `words` per the + // BitString backing-storage invariant. + let w = unsafe { *words_ptr.add(last_full) }; + if w != u64::MAX { + let tail = if end_rem != 0 { end_rem } else { 0 }; + return (tail + (!w).leading_zeros() as usize).min(bit_len); + } + } + + self.words() + .trailing_value_bits::(0, bit_len) + } +} + +/// Handles owned strings that fit in at most two words without entering the +/// generic reverse scanner. +/// +/// # Safety +/// +/// `words_ptr` must point to the backing storage for `bit_len` bits. +#[inline] +unsafe fn trailing_small(words_ptr: *const u64, bit_len: usize) -> Option { + if bit_len > WORD_BITS * 2 { + return None; + } + + let last_wi = (bit_len - 1) / WORD_BITS; + let used = bit_len - last_wi * WORD_BITS; + let mask = if used == WORD_BITS { + u64::MAX + } else { + (1u64 << used) - 1 + }; + // SAFETY: the caller guarantees backing storage for every covered word. + let last = unsafe { *words_ptr.add(last_wi) }; + let mismatch = (last ^ FILL) & mask; + if mismatch != 0 { + return Some((mismatch << (WORD_BITS - used)).leading_zeros() as usize); + } + if last_wi == 0 { + return Some(bit_len); + } + + // SAFETY: last_wi == 1, so the first backing word also exists. + let first_mismatch = unsafe { *words_ptr } ^ FILL; + if first_mismatch == 0 { + Some(bit_len) + } else { + Some(used + first_mismatch.leading_zeros() as usize) + } +} From 6273527b5da8f851e2b9bf3ce5b43ace474bd7d5 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 03:31:25 +0000 Subject: [PATCH 07/79] fix: harden codegen comparison tooling --- .github/workflows/gate.yml | 15 +- Cargo.toml | 2 +- scripts/bench_config.py | 9 +- scripts/codegen_harness.rs | 108 +++++++++ scripts/codegen_roots.toml | 89 +++++++ scripts/codegen_roots.txt | 18 -- scripts/compare_bench.py | 81 +++---- scripts/compare_codegen.py | 222 ++++++++++++------ scripts/divan_parser.py | 54 +++++ scripts/kernel_inventory.toml | 67 ++++++ scripts/run_bench_comparison.py | 5 +- scripts/run_codegen_comparison.py | 90 +++++-- scripts/tests/test_bench.py | 68 ++++++ scripts/tests/test_codegen.py | 95 ++++++++ .../impls_for_leading_zeros.rs | 65 +++-- 15 files changed, 805 insertions(+), 183 deletions(-) create mode 100644 scripts/codegen_harness.rs create mode 100644 scripts/codegen_roots.toml delete mode 100644 scripts/codegen_roots.txt create mode 100644 scripts/divan_parser.py create mode 100644 scripts/kernel_inventory.toml create mode 100644 scripts/tests/test_bench.py create mode 100644 scripts/tests/test_codegen.py diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index ef0abe6..afe3dec 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -19,27 +19,33 @@ jobs: fail-fast: false matrix: include: - - name: linux-x64-scalar + - name: linux-x64-baseline os: ubuntu-latest rustflags: "" + codegen_config: x86_64-baseline - name: linux-x64-sse4.1 os: ubuntu-latest rustflags: "-C target-feature=+sse4.1" + codegen_config: x86_64-sse41 - name: linux-x64-avx2 os: ubuntu-latest rustflags: "-C target-feature=+avx2" + codegen_config: x86_64-avx2 - name: linux-arm64-neon os: ubuntu-24.04-arm rustflags: "-C target-feature=+neon" + codegen_config: aarch64-neon env: RUSTFLAGS: ${{ matrix.rustflags }} steps: - uses: actions/checkout@v5 + with: + fetch-depth: 0 - uses: dtolnay/rust-toolchain@stable @@ -60,3 +66,10 @@ jobs: - name: Test run: cargo test --locked + + - name: Test comparison tooling + if: matrix.name == 'linux-x64-baseline' + run: python3 -m unittest discover -s scripts/tests -v + + - name: Compare codegen with fixed baseline + run: python3 scripts/run_codegen_comparison.py --config ${{ matrix.codegen_config }} diff --git a/Cargo.toml b/Cargo.toml index e187312..5c9ab82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/jcfangc/bit-string" license = "MIT OR Apache-2.0" keywords = ["bitstring", "bits", "bitset", "sequence"] categories = ["data-structures", "no-std"] -exclude = ["/benches", "/src/**/tests_for_*", "/tests", ".github/"] +exclude = ["/benches", "/scripts", "/src/**/tests_for_*", "/tests", ".github/"] [workspace] members = ["derive"] diff --git a/scripts/bench_config.py b/scripts/bench_config.py index 39a6a9b..17c09a0 100644 --- a/scripts/bench_config.py +++ b/scripts/bench_config.py @@ -1,4 +1,11 @@ """Shared constants for reproducible baseline/codegen benchmark runs.""" BASELINE_SHA = "c15fa1c254ea5b2bbabfe5f008d41f91e8297b89" -RUSTFLAGS = "-C target-cpu=native" +BENCH_RUSTFLAGS = "-C target-cpu=native" + +CODEGEN_CONFIGS = { + "x86_64-baseline": ("x86_64", "-C target-cpu=x86-64"), + "x86_64-sse41": ("x86_64", "-C target-cpu=x86-64 -C target-feature=+sse4.1"), + "x86_64-avx2": ("x86_64", "-C target-cpu=x86-64 -C target-feature=+avx2"), + "aarch64-neon": ("aarch64", "-C target-cpu=generic -C target-feature=+neon"), +} diff --git a/scripts/codegen_harness.rs b/scripts/codegen_harness.rs new file mode 100644 index 0000000..6390196 --- /dev/null +++ b/scripts/codegen_harness.rs @@ -0,0 +1,108 @@ +#![no_std] + +use bit_string::BitString; + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_from_bool_slice(values: *const bool, len: usize) -> BitString { + let values = unsafe { core::slice::from_raw_parts(values, len) }; + BitString::from(values) +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_from_str(value: *const u8, len: usize) -> BitString { + let bytes = unsafe { core::slice::from_raw_parts(value, len) }; + BitString::try_from(unsafe { core::str::from_utf8_unchecked(bytes) }).unwrap() +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_xor(lhs: *const BitString, rhs: *const BitString) -> BitString { + unsafe { (&*lhs).xor(&*rhs).unwrap() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_not(value: *const BitString) -> BitString { + unsafe { (&*value).not() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_shl(value: *const BitString, amount: usize) -> BitString { + unsafe { (&*value).shl(amount) } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_shr(value: *const BitString, amount: usize) -> BitString { + unsafe { (&*value).shr(amount) } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_slice_from(value: *const BitString, start: usize) -> BitString { + unsafe { (&*value).slice_from(start) } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_matches_at( + lhs: *const BitString, + index: usize, + rhs: *const BitString, +) -> bool { + unsafe { (&*lhs).matches_at_string(index, &*rhs) } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_contains(lhs: *const BitString, rhs: *const BitString) -> bool { + unsafe { (&*lhs).contains_string(&*rhs) } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_find( + lhs: *const BitString, + rhs: *const BitString, +) -> Option { + unsafe { (&*lhs).find_string(&*rhs) } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_rfind( + lhs: *const BitString, + rhs: *const BitString, +) -> Option { + unsafe { (&*lhs).rfind_string(&*rhs) } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_cmp( + lhs: *const BitString, + rhs: *const BitString, +) -> core::cmp::Ordering { + unsafe { (&*lhs).cmp_string(&*rhs) } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_count_ones(value: *const BitString) -> usize { + unsafe { (&*value).count_ones() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_leading_zeros(value: *const BitString) -> usize { + unsafe { (&*value).leading_zeros() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_trailing_zeros(value: *const BitString) -> usize { + unsafe { (&*value).trailing_zeros() } +} diff --git a/scripts/codegen_roots.toml b/scripts/codegen_roots.toml new file mode 100644 index 0000000..b9348db --- /dev/null +++ b/scripts/codegen_roots.toml @@ -0,0 +1,89 @@ +[[root]] +symbol = "codegen_bit_string_from_bool_slice" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_from_str" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_xor" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_not" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_shl" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_shr" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_slice_from" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_matches_at" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_contains" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_find" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_rfind" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_cmp" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_count_ones" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_leading_zeros" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_trailing_zeros" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_not_core::avx2::words" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::avx2::eq_words" +artifact = "library" +targets = ["x86_64-avx2"] diff --git a/scripts/codegen_roots.txt b/scripts/codegen_roots.txt deleted file mode 100644 index 7d4d1b0..0000000 --- a/scripts/codegen_roots.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Stable backend roots. Public wrappers may inline these, so backend symbols are -# used where they are emitted as standalone functions in the release rlib. -bit_string::traits::words_scan::funcs_for_count_ones::scalar::count_words -bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words -bit_string::traits::words_scan::funcs_for_count_ones::ssse3::count_words -bit_string::traits::words_scan::funcs_for_count_ones::neon::count_words -bit_string::traits::words_arith::funcs_for_binary_core::scalar::words -bit_string::traits::words_arith::funcs_for_binary_core::sse2::words -bit_string::traits::words_arith::funcs_for_binary_core::avx2::words -bit_string::traits::words_arith::funcs_for_binary_core::neon::words -bit_string::traits::words_arith::funcs_for_not_core::scalar::words -bit_string::traits::words_arith::funcs_for_not_core::sse2::words -bit_string::traits::words_arith::funcs_for_not_core::avx2::words -bit_string::traits::words_arith::funcs_for_not_core::neon::words -bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::scalar::eq_words -bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::avx2::eq_words -bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::sse41::eq_words -bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::neon::eq_words diff --git a/scripts/compare_bench.py b/scripts/compare_bench.py index a07a78d..8cbdd75 100755 --- a/scripts/compare_bench.py +++ b/scripts/compare_bench.py @@ -1,40 +1,27 @@ #!/usr/bin/env python3 -"""Compare Divan text reports using each benchmark's median time.""" +"""Compare repeated Divan reports using the median of per-run medians.""" from __future__ import annotations import argparse -import re import statistics import sys from pathlib import Path -TIME = re.compile(r"(?P[0-9]+(?:\.[0-9]+)?)\s*(?Pns|µs|us|ms|s)") -NAME = re.compile(r"^[├╰]─\s*(?P.+?)(?:\s{2,}|\s+Timer precision:)") -UNITS_TO_NS = {"ns": 1.0, "µs": 1_000.0, "us": 1_000.0, "ms": 1_000_000.0, "s": 1_000_000_000.0} +from divan_parser import parse_report -def parse_report(path: Path) -> dict[str, float]: - """Parse Divan's columnar text report, returning median values in ns.""" - result: dict[str, float] = {} - pending: str | None = None - for raw in path.read_text(encoding="utf-8").splitlines(): - match = NAME.match(raw) - if match: - pending = match.group("name").strip() - if pending is None: - continue - values = list(TIME.finditer(raw)) - # Data rows contain fastest, slowest, median, and mean in that order. - if len(values) >= 4: - median = values[2] - result[pending] = float(median.group("value")) * UNITS_TO_NS[median.group("unit")] - pending = None - elif raw.strip() and not raw.startswith((" ", "\t")): - pending = None - if not result: - raise ValueError(f"no Divan benchmark medians found in {path}") - return result +def aggregate(paths: list[Path], expected_runs: int) -> dict[str, float]: + if len(paths) != expected_runs: + raise ValueError(f"expected {expected_runs} reports, got {len(paths)}") + reports = [parse_report(path) for path in paths] + expected_names = set(reports[0]) + for path, report in zip(paths[1:], reports[1:], strict=True): + if set(report) != expected_names: + missing = sorted(expected_names - set(report)) + extra = sorted(set(report) - expected_names) + raise ValueError(f"{path}: inconsistent benchmark set; missing={missing}, extra={extra}") + return {name: statistics.median(report[name].median_ns for report in reports) for name in expected_names} def fmt_ns(value: float) -> str: @@ -44,6 +31,8 @@ def fmt_ns(value: float) -> str: return f"{value / 1_000_000:.3f} ms" if value >= 1_000: return f"{value / 1_000:.3f} µs" + if value < 1: + return f"{value * 1_000:.3f} ps" return f"{value:.3f} ns" @@ -51,43 +40,39 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("baseline", type=Path, nargs="+", help="baseline Divan text reports") parser.add_argument("--current", type=Path, nargs="+", required=True, help="current Divan text reports") + parser.add_argument("--expected-runs", type=int, default=5) parser.add_argument("--warning-threshold", type=float, default=5.0) args = parser.parse_args() - - def aggregate(paths: list[Path]) -> dict[str, float]: - samples: dict[str, list[float]] = {} - for path in paths: - for name, value in parse_report(path).items(): - samples.setdefault(name, []).append(value) - return {name: statistics.median(values) for name, values in samples.items()} - - baseline, current = aggregate(args.baseline), aggregate(args.current) - common = sorted(baseline.keys() & current.keys()) + try: + baseline = aggregate(args.baseline, args.expected_runs) + current = aggregate(args.current, args.expected_runs) + except ValueError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 2 only_baseline = sorted(baseline.keys() - current.keys()) only_current = sorted(current.keys() - baseline.keys()) + if only_baseline or only_current: + for name in only_baseline: + print(f"ERROR: missing in current: {name}", file=sys.stderr) + for name in only_current: + print(f"ERROR: missing in baseline: {name}", file=sys.stderr) + return 2 improved = unchanged = regressed = warnings = 0 print(f"{'benchmark':<50} {'baseline':>12} {'current':>12} {'delta':>10} status") print("-" * 96) - for name in common: + for name in sorted(baseline): before, after = baseline[name], current[name] delta = (after - before) / before * 100 - if delta < 0: - improved += 1 - elif delta > 0: - regressed += 1 - else: - unchanged += 1 + improved += delta < 0 + unchanged += delta == 0 + regressed += delta > 0 status = "WARNING" if delta > args.warning_threshold else "OK" if status == "WARNING": warnings += 1 print(f"WARNING: {name} regressed by {delta:+.2f}%", file=sys.stderr) print(f"{name:<50.50} {fmt_ns(before):>12} {fmt_ns(after):>12} {delta:>+9.2f}% {status}") - for name in only_baseline: - print(f"missing in current: {name}", file=sys.stderr) - for name in only_current: - print(f"missing in baseline: {name}", file=sys.stderr) print("\nSUMMARY") - print(f"matched benchmarks : {len(common)}") + print(f"matched benchmarks : {len(baseline)}") print(f"improved : {improved}") print(f"unchanged : {unchanged}") print(f"regressed : {regressed}") diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index ebf5e39..26136ba 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -1,27 +1,40 @@ #!/usr/bin/env python3 -"""Compare normalized release-rlib disassembly for selected codegen roots.""" +"""Compare normalized disassembly for explicitly configured codegen roots.""" from __future__ import annotations import argparse +import difflib import hashlib import re import subprocess import sys -from dataclasses import dataclass +import tomllib +from dataclasses import dataclass, field from pathlib import Path +from bench_config import CODEGEN_CONFIGS + FUNCTION = re.compile(r"^\s*[0-9a-f]+ <(.+)>:\s*$") -INSTRUCTION = re.compile(r"^\s*[0-9a-f]+:\s+(?:[0-9a-f]{2}(?:\s+[0-9a-f]{2})*\s+)?(.+?)\s*$") -TARGET_INSTRUCTION = re.compile(r"^(?:j[a-z]+|call|loop[a-z]*)(?:\s|$)") -CONDITIONAL_BRANCH = re.compile(r"^(?:ja|jae|jb|jbe|jc|je|jg|jge|jl|jle|jna|jnae|jnb|jnbe|jnc|jne|jng|jnge|jnl|jnle|jno|jnp|jns|jnz|jo|jp|jpe|jpo|js|jz)(?:\s|$)") -TARGET = re.compile(r"(?:0x[0-9a-f]+|[0-9a-f]+)\s+<[^>]+>") +INSTRUCTION = re.compile(r"^\s*([0-9a-f]+):\s+((?:[0-9a-f]{2}(?:\s+|$))+)(.*?)\s*$") +RELOCATION = re.compile(r"^\s*([0-9a-f]+):\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") +SYMBOL_TARGET = re.compile(r"(?:0x)?([0-9a-f]+)\s+<([^>]+)>") +RELOCATION_ADDEND = re.compile(r"[+-]0x[0-9a-f]+$") +OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") + + +@dataclass +class Instruction: + address: int + encoded: bytes + assembly: str + relocations: list[tuple[str, str]] = field(default_factory=list) @dataclass(frozen=True) class Function: instructions: tuple[str, ...] - bytes: int + encoded: bytes calls: int branches: int @@ -30,100 +43,165 @@ def digest(self) -> str: return hashlib.sha256("\n".join(self.instructions).encode()).hexdigest()[:16] -def normalized_instruction(text: str) -> str: - text = text.split("#", 1)[0].strip() - if not text: - return "" - if TARGET_INSTRUCTION.match(text): - text = TARGET.sub("", text) - return " ".join(text.split()) +@dataclass(frozen=True) +class Root: + symbol: str + artifact: str + targets: frozenset[str] -def load_functions(artifact: Path, objdump: str) -> dict[str, Function]: - output = subprocess.check_output([objdump, "-drwC", str(artifact)], text=True, errors="replace") - found: dict[str, list[str]] = {} - sizes: dict[str, int] = {} +def classify(old: Function, new: Function) -> str: + if old.instructions != new.instructions: + return "CODEGEN CHANGED" + if len(old.encoded) != len(new.encoded): + return "CODEGEN ENCODING CHANGED" + return "CODEGEN IDENTICAL" + + +def _relocation_target(value: str) -> str: + return RELOCATION_ADDEND.sub("", value.strip()) + + +def _is_call(mnemonic: str, arch: str) -> bool: + return mnemonic.startswith("call") if arch == "x86_64" else mnemonic in {"bl", "blr"} + + +def _is_conditional_branch(mnemonic: str, arch: str) -> bool: + if arch == "x86_64": + return mnemonic.startswith("j") and mnemonic not in {"jmp", "jmpq"} + return mnemonic.startswith("b.") or mnemonic in {"cbz", "cbnz", "tbz", "tbnz"} + + +def _has_control_target(mnemonic: str, arch: str) -> bool: + if arch == "x86_64": + return mnemonic.startswith(("j", "call", "loop")) + return mnemonic in {"b", "bl", "blr", "br", "cbz", "cbnz", "tbz", "tbnz"} or mnemonic.startswith("b.") + + +def _normalize(instructions: list[Instruction], arch: str) -> Function: + indexes = {instruction.address: index for index, instruction in enumerate(instructions)} + normalized: list[str] = [] + calls = branches = 0 + for instruction in instructions: + assembly = OBJDUMP_COMMENT.sub("", instruction.assembly).strip() + if not assembly: + continue + parts = assembly.split(None, 1) + mnemonic = parts[0] + operands = parts[1] if len(parts) == 2 else "" + calls += _is_call(mnemonic, arch) + branches += _is_conditional_branch(mnemonic, arch) + if _has_control_target(mnemonic, arch): + match = SYMBOL_TARGET.search(operands) + if match: + address = int(match.group(1), 16) + symbol = match.group(2) + target = f"@instruction_{indexes[address]}" if address in indexes and "+0x" in symbol else f"<{symbol}>" + operands = SYMBOL_TARGET.sub(target, operands) + relocations = " ".join(f"[{kind}:{_relocation_target(target)}]" for kind, target in instruction.relocations) + normalized.append(" ".join(f"{mnemonic} {operands} {relocations}".split())) + return Function(tuple(normalized), b"".join(item.encoded for item in instructions), calls, branches) + + +def parse_objdump(output: str, arch: str) -> dict[str, Function]: + raw: dict[str, list[Instruction]] = {} current: str | None = None + last_instruction: Instruction | None = None for line in output.splitlines(): label = FUNCTION.match(line) if label: current = label.group(1) - found.setdefault(current, []) - sizes.setdefault(current, 0) + if current in raw: + raise ValueError(f"duplicate disassembly symbol: {current}") + raw[current] = [] + last_instruction = None continue if current is None: continue + relocation = RELOCATION.match(line) + if relocation: + if last_instruction is not None: + last_instruction.relocations.append((relocation.group(2), relocation.group(3))) + continue match = INSTRUCTION.match(line) if not match: continue - raw = match.group(1) - normalized = normalized_instruction(raw) - if not normalized: + encoded = bytes.fromhex(match.group(2)) + assembly = match.group(3).strip() + if not assembly: continue - found[current].append(normalized) - # The byte column is the sequence before the mnemonic. Count bytes from - # the source line rather than estimating from the mnemonic. - prefix = line.split(":", 1)[1].split(raw, 1)[0] - sizes[current] += len(re.findall(r"\b[0-9a-f]{2}\b", prefix)) - result: dict[str, Function] = {} - for name, instructions in found.items(): - if not instructions: - continue - calls = sum(1 for item in instructions if item.startswith("call ")) - branches = sum(1 for item in instructions if CONDITIONAL_BRANCH.match(item)) - result[name] = Function(tuple(instructions), sizes[name], calls, branches) - return result + last_instruction = Instruction(int(match.group(1), 16), encoded, assembly) + raw[current].append(last_instruction) + return {name: _normalize(items, arch) for name, items in raw.items() if items} + +def load_functions(artifact: Path, objdump: str, arch: str) -> dict[str, Function]: + output = subprocess.check_output([objdump, "-drwC", str(artifact)], text=True, errors="replace") + return parse_objdump(output, arch) -def symbols(path: Path) -> list[str]: - return [line.strip() for line in path.read_text().splitlines() if line.strip() and not line.lstrip().startswith("#")] + +def load_roots(path: Path) -> list[Root]: + data = tomllib.loads(path.read_text(encoding="utf-8")) + roots = [Root(item["symbol"], item["artifact"], frozenset(item["targets"])) for item in data.get("root", [])] + identities = [(root.artifact, root.symbol) for root in roots] + if len(identities) != len(set(identities)): + raise ValueError(f"{path}: duplicate codegen root") + unknown_targets = set().union(*(root.targets for root in roots)) - CODEGEN_CONFIGS.keys() + if unknown_targets: + raise ValueError(f"{path}: unknown target configurations: {sorted(unknown_targets)}") + return roots def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("baseline", type=Path) parser.add_argument("current", type=Path) - parser.add_argument("--symbols", type=Path, required=True) + parser.add_argument("--roots", type=Path, required=True) + parser.add_argument("--artifact", required=True) + parser.add_argument("--target", required=True) + parser.add_argument("--arch", choices=("x86_64", "aarch64"), required=True) parser.add_argument("--objdump", default="objdump") args = parser.parse_args() - before, after = load_functions(args.baseline, args.objdump), load_functions(args.current, args.objdump) - changed = missing = unavailable = 0 - names = symbols(args.symbols) - print(f"{'symbol':<78} {'bytes':>13} {'insns':>13} {'calls':>11} {'branches':>14} status") - print("-" * 140) - for name in names: - old, new = before.get(name), after.get(name) + before = load_functions(args.baseline, args.objdump, args.arch) + after = load_functions(args.current, args.objdump, args.arch) + roots = [root for root in load_roots(args.roots) if root.artifact == args.artifact] + identical = encoding_changed = changed = errors = skipped = 0 + for root in roots: + if args.target not in root.targets: + skipped += 1 + continue + old, new = before.get(root.symbol), after.get(root.symbol) if old is None or new is None: - if old is None and new is None: - unavailable += 1 - status = "UNAVAILABLE" - else: - missing += 1 - status = "MISSING" - print(f"{name:<78} {'-':>6} -> {'-':<6} {'-':>6} -> {'-':<6} {'-':>5} -> {'-':<5} {'-':>7} -> {'-':<7} {status}") + errors += 1 + where = "both" if old is None and new is None else "baseline" if old is None else "current" + print(f"ERROR: expected symbol missing in {where}: {root.symbol}", file=sys.stderr) continue - identical = old.instructions == new.instructions - if not identical: + same_instructions = old.instructions == new.instructions + status = classify(old, new) + if status == "CODEGEN IDENTICAL": + identical += 1 + elif status == "CODEGEN ENCODING CHANGED": + encoding_changed += 1 + else: changed += 1 - status = "CODEGEN IDENTICAL" if identical else "CODEGEN CHANGED" - print(f"{name:<78} {old.bytes:>6} -> {new.bytes:<6} {len(old.instructions):>6} -> {len(new.instructions):<6} {old.calls:>5} -> {new.calls:<5} {old.branches:>7} -> {new.branches:<7} {old.digest} -> {new.digest} {status}") + print(f"{root.symbol}: {status}") + print(f" text bytes {len(old.encoded)} -> {len(new.encoded)}") + print(f" instructions {len(old.instructions)} -> {len(new.instructions)}") + print(f" calls {old.calls} -> {new.calls}") + print(f" branches {old.branches} -> {new.branches}") + print(f" asm hash {old.digest} -> {new.digest}") if new.calls > old.calls: - print(f"WARNING: {name} call count increased from {old.calls} to {new.calls}", file=sys.stderr) - if not identical: - print(f"--- baseline {name}\n+++ current {name}") - for left, right in zip(old.instructions, new.instructions): - if left != right: - print(f"- {left}\n+ {right}") - break - if len(old.instructions) != len(new.instructions): - print(f"instruction count: {len(old.instructions)} -> {len(new.instructions)}") + print(f"WARNING: {root.symbol} call count increased from {old.calls} to {new.calls}", file=sys.stderr) + if not same_instructions: + print("\n".join(difflib.unified_diff(old.instructions, new.instructions, "baseline", "current", lineterm=""))) print("\nCODEGEN SUMMARY") - print(f"compared : {len(names) - missing - unavailable}") - print(f"identical : {len(names) - missing - unavailable - changed}") - print(f"changed : {changed}") - print(f"missing : {missing}") - print(f"unavailable : {unavailable}") - return 0 if not changed and not missing else 1 + print(f"compared : {identical + encoding_changed + changed}") + print(f"identical : {identical}") + print(f"encoding changed : {encoding_changed}") + print(f"changed : {changed}") + print(f"errors : {errors}") + print(f"skipped : {skipped}") + return 0 if not (encoding_changed or changed or errors) else 1 if __name__ == "__main__": diff --git a/scripts/divan_parser.py b/scripts/divan_parser.py new file mode 100644 index 0000000..2915991 --- /dev/null +++ b/scripts/divan_parser.py @@ -0,0 +1,54 @@ +"""Strict parser for Divan's text table format.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +TIME = re.compile(r"(?P[0-9]+(?:\.[0-9]+)?)\s*(?Pps|ns|µs|us|ms|s)\b") +ROW = re.compile(r"^[│\s]*[├╰]─\s*(?P.+)$") +COUNTS = re.compile(r"│\s*(?P[0-9]+)\s*│\s*(?P[0-9]+)\s*$") +UNITS_TO_NS = {"ps": 1e-3, "ns": 1.0, "µs": 1_000.0, "us": 1_000.0, "ms": 1_000_000.0, "s": 1_000_000_000.0} + + +@dataclass(frozen=True) +class DivanResult: + fastest_ns: float + slowest_ns: float + median_ns: float + mean_ns: float + samples: int + iters: int + + +def _time_ns(match: re.Match[str]) -> float: + return float(match.group("value")) * UNITS_TO_NS[match.group("unit")] + + +def parse_report(path: Path) -> dict[str, DivanResult]: + result: dict[str, DivanResult] = {} + pending: str | None = None + for raw in path.read_text(encoding="utf-8").splitlines(): + row = ROW.match(raw) + if row: + body = row.group("body") + first_time = TIME.search(body) + end = first_time.start() if first_time else body.find("Timer precision:") + pending = body[: end if end >= 0 else None].rstrip() + if pending is None: + continue + times = list(TIME.finditer(raw)) + if len(times) < 4: + continue + counts = COUNTS.search(raw) + if counts is None: + raise ValueError(f"{path}: benchmark row for {pending!r} has no samples/iters") + if pending in result: + raise ValueError(f"{path}: duplicate benchmark name {pending!r}") + values = [_time_ns(match) for match in times[:4]] + result[pending] = DivanResult(*values, int(counts.group("samples")), int(counts.group("iters"))) + pending = None + if not result: + raise ValueError(f"{path}: no Divan benchmark rows found") + return result diff --git a/scripts/kernel_inventory.toml b/scripts/kernel_inventory.toml new file mode 100644 index 0000000..2aff064 --- /dev/null +++ b/scripts/kernel_inventory.toml @@ -0,0 +1,67 @@ +[[kernel]] +path = "src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs" +roots = ["codegen_bit_string_from_bool_slice"] + +[[kernel]] +path = "src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs" +roots = ["codegen_bit_string_from_str"] + +[[kernel]] +path = "src/traits/words_arith/funcs_for_binary_core.rs" +roots = ["codegen_bit_string_xor"] + +[[kernel]] +path = "src/traits/words_arith/funcs_for_not_core.rs" +roots = ["codegen_bit_string_not"] + +[[kernel]] +path = "src/traits/words_arith/funcs_for_shl_core.rs" +roots = ["codegen_bit_string_shl"] + +[[kernel]] +path = "src/traits/words_arith/funcs_for_shr_core.rs" +roots = ["codegen_bit_string_shr"] + +[[kernel]] +path = "src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs" +roots = ["codegen_bit_string_slice_from"] + +[[kernel]] +path = "src/traits/words_eq/funcs_for_eq_words_aligned_core.rs" +roots = ["codegen_bit_string_matches_at"] + +[[kernel]] +path = "src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs" +roots = ["codegen_bit_string_matches_at"] + +[[kernel]] +path = "src/traits/words_find/funcs_for_contains_core.rs" +roots = ["codegen_bit_string_contains"] + +[[kernel]] +path = "src/traits/words_find/funcs_for_find_core.rs" +roots = ["codegen_bit_string_find"] + +[[kernel]] +path = "src/traits/words_find/funcs_for_rfind_core.rs" +roots = ["codegen_bit_string_rfind"] + +[[kernel]] +path = "src/traits/words_ord/funcs_for_cmp_aligned_core.rs" +roots = ["codegen_bit_string_cmp"] + +[[kernel]] +path = "src/traits/words_ord/funcs_for_cmp_unaligned_core.rs" +roots = ["codegen_bit_string_cmp"] + +[[kernel]] +path = "src/traits/words_scan/funcs_for_count_ones.rs" +roots = ["codegen_bit_string_count_ones"] + +[[kernel]] +path = "src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs" +roots = ["codegen_bit_string_leading_zeros"] + +[[kernel]] +path = "src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs" +roots = ["codegen_bit_string_trailing_zeros"] diff --git a/scripts/run_bench_comparison.py b/scripts/run_bench_comparison.py index 19e3774..3c65635 100755 --- a/scripts/run_bench_comparison.py +++ b/scripts/run_bench_comparison.py @@ -9,7 +9,7 @@ import sys from pathlib import Path -from bench_config import BASELINE_SHA, RUSTFLAGS +from bench_config import BASELINE_SHA, BENCH_RUSTFLAGS ORDER = ("baseline", "current", "current", "baseline", "baseline", "current", "current", "baseline", "baseline", "current") @@ -42,7 +42,7 @@ def main() -> int: run(["git", "-C", str(baseline_dir), "checkout", "--detach", args.baseline], cwd=root) env = os.environ.copy() - env["RUSTFLAGS"] = RUSTFLAGS + env["RUSTFLAGS"] = BENCH_RUSTFLAGS command = ["cargo", "bench", "--", "--color", "never"] counts = {"baseline": 0, "current": 0} for revision in ORDER: @@ -54,6 +54,7 @@ def main() -> int: compare = [sys.executable, str(root / "scripts" / "compare_bench.py")] compare += [str(output_dir / f"baseline-{i}.txt") for i in range(1, 6)] compare += ["--current"] + [str(output_dir / f"current-{i}.txt") for i in range(1, 6)] + compare += ["--expected-runs", str(args.runs)] compare += ["--warning-threshold", str(args.warning_threshold)] return subprocess.run(compare, cwd=root, check=False).returncode diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py index 9c4c0f0..d6ce6e5 100755 --- a/scripts/run_codegen_comparison.py +++ b/scripts/run_codegen_comparison.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 -"""Build the fixed baseline and current worktree, then compare release codegen.""" +"""Build fixed-feature baseline/current artifacts and compare their codegen.""" from __future__ import annotations import argparse import os +import platform +import shlex import subprocess import sys from pathlib import Path -from bench_config import BASELINE_SHA, RUSTFLAGS +from bench_config import BASELINE_SHA, CODEGEN_CONFIGS def run(command: list[str], cwd: Path, env: dict[str, str]) -> None: @@ -19,27 +21,85 @@ def run(command: list[str], cwd: Path, env: dict[str, str]) -> None: raise SystemExit(completed.returncode) +def build_library(worktree: Path, target_dir: Path, rustflags: str) -> Path: + env = os.environ.copy() + env["RUSTFLAGS"] = rustflags + env["CARGO_TARGET_DIR"] = str(target_dir) + run(["cargo", "rustc", "--release", "--lib", "--", "--emit=obj"], worktree, env) + return target_dir / "release" / "libbit_string.rlib" + + +def build_harness(root: Path, library: Path, output: Path, rustflags: str) -> None: + dependency_dir = library.parent / "deps" + command = [ + "rustc", + str(root / "scripts" / "codegen_harness.rs"), + "--crate-name", + "bit_string_codegen_harness", + "--crate-type=lib", + "--edition=2024", + "-O", + "--emit=obj", + "-o", + str(output), + "--extern", + f"bit_string={library}", + "-L", + f"dependency={dependency_dir}", + *shlex.split(rustflags), + ] + run(command, root, os.environ.copy()) + + +def compare(root: Path, baseline: Path, current: Path, artifact: str, target: str, arch: str) -> int: + command = [ + sys.executable, + str(root / "scripts" / "compare_codegen.py"), + str(baseline), + str(current), + "--roots", + str(root / "scripts" / "codegen_roots.toml"), + "--artifact", + artifact, + "--target", + target, + "--arch", + arch, + ] + return subprocess.run(command, cwd=root, check=False).returncode + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--baseline", default=BASELINE_SHA) - parser.add_argument("--symbols", type=Path, default=None) + parser.add_argument("--config", choices=sorted(CODEGEN_CONFIGS), action="append") args = parser.parse_args() root = Path(__file__).resolve().parents[1] output = root / "target" / "codegen-comparison" - baseline = output / "baseline-worktree" + baseline_worktree = output / "baseline-worktree" output.mkdir(parents=True, exist_ok=True) - if not baseline.exists(): - run(["git", "worktree", "add", "--detach", str(baseline), args.baseline], root, os.environ.copy()) + if not baseline_worktree.exists(): + run(["git", "worktree", "add", "--detach", str(baseline_worktree), args.baseline], root, os.environ.copy()) else: - run(["git", "-C", str(baseline), "checkout", "--detach", args.baseline], root, os.environ.copy()) - env = os.environ.copy() - env["RUSTFLAGS"] = RUSTFLAGS - command = ["cargo", "rustc", "--release", "--lib", "--", "--emit=obj"] - run(command, baseline, env) - run(command, root, env) - symbols = args.symbols or (root / "scripts" / "codegen_roots.txt") - compare = [sys.executable, str(root / "scripts" / "compare_codegen.py"), str(baseline / "target" / "release" / "libbit_string.rlib"), str(root / "target" / "release" / "libbit_string.rlib"), "--symbols", str(symbols)] - return subprocess.run(compare, cwd=root, env=env, check=False).returncode + run(["git", "-C", str(baseline_worktree), "checkout", "--detach", args.baseline], root, os.environ.copy()) + host_arch = "aarch64" if platform.machine() in {"aarch64", "arm64"} else "x86_64" + configs = args.config or [name for name, (arch, _) in CODEGEN_CONFIGS.items() if arch == host_arch] + failed = False + for config in configs: + arch, rustflags = CODEGEN_CONFIGS[config] + if arch != host_arch: + print(f"ERROR: {config} requires {arch}, but host architecture is {host_arch}", file=sys.stderr) + return 2 + config_dir = output / config + baseline_library = build_library(baseline_worktree, config_dir / "baseline-target", rustflags) + current_library = build_library(root, config_dir / "current-target", rustflags) + baseline_harness = config_dir / "baseline-harness.o" + current_harness = config_dir / "current-harness.o" + build_harness(root, baseline_library, baseline_harness, rustflags) + build_harness(root, current_library, current_harness, rustflags) + failed |= compare(root, baseline_harness, current_harness, "harness", config, arch) != 0 + failed |= compare(root, baseline_library, current_library, "library", config, arch) != 0 + return int(failed) if __name__ == "__main__": diff --git a/scripts/tests/test_bench.py b/scripts/tests/test_bench.py new file mode 100644 index 0000000..7b091c9 --- /dev/null +++ b/scripts/tests/test_bench.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import contextlib +import io +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import compare_bench +from divan_parser import parse_report + +ROW = "├─ bench 500 ps │ 1 ns │ 750 ps │ 800 ps │ 100 │ 1000\n" + + +class DivanParserTests(unittest.TestCase): + def write(self, directory: str, name: str, content: str = ROW) -> Path: + path = Path(directory) / name + path.write_text(content) + return path + + def test_picoseconds_are_converted_to_ns(self) -> None: + with tempfile.TemporaryDirectory() as directory: + result = parse_report(self.write(directory, "report.txt"))["bench"] + self.assertEqual(result.median_ns, 0.75) + self.assertEqual(result.samples, 100) + self.assertEqual(result.iters, 1000) + + def test_duplicate_benchmark_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory, self.assertRaisesRegex(ValueError, "duplicate"): + parse_report(self.write(directory, "report.txt", ROW + ROW)) + + def test_expected_run_count_is_enforced(self) -> None: + with tempfile.TemporaryDirectory() as directory: + paths = [self.write(directory, f"report-{index}.txt") for index in range(4)] + with self.assertRaisesRegex(ValueError, "expected 5 reports"): + compare_bench.aggregate(paths, 5) + + def test_missing_benchmark_in_one_run_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + paths = [self.write(directory, f"report-{index}.txt") for index in range(5)] + paths[-1].write_text(ROW.replace("bench", "different")) + with self.assertRaisesRegex(ValueError, "inconsistent benchmark set"): + compare_bench.aggregate(paths, 5) + + def test_missing_between_revisions_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + baseline = [self.write(directory, f"baseline-{index}.txt") for index in range(5)] + current = [self.write(directory, f"current-{index}.txt", ROW.replace("bench", "other")) for index in range(5)] + argv = [ + "compare_bench.py", + *(str(path) for path in baseline), + "--current", + *(str(path) for path in current), + ] + with ( + mock.patch.object(sys, "argv", argv), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertNotEqual(compare_bench.main(), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py new file mode 100644 index 0000000..fed4036 --- /dev/null +++ b/scripts/tests/test_codegen.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import contextlib +import io +import sys +import tempfile +import tomllib +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import compare_codegen + + +def parse(body: str, arch: str = "x86_64") -> compare_codegen.Function: + return compare_codegen.parse_objdump(body, arch)["root"] + + +class CodegenParserTests(unittest.TestCase): + def test_call_target_identity_is_preserved(self) -> None: + foo = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 foo-0x4") + bar = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 bar-0x4") + self.assertNotEqual(foo.instructions, bar.instructions) + + def test_local_branch_uses_target_instruction_index(self) -> None: + old = parse("0000 :\n 0: 74 02 je 4 \n 2: 90 nop\n 4: c3 ret") + relocated = parse("0000 :\n 0: 74 08 je a \n 2: 66 90 xchg %ax,%ax\n a: c3 ret") + different = parse("0000 :\n 0: 74 00 je 2 \n 2: 90 nop\n 4: c3 ret") + self.assertEqual(old.instructions[0], relocated.instructions[0]) + self.assertNotEqual(old.instructions[0], different.instructions[0]) + + def test_relocation_is_not_an_instruction(self) -> None: + function = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 foo-0x4\n 5: c3 ret") + self.assertEqual(len(function.instructions), 2) + self.assertEqual(function.instructions[0], "call [R_X86_64_PLT32:foo]") + + def test_duplicate_disassembly_symbol_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "duplicate disassembly symbol"): + compare_codegen.parse_objdump("0000 :\n 0: c3 ret\n0000 :\n 0: c3 ret", "x86_64") + + def test_encoding_size_change_is_not_identical(self) -> None: + old = parse("0000 :\n 0: 90 nop\n 1: c3 ret") + new = parse("0000 :\n 0: 66 90 nop\n 2: c3 ret") + self.assertEqual(old.instructions, new.instructions) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN ENCODING CHANGED") + + def test_aarch64_calls_and_branches(self) -> None: + function = parse( + "0000 :\n 0: 00 00 00 94 bl 0 \n 4: 40 00 00 54 b.eq c \n 8: 00 04 00 91 add x0, x0, #0x1\n c: c0 03 5f d6 ret", + "aarch64", + ) + self.assertEqual(function.calls, 1) + self.assertEqual(function.branches, 1) + self.assertIn("@instruction_3", function.instructions[1]) + self.assertIn("#0x1", function.instructions[2]) + + def test_expected_missing_symbol_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + roots = Path(directory) / "roots.toml" + roots.write_text('[[root]]\nsymbol="expected"\nartifact="harness"\ntargets=["x86_64-avx2"]\n') + argv = [ + "compare_codegen.py", + "baseline.o", + "current.o", + "--roots", + str(roots), + "--artifact", + "harness", + "--target", + "x86_64-avx2", + "--arch", + "x86_64", + ] + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object(compare_codegen, "load_functions", return_value={}), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertNotEqual(compare_codegen.main(), 0) + + def test_inventory_has_root_for_each_kernel(self) -> None: + scripts = Path(__file__).resolve().parents[1] + roots = {root.symbol for root in compare_codegen.load_roots(scripts / "codegen_roots.toml")} + inventory = tomllib.loads((scripts / "kernel_inventory.toml").read_text())["kernel"] + self.assertEqual(len(inventory), 17) + for kernel in inventory: + self.assertTrue(kernel["roots"]) + self.assertLessEqual(set(kernel["roots"]), roots) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs index fdce4db..845e095 100644 --- a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs +++ b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs @@ -11,37 +11,47 @@ impl BitString { if bit_len == 0 { return 0; } + // SAFETY: `words` is always non-empty when bit_len > 0 + // (BitString invariants guarantee at least one word). let words_ptr = self.words.as_ptr(); - // SAFETY: `bit_len > 0`, so the first backing word exists. + + // ── First-word fast path ────────────────────────────────── + // SAFETY: `bit_len > 0` (guarded above), so `words` has at least + // one element per the BitString invariant. let w0 = unsafe { *words_ptr }; if w0 != 0 { return (w0.trailing_zeros() as usize).min(bit_len); } + + // One- and two-word strings are common enough to avoid the generic scan. if bit_len <= WORD_BITS { return bit_len; } if bit_len <= WORD_BITS * 2 { let used = bit_len - WORD_BITS; - // SAFETY: bit_len > WORD_BITS, so a second backing word exists. + // SAFETY: bit_len > WORD_BITS, so the second backing word exists. + let w1 = unsafe { *words_ptr.add(1) }; let mask = if used == WORD_BITS { u64::MAX } else { (1u64 << used) - 1 }; - let w1 = unsafe { *words_ptr.add(1) } & mask; - return if w1 == 0 { - bit_len - } else { - WORD_BITS + w1.trailing_zeros() as usize - }; + let w1 = w1 & mask; + if w1 == 0 { + return bit_len; + } + return WORD_BITS + w1.trailing_zeros() as usize; } + + // ── Tiny inputs ─────────────────────────────────── let last_wi = (bit_len - 1) / WORD_BITS; let end_rem = bit_len % WORD_BITS; let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; if mid_end < SMALL_WORDS { - let mut scanned = WORD_BITS; + let mut scanned = WORD_BITS; // word 0 already checked above + // SAFETY: i ranges in 1..mid_end. words contains every + // backing word covered by bit_len. for i in 1..mid_end { - // SAFETY: `i < mid_end`, which is covered by backing storage. let w = unsafe { *words_ptr.add(i) }; if w != 0 { return (scanned + w.trailing_zeros() as usize).min(bit_len); @@ -49,16 +59,17 @@ impl BitString { scanned += WORD_BITS; } if end_rem != 0 { - // SAFETY: a partial final word exists at `mid_end`. + // SAFETY: a partial final word exists at mid_end. let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); - return if last == 0 { - bit_len - } else { - (scanned + last.trailing_zeros() as usize).min(bit_len) - }; + if last == 0 { + return bit_len; + } + return (scanned + last.trailing_zeros() as usize).min(bit_len); } return bit_len; } + + // ── SIMD via trait ──────────────────────────────────────── self.words() .leading_value_bits::(0, bit_len) } @@ -71,18 +82,22 @@ impl BitString { return 0; } let words_ptr = self.words.as_ptr(); - // SAFETY: `bit_len > 0`, so the first backing word exists. + + // SAFETY: `bit_len > 0` (guarded above), so `words` has at least + // one element per the BitString invariant. let w0 = unsafe { *words_ptr }; if w0 != u64::MAX { return ((!w0).trailing_zeros() as usize).min(bit_len); } + let last_wi = (bit_len - 1) / WORD_BITS; let end_rem = bit_len % WORD_BITS; let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; if mid_end < SMALL_WORDS { let mut scanned = WORD_BITS; + // SAFETY: `i < mid_end`. `words` has at least + // `mid_end + (end_rem != 0) as usize` elements. for i in 1..mid_end { - // SAFETY: `i < mid_end`, which is covered by backing storage. let w = unsafe { *words_ptr.add(i) }; if w != u64::MAX { return (scanned + (!w).trailing_zeros() as usize).min(bit_len); @@ -90,17 +105,17 @@ impl BitString { scanned += WORD_BITS; } if end_rem != 0 { - // SAFETY: a partial final word exists at `mid_end`. + // SAFETY: `mid_end` is the index of the last partial word + // and is within bounds (backing storage covers all bits). let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); - let fill = (1u64 << end_rem).wrapping_sub(1); - return if last == fill { - bit_len - } else { - (scanned + (!last).trailing_zeros() as usize).min(bit_len) - }; + if last == ((1u64 << end_rem).wrapping_sub(1)) { + return bit_len; + } + return (scanned + (!last).trailing_zeros() as usize).min(bit_len); } return bit_len; } + self.words() .leading_value_bits::(0, bit_len) } From a70c16b947f98c1afbf83465be2885ea655d8165 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 03:50:27 +0000 Subject: [PATCH 08/79] fix: compare production release codegen --- scripts/codegen_roots.toml | 140 ++++++++++++++++++++++++++++++ scripts/compare_codegen.py | 20 +++-- scripts/kernel_inventory.toml | 20 +++++ scripts/run_codegen_comparison.py | 2 +- scripts/tests/test_codegen.py | 14 ++- 5 files changed, 188 insertions(+), 8 deletions(-) diff --git a/scripts/codegen_roots.toml b/scripts/codegen_roots.toml index b9348db..1a55be2 100644 --- a/scripts/codegen_roots.toml +++ b/scripts/codegen_roots.toml @@ -87,3 +87,143 @@ targets = ["x86_64-avx2"] symbol = "bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::avx2::eq_words" artifact = "library" targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::bit_string::impls_for_construction::funcs_for_pack_bools_core::avx2::words" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_shl_core::avx2::words" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_shr_core::avx2::words" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::traits::words_edit::bits_copied::funcs_for_copy_words_shifted_core::avx2::copy_words_shifted" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::traits::words_eq::funcs_for_eq_words_unaligned_core::avx2::eq_words_unaligned" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::traits::words_ord::funcs_for_cmp_aligned_core::avx2::cmp_aligned" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::traits::words_ord::funcs_for_cmp_unaligned_core::avx2::cmp_unaligned" +artifact = "library" +targets = ["x86_64-avx2"] + +[[root]] +symbol = "bit_string::bit_string::impls_for_construction::funcs_for_pack_bools_core::sse2::words" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::bit_string::impls_for_construction::impls_for_from_str::funcs_for_pack_str_core::sse2::words" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_not_core::sse2::words" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_shl_core::sse2::words" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_shr_core::sse2::words" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_edit::bits_copied::funcs_for_copy_words_shifted_core::sse2::copy_words_shifted" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::sse41::eq_words" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_eq::funcs_for_eq_words_unaligned_core::sse41::eq_words_unaligned" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_ord::funcs_for_cmp_aligned_core::sse41::cmp_aligned" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_ord::funcs_for_cmp_unaligned_core::sse41::cmp_unaligned" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_scan::funcs_for_count_ones::ssse3::count_words" +artifact = "library" +targets = ["x86_64-sse41"] + +[[root]] +symbol = "bit_string::bit_string::impls_for_construction::funcs_for_pack_bools_core::neon::words" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_not_core::neon::words" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_shl_core::neon::words" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_shr_core::neon::words" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_edit::bits_copied::funcs_for_copy_words_shifted_core::neon::copy_words_shifted" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::neon::eq_words" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_eq::funcs_for_eq_words_unaligned_core::neon::eq_words_unaligned" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_ord::funcs_for_cmp_aligned_core::neon::cmp_aligned" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_ord::funcs_for_cmp_unaligned_core::neon::cmp_unaligned" +artifact = "library" +targets = ["aarch64-neon"] + +[[root]] +symbol = "bit_string::traits::words_scan::funcs_for_count_ones::neon::count_words" +artifact = "library" +targets = ["aarch64-neon"] diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index 26136ba..86d6ce2 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -16,10 +16,12 @@ from bench_config import CODEGEN_CONFIGS FUNCTION = re.compile(r"^\s*[0-9a-f]+ <(.+)>:\s*$") -INSTRUCTION = re.compile(r"^\s*([0-9a-f]+):\s+((?:[0-9a-f]{2}(?:\s+|$))+)(.*?)\s*$") +X86_INSTRUCTION = re.compile(r"^\s*([0-9a-f]+):\s+((?:[0-9a-f]{2}(?:\s+|$))+)(.*?)\s*$") +AARCH64_INSTRUCTION = re.compile(r"^\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(.*?)\s*$") RELOCATION = re.compile(r"^\s*([0-9a-f]+):\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") SYMBOL_TARGET = re.compile(r"(?:0x)?([0-9a-f]+)\s+<([^>]+)>") -RELOCATION_ADDEND = re.compile(r"[+-]0x[0-9a-f]+$") +PC_RELATIVE_X86_RELOCATIONS = {"R_X86_64_PC32", "R_X86_64_PLT32"} +X86_PC_BIAS = re.compile(r"-0x4$") OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") @@ -58,8 +60,11 @@ def classify(old: Function, new: Function) -> str: return "CODEGEN IDENTICAL" -def _relocation_target(value: str) -> str: - return RELOCATION_ADDEND.sub("", value.strip()) +def _relocation_target(kind: str, value: str) -> str: + value = value.strip() + if kind in PC_RELATIVE_X86_RELOCATIONS: + return X86_PC_BIAS.sub("", value) + return value def _is_call(mnemonic: str, arch: str) -> bool: @@ -98,12 +103,15 @@ def _normalize(instructions: list[Instruction], arch: str) -> Function: symbol = match.group(2) target = f"@instruction_{indexes[address]}" if address in indexes and "+0x" in symbol else f"<{symbol}>" operands = SYMBOL_TARGET.sub(target, operands) - relocations = " ".join(f"[{kind}:{_relocation_target(target)}]" for kind, target in instruction.relocations) + relocations = " ".join( + f"[{kind}:{_relocation_target(kind, target)}]" for kind, target in instruction.relocations + ) normalized.append(" ".join(f"{mnemonic} {operands} {relocations}".split())) return Function(tuple(normalized), b"".join(item.encoded for item in instructions), calls, branches) def parse_objdump(output: str, arch: str) -> dict[str, Function]: + instruction_pattern = X86_INSTRUCTION if arch == "x86_64" else AARCH64_INSTRUCTION raw: dict[str, list[Instruction]] = {} current: str | None = None last_instruction: Instruction | None = None @@ -123,7 +131,7 @@ def parse_objdump(output: str, arch: str) -> dict[str, Function]: if last_instruction is not None: last_instruction.relocations.append((relocation.group(2), relocation.group(3))) continue - match = INSTRUCTION.match(line) + match = instruction_pattern.match(line) if not match: continue encoded = bytes.fromhex(match.group(2)) diff --git a/scripts/kernel_inventory.toml b/scripts/kernel_inventory.toml index 2aff064..1d796a2 100644 --- a/scripts/kernel_inventory.toml +++ b/scripts/kernel_inventory.toml @@ -1,67 +1,87 @@ +## Each entry records a stable caller observation root. Backend functions that +## remain as independent symbols require separate library roots. + [[kernel]] path = "src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs" +coverage = "caller" roots = ["codegen_bit_string_from_bool_slice"] [[kernel]] path = "src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs" +coverage = "caller" roots = ["codegen_bit_string_from_str"] [[kernel]] path = "src/traits/words_arith/funcs_for_binary_core.rs" +coverage = "caller" roots = ["codegen_bit_string_xor"] [[kernel]] path = "src/traits/words_arith/funcs_for_not_core.rs" +coverage = "caller" roots = ["codegen_bit_string_not"] [[kernel]] path = "src/traits/words_arith/funcs_for_shl_core.rs" +coverage = "caller" roots = ["codegen_bit_string_shl"] [[kernel]] path = "src/traits/words_arith/funcs_for_shr_core.rs" +coverage = "caller" roots = ["codegen_bit_string_shr"] [[kernel]] path = "src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs" +coverage = "caller" roots = ["codegen_bit_string_slice_from"] [[kernel]] path = "src/traits/words_eq/funcs_for_eq_words_aligned_core.rs" +coverage = "caller" roots = ["codegen_bit_string_matches_at"] [[kernel]] path = "src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs" +coverage = "caller" roots = ["codegen_bit_string_matches_at"] [[kernel]] path = "src/traits/words_find/funcs_for_contains_core.rs" +coverage = "caller" roots = ["codegen_bit_string_contains"] [[kernel]] path = "src/traits/words_find/funcs_for_find_core.rs" +coverage = "caller" roots = ["codegen_bit_string_find"] [[kernel]] path = "src/traits/words_find/funcs_for_rfind_core.rs" +coverage = "caller" roots = ["codegen_bit_string_rfind"] [[kernel]] path = "src/traits/words_ord/funcs_for_cmp_aligned_core.rs" +coverage = "caller" roots = ["codegen_bit_string_cmp"] [[kernel]] path = "src/traits/words_ord/funcs_for_cmp_unaligned_core.rs" +coverage = "caller" roots = ["codegen_bit_string_cmp"] [[kernel]] path = "src/traits/words_scan/funcs_for_count_ones.rs" +coverage = "caller" roots = ["codegen_bit_string_count_ones"] [[kernel]] path = "src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs" +coverage = "caller" roots = ["codegen_bit_string_leading_zeros"] [[kernel]] path = "src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs" +coverage = "caller" roots = ["codegen_bit_string_trailing_zeros"] diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py index d6ce6e5..60cb436 100755 --- a/scripts/run_codegen_comparison.py +++ b/scripts/run_codegen_comparison.py @@ -25,7 +25,7 @@ def build_library(worktree: Path, target_dir: Path, rustflags: str) -> Path: env = os.environ.copy() env["RUSTFLAGS"] = rustflags env["CARGO_TARGET_DIR"] = str(target_dir) - run(["cargo", "rustc", "--release", "--lib", "--", "--emit=obj"], worktree, env) + run(["cargo", "build", "--release", "--lib"], worktree, env) return target_dir / "release" / "libbit_string.rlib" diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index fed4036..a6518aa 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -36,6 +36,11 @@ def test_relocation_is_not_an_instruction(self) -> None: self.assertEqual(len(function.instructions), 2) self.assertEqual(function.instructions[0], "call [R_X86_64_PLT32:foo]") + def test_semantic_relocation_addend_is_preserved(self) -> None: + first = parse("0000 :\n 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n 3: R_X86_64_PC32 .rodata+0x20") + second = parse("0000 :\n 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n 3: R_X86_64_PC32 .rodata+0x40") + self.assertNotEqual(first.instructions, second.instructions) + def test_duplicate_disassembly_symbol_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "duplicate disassembly symbol"): compare_codegen.parse_objdump("0000 :\n 0: c3 ret\n0000 :\n 0: c3 ret", "x86_64") @@ -48,13 +53,19 @@ def test_encoding_size_change_is_not_identical(self) -> None: def test_aarch64_calls_and_branches(self) -> None: function = parse( - "0000 :\n 0: 00 00 00 94 bl 0 \n 4: 40 00 00 54 b.eq c \n 8: 00 04 00 91 add x0, x0, #0x1\n c: c0 03 5f d6 ret", + "0000000000000000 :\n" + " 0: 94000000 bl 0 \n" + " 0: R_AARCH64_CALL26 callee\n" + " 4: 54000040 b.eq c \n" + " 8: 91000400 add x0, x0, #0x1\n" + " c: d65f03c0 ret", "aarch64", ) self.assertEqual(function.calls, 1) self.assertEqual(function.branches, 1) self.assertIn("@instruction_3", function.instructions[1]) self.assertIn("#0x1", function.instructions[2]) + self.assertIn("[R_AARCH64_CALL26:callee]", function.instructions[0]) def test_expected_missing_symbol_fails(self) -> None: with tempfile.TemporaryDirectory() as directory: @@ -87,6 +98,7 @@ def test_inventory_has_root_for_each_kernel(self) -> None: inventory = tomllib.loads((scripts / "kernel_inventory.toml").read_text())["kernel"] self.assertEqual(len(inventory), 17) for kernel in inventory: + self.assertEqual(kernel["coverage"], "caller") self.assertTrue(kernel["roots"]) self.assertLessEqual(set(kernel["roots"]), roots) From f6c328fbfd2877ce0cdc9677481fbcd97949c9e6 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 04:04:45 +0000 Subject: [PATCH 09/79] fix: audit independent backend codegen roots --- scripts/codegen_roots.toml | 17 +++++++++++------ scripts/compare_codegen.py | 30 ++++++++++++++++++++++++++++++ scripts/run_codegen_comparison.py | 2 ++ scripts/tests/test_codegen.py | 16 ++++++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/scripts/codegen_roots.toml b/scripts/codegen_roots.toml index 1a55be2..81e3762 100644 --- a/scripts/codegen_roots.toml +++ b/scripts/codegen_roots.toml @@ -126,32 +126,37 @@ targets = ["x86_64-avx2"] [[root]] symbol = "bit_string::bit_string::impls_for_construction::funcs_for_pack_bools_core::sse2::words" artifact = "library" -targets = ["x86_64-sse41"] +targets = ["x86_64-baseline", "x86_64-sse41"] [[root]] symbol = "bit_string::bit_string::impls_for_construction::impls_for_from_str::funcs_for_pack_str_core::sse2::words" artifact = "library" -targets = ["x86_64-sse41"] +targets = ["x86_64-baseline", "x86_64-sse41"] [[root]] symbol = "bit_string::traits::words_arith::funcs_for_not_core::sse2::words" artifact = "library" -targets = ["x86_64-sse41"] +targets = ["x86_64-baseline", "x86_64-sse41"] [[root]] symbol = "bit_string::traits::words_arith::funcs_for_shl_core::sse2::words" artifact = "library" -targets = ["x86_64-sse41"] +targets = ["x86_64-baseline", "x86_64-sse41"] [[root]] symbol = "bit_string::traits::words_arith::funcs_for_shr_core::sse2::words" artifact = "library" -targets = ["x86_64-sse41"] +targets = ["x86_64-baseline", "x86_64-sse41"] [[root]] symbol = "bit_string::traits::words_edit::bits_copied::funcs_for_copy_words_shifted_core::sse2::copy_words_shifted" artifact = "library" -targets = ["x86_64-sse41"] +targets = ["x86_64-baseline", "x86_64-sse41"] + +[[root]] +symbol = "bit_string::traits::words_arith::funcs_for_shl_core::scalar::words" +artifact = "library" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] [[root]] symbol = "bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::sse41::eq_words" diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index 86d6ce2..e0d6cc5 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -23,6 +23,7 @@ PC_RELATIVE_X86_RELOCATIONS = {"R_X86_64_PC32", "R_X86_64_PLT32"} X86_PC_BIAS = re.compile(r"-0x4$") OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") +BACKEND_MODULE = re.compile(r"::(?:scalar|sse2|ssse3|sse41|avx2|neon)::") @dataclass @@ -160,12 +161,32 @@ def load_roots(path: Path) -> list[Root]: return roots +def load_kernel_modules(path: Path) -> tuple[str, ...]: + data = tomllib.loads(path.read_text(encoding="utf-8")) + modules = [] + for kernel in data.get("kernel", []): + source = Path(kernel["path"]) + if source.parts[0] != "src" or source.suffix != ".rs": + raise ValueError(f"{path}: invalid kernel source path: {source}") + modules.append("bit_string::" + "::".join(source.with_suffix("").parts[1:])) + return tuple(modules) + + +def independent_backend_symbols(functions: dict[str, Function], modules: tuple[str, ...]) -> set[str]: + return { + symbol + for symbol in functions + if BACKEND_MODULE.search(symbol) and any(symbol.startswith(f"{module}::") for module in modules) + } + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("baseline", type=Path) parser.add_argument("current", type=Path) parser.add_argument("--roots", type=Path, required=True) parser.add_argument("--artifact", required=True) + parser.add_argument("--inventory", type=Path) parser.add_argument("--target", required=True) parser.add_argument("--arch", choices=("x86_64", "aarch64"), required=True) parser.add_argument("--objdump", default="objdump") @@ -174,6 +195,15 @@ def main() -> int: after = load_functions(args.current, args.objdump, args.arch) roots = [root for root in load_roots(args.roots) if root.artifact == args.artifact] identical = encoding_changed = changed = errors = skipped = 0 + if args.artifact == "library": + if args.inventory is None: + parser.error("--inventory is required for library artifacts") + modules = load_kernel_modules(args.inventory) + registered = {root.symbol for root in roots if args.target in root.targets} + discovered = independent_backend_symbols(before, modules) | independent_backend_symbols(after, modules) + for symbol in sorted(discovered - registered): + errors += 1 + print(f"ERROR: independent backend symbol is not a configured root: {symbol}", file=sys.stderr) for root in roots: if args.target not in root.targets: skipped += 1 diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py index 60cb436..41aaa97 100755 --- a/scripts/run_codegen_comparison.py +++ b/scripts/run_codegen_comparison.py @@ -61,6 +61,8 @@ def compare(root: Path, baseline: Path, current: Path, artifact: str, target: st str(root / "scripts" / "codegen_roots.toml"), "--artifact", artifact, + "--inventory", + str(root / "scripts" / "kernel_inventory.toml"), "--target", target, "--arch", diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index a6518aa..395265a 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -102,6 +102,22 @@ def test_inventory_has_root_for_each_kernel(self) -> None: self.assertTrue(kernel["roots"]) self.assertLessEqual(set(kernel["roots"]), roots) + def test_independent_backend_symbols_are_discovered_from_inventory_modules(self) -> None: + function = parse("0000 :\n 0: c3 ret") + functions = { + "bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words": function, + "bit_string::traits::words_scan::unrelated::avx2::count_words": function, + "bit_string::traits::words_scan::funcs_for_count_ones::entry": function, + } + discovered = compare_codegen.independent_backend_symbols( + functions, + ("bit_string::traits::words_scan::funcs_for_count_ones",), + ) + self.assertEqual( + discovered, + {"bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words"}, + ) + if __name__ == "__main__": unittest.main() From 5cd550a7d33ffa8cd45902897eab34f89c76e0b5 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 04:21:00 +0000 Subject: [PATCH 10/79] refactor: split count_ones backends --- src/traits/words_scan/funcs_for_count_ones.rs | 232 +----------------- .../words_scan/funcs_for_count_ones/avx2.rs | 80 ++++++ .../words_scan/funcs_for_count_ones/neon.rs | 43 ++++ .../words_scan/funcs_for_count_ones/scalar.rs | 20 ++ .../words_scan/funcs_for_count_ones/ssse3.rs | 77 ++++++ 5 files changed, 224 insertions(+), 228 deletions(-) create mode 100644 src/traits/words_scan/funcs_for_count_ones/avx2.rs create mode 100644 src/traits/words_scan/funcs_for_count_ones/neon.rs create mode 100644 src/traits/words_scan/funcs_for_count_ones/scalar.rs create mode 100644 src/traits/words_scan/funcs_for_count_ones/ssse3.rs diff --git a/src/traits/words_scan/funcs_for_count_ones.rs b/src/traits/words_scan/funcs_for_count_ones.rs index e23ac4f..13c244b 100644 --- a/src/traits/words_scan/funcs_for_count_ones.rs +++ b/src/traits/words_scan/funcs_for_count_ones.rs @@ -86,243 +86,19 @@ unsafe fn dispatch(src: *const u64, len: usize) -> usize { } #[allow(unused)] -mod scalar { - /// Scalar backend for counting set bits in `src[0..len]`. - /// - /// # Safety - /// - /// - `src` must be valid for reads of `len` initialized `u64` values. - #[inline] - pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { - let mut count = 0usize; - - for i in 0..len { - // SAFETY: - // - `i < len`. - // - Pointer validity is guaranteed by the caller. - unsafe { - count += src.add(i).read().count_ones() as usize; - } - } - - count - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_add_epi8, _mm256_add_epi64, _mm256_and_si256, _mm256_loadu_si256, - _mm256_sad_epu8, _mm256_set1_epi8, _mm256_setr_epi8, _mm256_setzero_si256, - _mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_storeu_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_add_epi8, _mm256_add_epi64, _mm256_and_si256, _mm256_loadu_si256, - _mm256_sad_epu8, _mm256_set1_epi8, _mm256_setr_epi8, _mm256_setzero_si256, - _mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_storeu_si256, - }; - - const LANES: usize = 4; - - /// AVX2 backend for counting set bits in `src[0..len]`. - /// - /// Uses nibble lookup with `vpshufb`, then sums byte counts with `vpsadbw`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `src` must be valid for reads of `len` initialized `u64` values. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { - let chunks = len / LANES; - - // SAFETY: - // - These constructors require AVX2 to be available. - // - This function is compiled with `target_feature = "avx2"`. - let lookup = _mm256_setr_epi8( - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, // - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, - ); - let low_mask = _mm256_set1_epi8(0x0F); - let zero = _mm256_setzero_si256(); - - // Accumulate popcounts in a SIMD register to avoid per-chunk - // store-to-memory round-trips. Reduced to scalar only once at the end. - let mut acc = zero; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm256_loadu_si256` permits unaligned reads. - // - `src` validity is guaranteed by the caller. - unsafe { - let bytes = _mm256_loadu_si256(src.add(offset).cast::<__m256i>()); - - let low = _mm256_and_si256(bytes, low_mask); - let high = _mm256_and_si256(_mm256_srli_epi16(bytes, 4), low_mask); - - let low_counts = _mm256_shuffle_epi8(lookup, low); - let high_counts = _mm256_shuffle_epi8(lookup, high); - let byte_counts = _mm256_add_epi8(low_counts, high_counts); - - let sums = _mm256_sad_epu8(byte_counts, zero); - acc = _mm256_add_epi64(acc, sums); - } - } - - let mut lane_sums = [0u64; LANES]; - // SAFETY: `lane_sums` has sufficient space for 4 u64 values. - unsafe { _mm256_storeu_si256(lane_sums.as_mut_ptr().cast::<__m256i>(), acc) }; - let mut count = lane_sums.iter().map(|&sum| sum as usize).sum::(); - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity is guaranteed by the caller. - count += unsafe { scalar::count_words(src.add(done), len - done) }; - count - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod ssse3 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_add_epi8, _mm_add_epi64, _mm_and_si128, _mm_loadu_si128, _mm_sad_epu8, - _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, - _mm_storeu_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_add_epi8, _mm_add_epi64, _mm_and_si128, _mm_loadu_si128, _mm_sad_epu8, - _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, - _mm_storeu_si128, - }; - - const LANES: usize = 2; - - /// SSSE3 backend for counting set bits in `src[0..len]`. - /// - /// Uses nibble lookup with `pshufb`, then sums byte counts with `psadbw`. - /// - /// # Safety - /// - /// - Caller must only call this when SSSE3 is available. - /// - `src` must be valid for reads of `len` initialized `u64` values. - #[target_feature(enable = "ssse3")] - pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { - let chunks = len / LANES; - - let lookup = _mm_setr_epi8( - 0, 1, 1, 2, 1, 2, 2, 3, // - 1, 2, 2, 3, 2, 3, 3, 4, - ); - let low_mask = _mm_set1_epi8(0x0F); - let zero = _mm_setzero_si128(); - - // Accumulate popcounts in a SIMD register to avoid per-chunk - // store-to-memory round-trips. Reduced to scalar only once at the end. - let mut acc = zero; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm_loadu_si128` permits unaligned reads. - // - `src` validity is guaranteed by the caller. - unsafe { - let bytes = _mm_loadu_si128(src.add(offset).cast::<__m128i>()); - - let low = _mm_and_si128(bytes, low_mask); - let high = _mm_and_si128(_mm_srli_epi16(bytes, 4), low_mask); - - let low_counts = _mm_shuffle_epi8(lookup, low); - let high_counts = _mm_shuffle_epi8(lookup, high); - let byte_counts = _mm_add_epi8(low_counts, high_counts); - - let sums = _mm_sad_epu8(byte_counts, zero); - acc = _mm_add_epi64(acc, sums); - } - } - - let mut lane_sums = [0u64; LANES]; - // SAFETY: `lane_sums` has sufficient space for 2 u64 values. - unsafe { _mm_storeu_si128(lane_sums.as_mut_ptr().cast::<__m128i>(), acc) }; - let mut count = lane_sums.iter().map(|&sum| sum as usize).sum::(); - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity is guaranteed by the caller. - count += unsafe { scalar::count_words(src.add(done), len - done) }; - count - } -} +mod ssse3; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::scalar; - - use core::arch::aarch64::{vaddvq_u8, vcntq_u8, vld1q_u64, vreinterpretq_u8_u64}; - - const LANES: usize = 2; - - /// NEON backend for counting set bits in `src[0..len]`. - /// - /// Counts bits per byte with `vcntq_u8`, then horizontally sums byte counts. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `src` must be valid for reads of `len` initialized `u64` values. - #[target_feature(enable = "neon")] - pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { - let chunks = len / LANES; - let mut count = 0usize; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `vld1q_u64` reads exactly 2 u64 values. - // - `src` validity is guaranteed by the caller. - unsafe { - let words = vld1q_u64(src.add(offset)); - let bytes = vreinterpretq_u8_u64(words); - let byte_counts = vcntq_u8(bytes); - - count += vaddvq_u8(byte_counts) as usize; - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity is guaranteed by the caller. - count + unsafe { scalar::count_words(src.add(done), len - done) } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_scan/funcs_for_count_ones/avx2.rs b/src/traits/words_scan/funcs_for_count_ones/avx2.rs new file mode 100644 index 0000000..78cf8e6 --- /dev/null +++ b/src/traits/words_scan/funcs_for_count_ones/avx2.rs @@ -0,0 +1,80 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_add_epi8, _mm256_add_epi64, _mm256_and_si256, _mm256_loadu_si256, + _mm256_sad_epu8, _mm256_set1_epi8, _mm256_setr_epi8, _mm256_setzero_si256, _mm256_shuffle_epi8, + _mm256_srli_epi16, _mm256_storeu_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_add_epi8, _mm256_add_epi64, _mm256_and_si256, _mm256_loadu_si256, + _mm256_sad_epu8, _mm256_set1_epi8, _mm256_setr_epi8, _mm256_setzero_si256, _mm256_shuffle_epi8, + _mm256_srli_epi16, _mm256_storeu_si256, +}; + +const LANES: usize = 4; + +/// AVX2 backend for counting set bits in `src[0..len]`. +/// +/// Uses nibble lookup with `vpshufb`, then sums byte counts with `vpsadbw`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `src` must be valid for reads of `len` initialized `u64` values. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { + let chunks = len / LANES; + + // SAFETY: + // - These constructors require AVX2 to be available. + // - This function is compiled with `target_feature = "avx2"`. + let lookup = _mm256_setr_epi8( + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, // + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + ); + let low_mask = _mm256_set1_epi8(0x0F); + let zero = _mm256_setzero_si256(); + + // Accumulate popcounts in a SIMD register to avoid per-chunk + // store-to-memory round-trips. Reduced to scalar only once at the end. + let mut acc = zero; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm256_loadu_si256` permits unaligned reads. + // - `src` validity is guaranteed by the caller. + unsafe { + let bytes = _mm256_loadu_si256(src.add(offset).cast::<__m256i>()); + + let low = _mm256_and_si256(bytes, low_mask); + let high = _mm256_and_si256(_mm256_srli_epi16(bytes, 4), low_mask); + + let low_counts = _mm256_shuffle_epi8(lookup, low); + let high_counts = _mm256_shuffle_epi8(lookup, high); + let byte_counts = _mm256_add_epi8(low_counts, high_counts); + + let sums = _mm256_sad_epu8(byte_counts, zero); + acc = _mm256_add_epi64(acc, sums); + } + } + + let mut lane_sums = [0u64; LANES]; + // SAFETY: `lane_sums` has sufficient space for 4 u64 values. + unsafe { _mm256_storeu_si256(lane_sums.as_mut_ptr().cast::<__m256i>(), acc) }; + let mut count = lane_sums.iter().map(|&sum| sum as usize).sum::(); + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity is guaranteed by the caller. + count += unsafe { scalar::count_words(src.add(done), len - done) }; + count +} diff --git a/src/traits/words_scan/funcs_for_count_ones/neon.rs b/src/traits/words_scan/funcs_for_count_ones/neon.rs new file mode 100644 index 0000000..62ff9dd --- /dev/null +++ b/src/traits/words_scan/funcs_for_count_ones/neon.rs @@ -0,0 +1,43 @@ +use super::scalar; + +use core::arch::aarch64::{vaddvq_u8, vcntq_u8, vld1q_u64, vreinterpretq_u8_u64}; + +const LANES: usize = 2; + +/// NEON backend for counting set bits in `src[0..len]`. +/// +/// Counts bits per byte with `vcntq_u8`, then horizontally sums byte counts. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `src` must be valid for reads of `len` initialized `u64` values. +#[target_feature(enable = "neon")] +pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { + let chunks = len / LANES; + let mut count = 0usize; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `vld1q_u64` reads exactly 2 u64 values. + // - `src` validity is guaranteed by the caller. + unsafe { + let words = vld1q_u64(src.add(offset)); + let bytes = vreinterpretq_u8_u64(words); + let byte_counts = vcntq_u8(bytes); + + count += vaddvq_u8(byte_counts) as usize; + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity is guaranteed by the caller. + count + unsafe { scalar::count_words(src.add(done), len - done) } +} diff --git a/src/traits/words_scan/funcs_for_count_ones/scalar.rs b/src/traits/words_scan/funcs_for_count_ones/scalar.rs new file mode 100644 index 0000000..9fcf732 --- /dev/null +++ b/src/traits/words_scan/funcs_for_count_ones/scalar.rs @@ -0,0 +1,20 @@ +/// Scalar backend for counting set bits in `src[0..len]`. +/// +/// # Safety +/// +/// - `src` must be valid for reads of `len` initialized `u64` values. +#[inline] +pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { + let mut count = 0usize; + + for i in 0..len { + // SAFETY: + // - `i < len`. + // - Pointer validity is guaranteed by the caller. + unsafe { + count += src.add(i).read().count_ones() as usize; + } + } + + count +} diff --git a/src/traits/words_scan/funcs_for_count_ones/ssse3.rs b/src/traits/words_scan/funcs_for_count_ones/ssse3.rs new file mode 100644 index 0000000..f677134 --- /dev/null +++ b/src/traits/words_scan/funcs_for_count_ones/ssse3.rs @@ -0,0 +1,77 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_add_epi8, _mm_add_epi64, _mm_and_si128, _mm_loadu_si128, _mm_sad_epu8, + _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, + _mm_storeu_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_add_epi8, _mm_add_epi64, _mm_and_si128, _mm_loadu_si128, _mm_sad_epu8, + _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, + _mm_storeu_si128, +}; + +const LANES: usize = 2; + +/// SSSE3 backend for counting set bits in `src[0..len]`. +/// +/// Uses nibble lookup with `pshufb`, then sums byte counts with `psadbw`. +/// +/// # Safety +/// +/// - Caller must only call this when SSSE3 is available. +/// - `src` must be valid for reads of `len` initialized `u64` values. +#[target_feature(enable = "ssse3")] +pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { + let chunks = len / LANES; + + let lookup = _mm_setr_epi8( + 0, 1, 1, 2, 1, 2, 2, 3, // + 1, 2, 2, 3, 2, 3, 3, 4, + ); + let low_mask = _mm_set1_epi8(0x0F); + let zero = _mm_setzero_si128(); + + // Accumulate popcounts in a SIMD register to avoid per-chunk + // store-to-memory round-trips. Reduced to scalar only once at the end. + let mut acc = zero; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm_loadu_si128` permits unaligned reads. + // - `src` validity is guaranteed by the caller. + unsafe { + let bytes = _mm_loadu_si128(src.add(offset).cast::<__m128i>()); + + let low = _mm_and_si128(bytes, low_mask); + let high = _mm_and_si128(_mm_srli_epi16(bytes, 4), low_mask); + + let low_counts = _mm_shuffle_epi8(lookup, low); + let high_counts = _mm_shuffle_epi8(lookup, high); + let byte_counts = _mm_add_epi8(low_counts, high_counts); + + let sums = _mm_sad_epu8(byte_counts, zero); + acc = _mm_add_epi64(acc, sums); + } + } + + let mut lane_sums = [0u64; LANES]; + // SAFETY: `lane_sums` has sufficient space for 2 u64 values. + unsafe { _mm_storeu_si128(lane_sums.as_mut_ptr().cast::<__m128i>(), acc) }; + let mut count = lane_sums.iter().map(|&sum| sum as usize).sum::(); + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity is guaranteed by the caller. + count += unsafe { scalar::count_words(src.add(done), len - done) }; + count +} From ec585dc8b27cf5c37b7eae9733f60a5d357408e0 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 04:27:13 +0000 Subject: [PATCH 11/79] fix: materialize codegen harness source --- scripts/run_codegen_comparison.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py index 41aaa97..b63805b 100755 --- a/scripts/run_codegen_comparison.py +++ b/scripts/run_codegen_comparison.py @@ -29,11 +29,21 @@ def build_library(worktree: Path, target_dir: Path, rustflags: str) -> Path: return target_dir / "release" / "libbit_string.rlib" +def materialize_harness(root: Path, output: Path) -> Path: + source = root / "scripts" / "codegen_harness.rs" + if source.is_file(): + return source + source = output.with_suffix(".rs") + source.write_bytes(subprocess.check_output(["git", "show", "HEAD:scripts/codegen_harness.rs"], cwd=root)) + return source + + def build_harness(root: Path, library: Path, output: Path, rustflags: str) -> None: dependency_dir = library.parent / "deps" + source = materialize_harness(root, output) command = [ "rustc", - str(root / "scripts" / "codegen_harness.rs"), + str(source), "--crate-name", "bit_string_codegen_harness", "--crate-type=lib", From f8a5b25ad6cfe461a4f3b173b26ab1e320226a0e Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 04:30:37 +0000 Subject: [PATCH 12/79] fix: pin codegen harness revision --- scripts/run_codegen_comparison.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py index b63805b..40b1157 100755 --- a/scripts/run_codegen_comparison.py +++ b/scripts/run_codegen_comparison.py @@ -29,18 +29,15 @@ def build_library(worktree: Path, target_dir: Path, rustflags: str) -> Path: return target_dir / "release" / "libbit_string.rlib" -def materialize_harness(root: Path, output: Path) -> Path: - source = root / "scripts" / "codegen_harness.rs" - if source.is_file(): - return source +def materialize_harness(root: Path, output: Path, revision: str) -> Path: source = output.with_suffix(".rs") - source.write_bytes(subprocess.check_output(["git", "show", "HEAD:scripts/codegen_harness.rs"], cwd=root)) + source.write_bytes(subprocess.check_output(["git", "show", f"{revision}:scripts/codegen_harness.rs"], cwd=root)) return source -def build_harness(root: Path, library: Path, output: Path, rustflags: str) -> None: +def build_harness(root: Path, library: Path, output: Path, rustflags: str, revision: str) -> None: dependency_dir = library.parent / "deps" - source = materialize_harness(root, output) + source = materialize_harness(root, output, revision) command = [ "rustc", str(source), @@ -87,12 +84,20 @@ def main() -> int: parser.add_argument("--config", choices=sorted(CODEGEN_CONFIGS), action="append") args = parser.parse_args() root = Path(__file__).resolve().parents[1] + current_revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() output = root / "target" / "codegen-comparison" baseline_worktree = output / "baseline-worktree" output.mkdir(parents=True, exist_ok=True) if not baseline_worktree.exists(): run(["git", "worktree", "add", "--detach", str(baseline_worktree), args.baseline], root, os.environ.copy()) else: + worktree_root = subprocess.check_output( + ["git", "-C", str(baseline_worktree), "rev-parse", "--show-toplevel"], + cwd=root, + text=True, + ).strip() + if Path(worktree_root).resolve() == root.resolve(): + raise SystemExit("baseline worktree resolves to the current worktree") run(["git", "-C", str(baseline_worktree), "checkout", "--detach", args.baseline], root, os.environ.copy()) host_arch = "aarch64" if platform.machine() in {"aarch64", "arm64"} else "x86_64" configs = args.config or [name for name, (arch, _) in CODEGEN_CONFIGS.items() if arch == host_arch] @@ -107,8 +112,8 @@ def main() -> int: current_library = build_library(root, config_dir / "current-target", rustflags) baseline_harness = config_dir / "baseline-harness.o" current_harness = config_dir / "current-harness.o" - build_harness(root, baseline_library, baseline_harness, rustflags) - build_harness(root, current_library, current_harness, rustflags) + build_harness(root, baseline_library, baseline_harness, rustflags, current_revision) + build_harness(root, current_library, current_harness, rustflags, current_revision) failed |= compare(root, baseline_harness, current_harness, "harness", config, arch) != 0 failed |= compare(root, baseline_library, current_library, "library", config, arch) != 0 return int(failed) From 55dea04c18cb903f0c1a39c57951fe650b8d1c3f Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 04:33:02 +0000 Subject: [PATCH 13/79] fix: isolate baseline codegen worktree --- scripts/run_codegen_comparison.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py index 40b1157..1dadaa0 100755 --- a/scripts/run_codegen_comparison.py +++ b/scripts/run_codegen_comparison.py @@ -4,11 +4,13 @@ from __future__ import annotations import argparse +import atexit import os import platform import shlex import subprocess import sys +import tempfile from pathlib import Path from bench_config import BASELINE_SHA, CODEGEN_CONFIGS @@ -86,19 +88,21 @@ def main() -> int: root = Path(__file__).resolve().parents[1] current_revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() output = root / "target" / "codegen-comparison" - baseline_worktree = output / "baseline-worktree" output.mkdir(parents=True, exist_ok=True) - if not baseline_worktree.exists(): - run(["git", "worktree", "add", "--detach", str(baseline_worktree), args.baseline], root, os.environ.copy()) - else: - worktree_root = subprocess.check_output( - ["git", "-C", str(baseline_worktree), "rev-parse", "--show-toplevel"], + baseline_worktree = Path(tempfile.mkdtemp(prefix="bit-string-codegen-baseline-")) + baseline_worktree.rmdir() + run(["git", "worktree", "add", "--detach", str(baseline_worktree), args.baseline], root, os.environ.copy()) + + def cleanup_worktree() -> None: + subprocess.run( + ["git", "worktree", "remove", "--force", str(baseline_worktree)], cwd=root, - text=True, - ).strip() - if Path(worktree_root).resolve() == root.resolve(): - raise SystemExit("baseline worktree resolves to the current worktree") - run(["git", "-C", str(baseline_worktree), "checkout", "--detach", args.baseline], root, os.environ.copy()) + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + atexit.register(cleanup_worktree) host_arch = "aarch64" if platform.machine() in {"aarch64", "arm64"} else "x86_64" configs = args.config or [name for name, (arch, _) in CODEGEN_CONFIGS.items() if arch == host_arch] failed = False From 0a6ddd54f542bf61ceb19d9fa677fb97a2c059d8 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 04:52:47 +0000 Subject: [PATCH 14/79] refactor: split pack_bools backends --- .../funcs_for_pack_bools_core.rs | 259 +----------------- .../funcs_for_pack_bools_core/avx2.rs | 61 +++++ .../funcs_for_pack_bools_core/neon.rs | 60 ++++ .../funcs_for_pack_bools_core/scalar.rs | 70 +++++ .../funcs_for_pack_bools_core/sse2.rs | 54 ++++ 5 files changed, 249 insertions(+), 255 deletions(-) create mode 100644 src/bit_string/impls_for_construction/funcs_for_pack_bools_core/avx2.rs create mode 100644 src/bit_string/impls_for_construction/funcs_for_pack_bools_core/neon.rs create mode 100644 src/bit_string/impls_for_construction/funcs_for_pack_bools_core/scalar.rs create mode 100644 src/bit_string/impls_for_construction/funcs_for_pack_bools_core/sse2.rs diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs index a395135..42edde1 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs @@ -70,270 +70,19 @@ unsafe fn dispatch(dst: *mut u64, src: *const u8, bit_len: usize) { } #[allow(unused)] -mod scalar { - use super::scalar; - - /// Scalar backend: accumulate 64 bytes at a time into one u64. - /// - /// # Safety - /// - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[inline] - pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { - while bit_len >= 64 { - // SAFETY: `bit_len >= 64`, so reading 64 bytes from `src` is valid. - // `dst` points to the next output slot. - unsafe { - *dst = scalar::pack_64(src); - } - // SAFETY: - // - `dst` advances by 1 word; the caller ensures the destination - // has enough capacity for all full-word writes. - // - `src` advances by 64 bytes; `bit_len >= 64` ensures read bounds. - unsafe { - dst = dst.add(1); - src = src.add(64); - } - bit_len -= 64; - } - - if bit_len > 0 { - // SAFETY: `bit_len > 0`, so reading `bit_len` bytes from `src` is valid. - unsafe { - *dst = scalar::pack_partial(src, bit_len); - } - } - } - - /// Pack exactly 64 bytes into one u64 (little-endian: byte i → bit i). - /// - /// # Safety - /// - /// `src` must be valid for reads of 64 u8 values. - #[inline] - unsafe fn pack_64(src: *const u8) -> u64 { - let mut word = 0u64; - for i in 0..64 { - // SAFETY: - // - `i < 64`, offset is in bounds. - // - `src` is valid for 64 reads per caller contract. - let byte = unsafe { src.add(i).read() }; - word |= ((byte & 1) as u64) << i; - } - word - } - - /// Pack fewer than 64 bytes into one u64. - /// - /// # Safety - /// - /// `src` must be valid for reads of `len` u8 values (`len < 64`). - #[inline] - unsafe fn pack_partial(src: *const u8, len: usize) -> u64 { - let mut word = 0u64; - for i in 0..len { - // SAFETY: - // - `i < len < 64`. - // - `src` is valid for `len` reads per caller contract. - let byte = unsafe { src.add(i).read() }; - word |= ((byte & 1) as u64) << i; - } - word - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi8, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi8, - }; - - const LANES: usize = 32; - - /// AVX2 backend: 32 bytes → 32 movemask bits, 2 iterations per u64. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { - // SAFETY: this function is only callable when AVX2 is available - // (enforced by the caller / dispatch gating). - let ones = _mm256_set1_epi8(1); - - while bit_len >= 64 { - // SAFETY: - // - `bit_len >= 64`, so two 32-byte reads from `src` are in bounds. - // - `_mm256_loadu_si256` permits unaligned loads. - let lo = unsafe { _mm256_loadu_si256(src.cast::<__m256i>()) }; - // SAFETY: `src + 32` is valid; `bit_len >= 64`. - let hi = unsafe { _mm256_loadu_si256(src.add(LANES).cast::<__m256i>()) }; - - // cmpeq extracts LSB: 0x01 → 0xFF, 0x00 → 0x00 - let lo_eq = _mm256_cmpeq_epi8(lo, ones); - let hi_eq = _mm256_cmpeq_epi8(hi, ones); - - // movemask takes the MSB of each byte → the comparison result. - let lo_bits = _mm256_movemask_epi8(lo_eq) as u32 as u64; - let hi_bits = _mm256_movemask_epi8(hi_eq) as u32 as u64; - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = lo_bits | (hi_bits << 32); - } - - // SAFETY: destination has capacity; source has `bit_len >= 64`. - unsafe { - dst = dst.add(1); - src = src.add(64); - } - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`, delegate tail to scalar. - unsafe { - scalar::words(dst, src, bit_len); - } - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi8, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi8, - }; - - const LANES: usize = 16; - - /// SSE2 backend: 16 bytes → 16 movemask bits, 4 iterations per u64. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { - let ones = _mm_set1_epi8(1); - - while bit_len >= 64 { - // SAFETY: `bit_len >= 64`, so four 16-byte reads from `src` are in bounds. - // `_mm_loadu_si128` permits unaligned loads. - let v0 = unsafe { _mm_loadu_si128(src.cast::<__m128i>()) }; - let v1 = unsafe { _mm_loadu_si128(src.add(LANES).cast::<__m128i>()) }; - let v2 = unsafe { _mm_loadu_si128(src.add(LANES * 2).cast::<__m128i>()) }; - let v3 = unsafe { _mm_loadu_si128(src.add(LANES * 3).cast::<__m128i>()) }; - - let m0 = _mm_movemask_epi8(_mm_cmpeq_epi8(v0, ones)) as u32 as u64; - let m1 = _mm_movemask_epi8(_mm_cmpeq_epi8(v1, ones)) as u32 as u64; - let m2 = _mm_movemask_epi8(_mm_cmpeq_epi8(v2, ones)) as u32 as u64; - let m3 = _mm_movemask_epi8(_mm_cmpeq_epi8(v3, ones)) as u32 as u64; - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); - } - - // SAFETY: destination has capacity; source has `bit_len >= 64`. - unsafe { - dst = dst.add(1); - src = src.add(64); - } - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`, delegate tail to scalar. - unsafe { - scalar::words(dst, src, bit_len); - } - } -} +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::scalar; - - use core::arch::aarch64::{ - vand_u8, vceq_u8, vdup_n_u8, vget_lane_u64, vld1_u8, vpaddl_u8, vpaddl_u16, vpaddl_u32, - }; - - /// Bit-position masks: [1, 2, 4, 8, 16, 32, 64, 128]. - const BIT_MASKS: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; - - /// NEON backend: packs 64 bytes at a time into one u64. - /// - /// Each 8-byte group contributes bits at `group * 8` offset within the - /// current word, then 8 groups are OR'd together. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { - // SAFETY: constant pointer to static mask array. - let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; - let ones = vdup_n_u8(1); - - while bit_len >= 64 { - // Accumulate 8 groups × 8 bytes → one u64. - let mut word = 0u64; - for group in 0..8 { - // SAFETY: `bit_len >= 64` and `group < 8`, so the load - // is within bounds. `vld1_u8` permits unaligned reads. - let bytes = unsafe { vld1_u8(src.add(group * 8)) }; - - // Expand 1 → 0xFF (0 stays 0x00), then mask to position - // each bit before reducing to a single u64 via pairwise adds. - let is_one = vceq_u8(bytes, ones); - let masked = vand_u8(is_one, bit_masks); - let sum16 = vpaddl_u8(masked); - let sum32 = vpaddl_u16(sum16); - let sum64 = vpaddl_u32(sum32); - - let group_bits = vget_lane_u64::<0>(sum64); - word |= group_bits << (group * 8); - } - - // SAFETY: `dst` points to the current output slot; destination - // has capacity for this write per the caller's contract. - unsafe { - *dst = word; - dst = dst.add(1); - src = src.add(64); - } - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`, delegate tail to scalar. - unsafe { - scalar::words(dst, src, bit_len); - } - } -} +mod neon; mod impls_for_from_bool_iter; mod impls_for_from_bool_slice; diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/avx2.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/avx2.rs new file mode 100644 index 0000000..a14044e --- /dev/null +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/avx2.rs @@ -0,0 +1,61 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi8, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi8, +}; + +const LANES: usize = 32; + +/// AVX2 backend: 32 bytes → 32 movemask bits, 2 iterations per u64. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { + // SAFETY: this function is only callable when AVX2 is available + // (enforced by the caller / dispatch gating). + let ones = _mm256_set1_epi8(1); + + while bit_len >= 64 { + // SAFETY: + // - `bit_len >= 64`, so two 32-byte reads from `src` are in bounds. + // - `_mm256_loadu_si256` permits unaligned loads. + let lo = unsafe { _mm256_loadu_si256(src.cast::<__m256i>()) }; + // SAFETY: `src + 32` is valid; `bit_len >= 64`. + let hi = unsafe { _mm256_loadu_si256(src.add(LANES).cast::<__m256i>()) }; + + // cmpeq extracts LSB: 0x01 → 0xFF, 0x00 → 0x00 + let lo_eq = _mm256_cmpeq_epi8(lo, ones); + let hi_eq = _mm256_cmpeq_epi8(hi, ones); + + // movemask takes the MSB of each byte → the comparison result. + let lo_bits = _mm256_movemask_epi8(lo_eq) as u32 as u64; + let hi_bits = _mm256_movemask_epi8(hi_eq) as u32 as u64; + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = lo_bits | (hi_bits << 32); + } + + // SAFETY: destination has capacity; source has `bit_len >= 64`. + unsafe { + dst = dst.add(1); + src = src.add(64); + } + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`, delegate tail to scalar. + unsafe { + scalar::words(dst, src, bit_len); + } +} diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/neon.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/neon.rs new file mode 100644 index 0000000..95c7fce --- /dev/null +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/neon.rs @@ -0,0 +1,60 @@ +use super::scalar; + +use core::arch::aarch64::{ + vand_u8, vceq_u8, vdup_n_u8, vget_lane_u64, vld1_u8, vpaddl_u8, vpaddl_u16, vpaddl_u32, +}; + +/// Bit-position masks: [1, 2, 4, 8, 16, 32, 64, 128]. +const BIT_MASKS: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; + +/// NEON backend: packs 64 bytes at a time into one u64. +/// +/// Each 8-byte group contributes bits at `group * 8` offset within the +/// current word, then 8 groups are OR'd together. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { + // SAFETY: constant pointer to static mask array. + let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; + let ones = vdup_n_u8(1); + + while bit_len >= 64 { + // Accumulate 8 groups × 8 bytes → one u64. + let mut word = 0u64; + for group in 0..8 { + // SAFETY: `bit_len >= 64` and `group < 8`, so the load + // is within bounds. `vld1_u8` permits unaligned reads. + let bytes = unsafe { vld1_u8(src.add(group * 8)) }; + + // Expand 1 → 0xFF (0 stays 0x00), then mask to position + // each bit before reducing to a single u64 via pairwise adds. + let is_one = vceq_u8(bytes, ones); + let masked = vand_u8(is_one, bit_masks); + let sum16 = vpaddl_u8(masked); + let sum32 = vpaddl_u16(sum16); + let sum64 = vpaddl_u32(sum32); + + let group_bits = vget_lane_u64::<0>(sum64); + word |= group_bits << (group * 8); + } + + // SAFETY: `dst` points to the current output slot; destination + // has capacity for this write per the caller's contract. + unsafe { + *dst = word; + dst = dst.add(1); + src = src.add(64); + } + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`, delegate tail to scalar. + unsafe { + scalar::words(dst, src, bit_len); + } +} diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/scalar.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/scalar.rs new file mode 100644 index 0000000..7c0637c --- /dev/null +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/scalar.rs @@ -0,0 +1,70 @@ +use super::scalar; + +/// Scalar backend: accumulate 64 bytes at a time into one u64. +/// +/// # Safety +/// +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[inline] +pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { + while bit_len >= 64 { + // SAFETY: `bit_len >= 64`, so reading 64 bytes from `src` is valid. + // `dst` points to the next output slot. + unsafe { + *dst = scalar::pack_64(src); + } + // SAFETY: + // - `dst` advances by 1 word; the caller ensures the destination + // has enough capacity for all full-word writes. + // - `src` advances by 64 bytes; `bit_len >= 64` ensures read bounds. + unsafe { + dst = dst.add(1); + src = src.add(64); + } + bit_len -= 64; + } + + if bit_len > 0 { + // SAFETY: `bit_len > 0`, so reading `bit_len` bytes from `src` is valid. + unsafe { + *dst = scalar::pack_partial(src, bit_len); + } + } +} + +/// Pack exactly 64 bytes into one u64 (little-endian: byte i → bit i). +/// +/// # Safety +/// +/// `src` must be valid for reads of 64 u8 values. +#[inline] +unsafe fn pack_64(src: *const u8) -> u64 { + let mut word = 0u64; + for i in 0..64 { + // SAFETY: + // - `i < 64`, offset is in bounds. + // - `src` is valid for 64 reads per caller contract. + let byte = unsafe { src.add(i).read() }; + word |= ((byte & 1) as u64) << i; + } + word +} + +/// Pack fewer than 64 bytes into one u64. +/// +/// # Safety +/// +/// `src` must be valid for reads of `len` u8 values (`len < 64`). +#[inline] +unsafe fn pack_partial(src: *const u8, len: usize) -> u64 { + let mut word = 0u64; + for i in 0..len { + // SAFETY: + // - `i < len < 64`. + // - `src` is valid for `len` reads per caller contract. + let byte = unsafe { src.add(i).read() }; + word |= ((byte & 1) as u64) << i; + } + word +} diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/sse2.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/sse2.rs new file mode 100644 index 0000000..4f29401 --- /dev/null +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/sse2.rs @@ -0,0 +1,54 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi8}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi8, +}; + +const LANES: usize = 16; + +/// SSE2 backend: 16 bytes → 16 movemask bits, 4 iterations per u64. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { + let ones = _mm_set1_epi8(1); + + while bit_len >= 64 { + // SAFETY: `bit_len >= 64`, so four 16-byte reads from `src` are in bounds. + // `_mm_loadu_si128` permits unaligned loads. + let v0 = unsafe { _mm_loadu_si128(src.cast::<__m128i>()) }; + let v1 = unsafe { _mm_loadu_si128(src.add(LANES).cast::<__m128i>()) }; + let v2 = unsafe { _mm_loadu_si128(src.add(LANES * 2).cast::<__m128i>()) }; + let v3 = unsafe { _mm_loadu_si128(src.add(LANES * 3).cast::<__m128i>()) }; + + let m0 = _mm_movemask_epi8(_mm_cmpeq_epi8(v0, ones)) as u32 as u64; + let m1 = _mm_movemask_epi8(_mm_cmpeq_epi8(v1, ones)) as u32 as u64; + let m2 = _mm_movemask_epi8(_mm_cmpeq_epi8(v2, ones)) as u32 as u64; + let m3 = _mm_movemask_epi8(_mm_cmpeq_epi8(v3, ones)) as u32 as u64; + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); + } + + // SAFETY: destination has capacity; source has `bit_len >= 64`. + unsafe { + dst = dst.add(1); + src = src.add(64); + } + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`, delegate tail to scalar. + unsafe { + scalar::words(dst, src, bit_len); + } +} From 5de3844a7f471b3fcb5e1e945a87b834d167dcc1 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 05:06:08 +0000 Subject: [PATCH 15/79] refactor: split pack_str backends --- .../funcs_for_pack_str_core.rs | 337 +----------------- .../funcs_for_pack_str_core/avx2.rs | 79 ++++ .../funcs_for_pack_str_core/neon.rs | 88 +++++ .../funcs_for_pack_str_core/scalar.rs | 58 +++ .../funcs_for_pack_str_core/sse2.rs | 81 +++++ 5 files changed, 310 insertions(+), 333 deletions(-) create mode 100644 src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/avx2.rs create mode 100644 src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/neon.rs create mode 100644 src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/scalar.rs create mode 100644 src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/sse2.rs diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs index d4dd417..eb713ec 100644 --- a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs @@ -78,348 +78,19 @@ unsafe fn dispatch(dst: *mut u64, src: *const u8, bit_len: usize) -> Option<(usi } } -// --------------------------------------------------------------------------- -// Scalar backend -// --------------------------------------------------------------------------- - #[allow(unused)] -mod scalar { - /// Validate-and-pack: process 64 bytes at a time. - /// - /// # Safety - /// - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[inline] - pub(super) unsafe fn words( - mut dst: *mut u64, - mut src: *const u8, - mut bit_len: usize, - ) -> Option<(usize, u8)> { - let mut global_offset = 0usize; - - while bit_len >= 64 { - let mut word = 0u64; - for i in 0..64 { - // SAFETY: `i < 64` and `bit_len >= 64`. - let byte = unsafe { src.add(i).read() }; - let bit = match byte { - b'0' => 0u64, - b'1' => 1u64, - _ => return Some((global_offset + i, byte)), - }; - word |= bit << i; - } - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = word; - dst = dst.add(1); - src = src.add(64); - } - global_offset += 64; - bit_len -= 64; - } - - if bit_len > 0 { - let mut word = 0u64; - for i in 0..bit_len { - // SAFETY: `i < bit_len`. - let byte = unsafe { src.add(i).read() }; - let bit = match byte { - b'0' => 0u64, - b'1' => 1u64, - _ => return Some((global_offset + i, byte)), - }; - word |= bit << i; - } - - // SAFETY: `dst` points to the last output slot. - unsafe { - *dst = word; - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// AVX2 backend -// --------------------------------------------------------------------------- +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, - _mm256_set1_epi8, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, - _mm256_set1_epi8, - }; - - const LANES: usize = 32; - - /// AVX2 backend: validate + pack, 32 bytes / lane. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words( - mut dst: *mut u64, - mut src: *const u8, - mut bit_len: usize, - ) -> Option<(usize, u8)> { - let ones = _mm256_set1_epi8(b'1' as i8); - let zeros = _mm256_set1_epi8(b'0' as i8); - let mut global_offset = 0usize; - - while bit_len >= 64 { - // Load 2 × 32 bytes. - // SAFETY: `bit_len >= 64`, two unaligned 32-byte loads are in bounds. - let lo = unsafe { _mm256_loadu_si256(src.cast::<__m256i>()) }; - let hi = unsafe { _mm256_loadu_si256(src.add(LANES).cast::<__m256i>()) }; - - // Validate: each byte must equal b'0' or b'1'. - // cmpeq(v, b'1') → 0xFF for '1', 0x00 otherwise. - // cmpeq(v, b'0') → 0xFF for '0', 0x00 otherwise. - let lo_ones = _mm256_cmpeq_epi8(lo, ones); - let lo_zeros = _mm256_cmpeq_epi8(lo, zeros); - let hi_ones = _mm256_cmpeq_epi8(hi, ones); - let hi_zeros = _mm256_cmpeq_epi8(hi, zeros); - - let lo_valid = _mm256_or_si256(lo_ones, lo_zeros); - let hi_valid = _mm256_or_si256(hi_ones, hi_zeros); - - let lo_valid_mask = _mm256_movemask_epi8(lo_valid) as u32; - let hi_valid_mask = _mm256_movemask_epi8(hi_valid) as u32; - - if lo_valid_mask != 0xFFFF_FFFF || hi_valid_mask != 0xFFFF_FFFF { - // Fall back to scalar within this 64-byte chunk for exact - // error position. - // SAFETY: src points at this chunk, dst points at current slot. - let (i, b) = - unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); - return Some((global_offset + i, b)); - } - - // Pack: movemask extracts the MSB of cmpeq result. - // b'1' → 0xFF (MSB=1), b'0' → 0x00 (MSB=0). - let lo_bits = _mm256_movemask_epi8(lo_ones) as u32 as u64; - let hi_bits = _mm256_movemask_epi8(hi_ones) as u32 as u64; - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = lo_bits | (hi_bits << 32); - dst = dst.add(1); - src = src.add(64); - } - global_offset += 64; - bit_len -= 64; - } - - // Delegate tail to scalar. - // SAFETY: `bit_len < 64`. - unsafe { - if let Some((i, b)) = scalar::words(dst, src, bit_len) { - return Some((global_offset + i, b)); - } - } - None - } -} - -// --------------------------------------------------------------------------- -// SSE2 backend -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, - }; - - const LANES: usize = 16; - - /// SSE2 backend: validate + pack, 16 bytes / lane. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words( - mut dst: *mut u64, - mut src: *const u8, - mut bit_len: usize, - ) -> Option<(usize, u8)> { - let ones = _mm_set1_epi8(b'1' as i8); - let zeros = _mm_set1_epi8(b'0' as i8); - let mut global_offset = 0usize; - - while bit_len >= 64 { - // Load 4 × 16 bytes. - // SAFETY: `bit_len >= 64`, four unaligned 16-byte loads are in bounds. - let v0 = unsafe { _mm_loadu_si128(src.cast::<__m128i>()) }; - let v1 = unsafe { _mm_loadu_si128(src.add(LANES).cast::<__m128i>()) }; - let v2 = unsafe { _mm_loadu_si128(src.add(LANES * 2).cast::<__m128i>()) }; - let v3 = unsafe { _mm_loadu_si128(src.add(LANES * 3).cast::<__m128i>()) }; - - let ones0 = _mm_cmpeq_epi8(v0, ones); - let zeros0 = _mm_cmpeq_epi8(v0, zeros); - let ones1 = _mm_cmpeq_epi8(v1, ones); - let zeros1 = _mm_cmpeq_epi8(v1, zeros); - let ones2 = _mm_cmpeq_epi8(v2, ones); - let zeros2 = _mm_cmpeq_epi8(v2, zeros); - let ones3 = _mm_cmpeq_epi8(v3, ones); - let zeros3 = _mm_cmpeq_epi8(v3, zeros); - - let valid0 = _mm_or_si128(ones0, zeros0); - let valid1 = _mm_or_si128(ones1, zeros1); - let valid2 = _mm_or_si128(ones2, zeros2); - let valid3 = _mm_or_si128(ones3, zeros3); - - let mask0 = _mm_movemask_epi8(valid0) as u16; - let mask1 = _mm_movemask_epi8(valid1) as u16; - let mask2 = _mm_movemask_epi8(valid2) as u16; - let mask3 = _mm_movemask_epi8(valid3) as u16; - - if mask0 != 0xFFFF || mask1 != 0xFFFF || mask2 != 0xFFFF || mask3 != 0xFFFF { - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - let (i, b) = - unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); - return Some((global_offset + i, b)); - } - - let b0 = _mm_movemask_epi8(ones0) as u32 as u64; - let b1 = _mm_movemask_epi8(ones1) as u32 as u64; - let b2 = _mm_movemask_epi8(ones2) as u32 as u64; - let b3 = _mm_movemask_epi8(ones3) as u32 as u64; - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = b0 | (b1 << 16) | (b2 << 32) | (b3 << 48); - dst = dst.add(1); - src = src.add(64); - } - global_offset += 64; - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`. - unsafe { - if let Some((i, b)) = scalar::words(dst, src, bit_len) { - return Some((global_offset + i, b)); - } - } - None - } -} - -// --------------------------------------------------------------------------- -// NEON backend (uses XOR approach: xor0 serves both validation and extraction) -// --------------------------------------------------------------------------- +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::scalar; - - use core::arch::aarch64::{ - vand_u8, vceq_u8, vdup_n_u8, veor_u8, vget_lane_u64, vld1_u8, vpaddl_u8, vpaddl_u16, - vpaddl_u32, vreinterpret_u64_u8, - }; - - /// Bit-position masks for vpaddl reduction. - const BIT_MASKS: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; - - /// NEON backend: validate + pack, 64 bytes at a time. - /// - /// Uses `(v ^ 0x30) & (v ^ 0x31)` for validation, reusing `v ^ 0x30` - /// (which holds the bit value in LSB) for extraction. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words( - mut dst: *mut u64, - mut src: *const u8, - mut bit_len: usize, - ) -> Option<(usize, u8)> { - let zero_byte = vdup_n_u8(b'0'); - let one_byte = vdup_n_u8(b'1'); - // SAFETY: `BIT_MASKS` is a static array of 8 u8 values, so its pointer is - // valid for reads of 8 bytes. - let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; - let mut global_offset = 0usize; - - while bit_len >= 64 { - // Validate all 8 groups, then pack them into one u64. - let mut word = 0u64; - for group in 0..8 { - // SAFETY: `bit_len >= 64` and `group < 8`, so the - // 8-byte read from `src + group*8` is in bounds. - let v = unsafe { vld1_u8(src.add(group * 8)) }; - - // `xor0 = v ^ 0x30`: - // b'0' → 0x00, b'1' → 0x01 (bit value in LSB) - // `xor1 = v ^ 0x31`: - // b'0' → 0x01, b'1' → 0x00 - // `invalid = xor0 & xor1`: - // valid → 0x00, invalid → non-zero - let xor0 = veor_u8(v, zero_byte); - let xor1 = veor_u8(v, one_byte); - let invalid = vand_u8(xor0, xor1); - - // Cheap all-zero test via reinterpret as u64. - let invalid_u64 = vget_lane_u64::<0>(vreinterpret_u64_u8(invalid)); - if invalid_u64 != 0 { - // Fall back to scalar for exact error position within - // this 64-byte chunk. - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - let (i, b) = - unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); - return Some((global_offset + i, b)); - } - - // Pack: xor0 holds the bit value (0x00 or 0x01). - // Expand 0x01 → 0xFF via vceq_u8 so that vand with - // bit-position masks correctly captures every lane, - // then horizontal pairwise add collapses to one u64. - let is_one = vceq_u8(xor0, vdup_n_u8(1)); - let masked = vand_u8(is_one, bit_masks); - let sum16 = vpaddl_u8(masked); - let sum32 = vpaddl_u16(sum16); - let sum64 = vpaddl_u32(sum32); - - let group_bits = vget_lane_u64::<0>(sum64); - word |= group_bits << (group * 8); - } - - // SAFETY: `dst` points to the current output slot. - unsafe { - *dst = word; - dst = dst.add(1); - src = src.add(64); - } - global_offset += 64; - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`. - unsafe { - if let Some((i, b)) = scalar::words(dst, src, bit_len) { - return Some((global_offset + i, b)); - } - } - None - } -} - +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/avx2.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/avx2.rs new file mode 100644 index 0000000..fa44613 --- /dev/null +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/avx2.rs @@ -0,0 +1,79 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, + _mm256_set1_epi8, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, + _mm256_set1_epi8, +}; + +const LANES: usize = 32; + +/// AVX2 backend: validate + pack, 32 bytes / lane. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words( + mut dst: *mut u64, + mut src: *const u8, + mut bit_len: usize, +) -> Option<(usize, u8)> { + let ones = _mm256_set1_epi8(b'1' as i8); + let zeros = _mm256_set1_epi8(b'0' as i8); + let mut global_offset = 0usize; + + while bit_len >= 64 { + // Load 2 × 32 bytes. + // SAFETY: `bit_len >= 64`, two unaligned 32-byte loads are in bounds. + let lo = unsafe { _mm256_loadu_si256(src.cast::<__m256i>()) }; + let hi = unsafe { _mm256_loadu_si256(src.add(LANES).cast::<__m256i>()) }; + + // Validate: each byte must equal b'0' or b'1'. + // cmpeq(v, b'1') → 0xFF for '1', 0x00 otherwise. + // cmpeq(v, b'0') → 0xFF for '0', 0x00 otherwise. + let lo_ones = _mm256_cmpeq_epi8(lo, ones); + let lo_zeros = _mm256_cmpeq_epi8(lo, zeros); + let hi_ones = _mm256_cmpeq_epi8(hi, ones); + let hi_zeros = _mm256_cmpeq_epi8(hi, zeros); + + let lo_valid = _mm256_or_si256(lo_ones, lo_zeros); + let hi_valid = _mm256_or_si256(hi_ones, hi_zeros); + + let lo_valid_mask = _mm256_movemask_epi8(lo_valid) as u32; + let hi_valid_mask = _mm256_movemask_epi8(hi_valid) as u32; + + if lo_valid_mask != 0xFFFF_FFFF || hi_valid_mask != 0xFFFF_FFFF { + // Fall back to scalar within this 64-byte chunk for exact + // error position. + // SAFETY: src points at this chunk, dst points at current slot. + let (i, b) = unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); + return Some((global_offset + i, b)); + } + + // Pack: movemask extracts the MSB of cmpeq result. + // b'1' → 0xFF (MSB=1), b'0' → 0x00 (MSB=0). + let lo_bits = _mm256_movemask_epi8(lo_ones) as u32 as u64; + let hi_bits = _mm256_movemask_epi8(hi_ones) as u32 as u64; + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = lo_bits | (hi_bits << 32); + dst = dst.add(1); + src = src.add(64); + } + global_offset += 64; + bit_len -= 64; + } + + // Delegate tail to scalar. + // SAFETY: `bit_len < 64`. + unsafe { + if let Some((i, b)) = scalar::words(dst, src, bit_len) { + return Some((global_offset + i, b)); + } + } + None +} diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/neon.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/neon.rs new file mode 100644 index 0000000..04e1575 --- /dev/null +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/neon.rs @@ -0,0 +1,88 @@ +use super::scalar; + +use core::arch::aarch64::{ + vand_u8, vceq_u8, vdup_n_u8, veor_u8, vget_lane_u64, vld1_u8, vpaddl_u8, vpaddl_u16, + vpaddl_u32, vreinterpret_u64_u8, +}; + +/// Bit-position masks for vpaddl reduction. +const BIT_MASKS: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; + +/// NEON backend: validate + pack, 64 bytes at a time. +/// +/// Uses `(v ^ 0x30) & (v ^ 0x31)` for validation, reusing `v ^ 0x30` +/// (which holds the bit value in LSB) for extraction. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words( + mut dst: *mut u64, + mut src: *const u8, + mut bit_len: usize, +) -> Option<(usize, u8)> { + let zero_byte = vdup_n_u8(b'0'); + let one_byte = vdup_n_u8(b'1'); + // SAFETY: `BIT_MASKS` is a static array of 8 u8 values, so its pointer is + // valid for reads of 8 bytes. + let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; + let mut global_offset = 0usize; + + while bit_len >= 64 { + // Validate all 8 groups, then pack them into one u64. + let mut word = 0u64; + for group in 0..8 { + // SAFETY: `bit_len >= 64` and `group < 8`, so the + // 8-byte read from `src + group*8` is in bounds. + let v = unsafe { vld1_u8(src.add(group * 8)) }; + + // `xor0 = v ^ 0x30`: + // b'0' → 0x00, b'1' → 0x01 (bit value in LSB) + // `xor1 = v ^ 0x31`: + // b'0' → 0x01, b'1' → 0x00 + // `invalid = xor0 & xor1`: + // valid → 0x00, invalid → non-zero + let xor0 = veor_u8(v, zero_byte); + let xor1 = veor_u8(v, one_byte); + let invalid = vand_u8(xor0, xor1); + + // Cheap all-zero test via reinterpret as u64. + let invalid_u64 = vget_lane_u64::<0>(vreinterpret_u64_u8(invalid)); + if invalid_u64 != 0 { + // Fall back to scalar for exact error position within + // this 64-byte chunk. + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + let (i, b) = + unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); + return Some((global_offset + i, b)); + } + + // Pack: xor0 holds the bit value (0x00 or 0x01). + // Expand 0x01 → 0xFF via vceq_u8 so that vand with + // bit-position masks correctly captures every lane, + // then horizontal pairwise add collapses to one u64. + let is_one = vceq_u8(xor0, vdup_n_u8(1)); + let masked = vand_u8(is_one, bit_masks); + let sum16 = vpaddl_u8(masked); + let sum32 = vpaddl_u16(sum16); + let sum64 = vpaddl_u32(sum32); + + let group_bits = vget_lane_u64::<0>(sum64); + word |= group_bits << (group * 8); + } + + // SAFETY: `dst` points to the current output slot. + unsafe { + *dst = word; + dst = dst.add(1); + src = src.add(64); + } + global_offset += 64; + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`. + unsafe { + if let Some((i, b)) = scalar::words(dst, src, bit_len) { + return Some((global_offset + i, b)); + } + } + None +} diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/scalar.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/scalar.rs new file mode 100644 index 0000000..eb9d2b2 --- /dev/null +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/scalar.rs @@ -0,0 +1,58 @@ +/// Validate-and-pack: process 64 bytes at a time. +/// +/// # Safety +/// +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[inline] +pub(super) unsafe fn words( + mut dst: *mut u64, + mut src: *const u8, + mut bit_len: usize, +) -> Option<(usize, u8)> { + let mut global_offset = 0usize; + + while bit_len >= 64 { + let mut word = 0u64; + for i in 0..64 { + // SAFETY: `i < 64` and `bit_len >= 64`. + let byte = unsafe { src.add(i).read() }; + let bit = match byte { + b'0' => 0u64, + b'1' => 1u64, + _ => return Some((global_offset + i, byte)), + }; + word |= bit << i; + } + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = word; + dst = dst.add(1); + src = src.add(64); + } + global_offset += 64; + bit_len -= 64; + } + + if bit_len > 0 { + let mut word = 0u64; + for i in 0..bit_len { + // SAFETY: `i < bit_len`. + let byte = unsafe { src.add(i).read() }; + let bit = match byte { + b'0' => 0u64, + b'1' => 1u64, + _ => return Some((global_offset + i, byte)), + }; + word |= bit << i; + } + + // SAFETY: `dst` points to the last output slot. + unsafe { + *dst = word; + } + } + + None +} diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/sse2.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/sse2.rs new file mode 100644 index 0000000..d947c44 --- /dev/null +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/sse2.rs @@ -0,0 +1,81 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, +}; + +const LANES: usize = 16; + +/// SSE2 backend: validate + pack, 16 bytes / lane. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words( + mut dst: *mut u64, + mut src: *const u8, + mut bit_len: usize, +) -> Option<(usize, u8)> { + let ones = _mm_set1_epi8(b'1' as i8); + let zeros = _mm_set1_epi8(b'0' as i8); + let mut global_offset = 0usize; + + while bit_len >= 64 { + // Load 4 × 16 bytes. + // SAFETY: `bit_len >= 64`, four unaligned 16-byte loads are in bounds. + let v0 = unsafe { _mm_loadu_si128(src.cast::<__m128i>()) }; + let v1 = unsafe { _mm_loadu_si128(src.add(LANES).cast::<__m128i>()) }; + let v2 = unsafe { _mm_loadu_si128(src.add(LANES * 2).cast::<__m128i>()) }; + let v3 = unsafe { _mm_loadu_si128(src.add(LANES * 3).cast::<__m128i>()) }; + + let ones0 = _mm_cmpeq_epi8(v0, ones); + let zeros0 = _mm_cmpeq_epi8(v0, zeros); + let ones1 = _mm_cmpeq_epi8(v1, ones); + let zeros1 = _mm_cmpeq_epi8(v1, zeros); + let ones2 = _mm_cmpeq_epi8(v2, ones); + let zeros2 = _mm_cmpeq_epi8(v2, zeros); + let ones3 = _mm_cmpeq_epi8(v3, ones); + let zeros3 = _mm_cmpeq_epi8(v3, zeros); + + let valid0 = _mm_or_si128(ones0, zeros0); + let valid1 = _mm_or_si128(ones1, zeros1); + let valid2 = _mm_or_si128(ones2, zeros2); + let valid3 = _mm_or_si128(ones3, zeros3); + + let mask0 = _mm_movemask_epi8(valid0) as u16; + let mask1 = _mm_movemask_epi8(valid1) as u16; + let mask2 = _mm_movemask_epi8(valid2) as u16; + let mask3 = _mm_movemask_epi8(valid3) as u16; + + if mask0 != 0xFFFF || mask1 != 0xFFFF || mask2 != 0xFFFF || mask3 != 0xFFFF { + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + let (i, b) = unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); + return Some((global_offset + i, b)); + } + + let b0 = _mm_movemask_epi8(ones0) as u32 as u64; + let b1 = _mm_movemask_epi8(ones1) as u32 as u64; + let b2 = _mm_movemask_epi8(ones2) as u32 as u64; + let b3 = _mm_movemask_epi8(ones3) as u32 as u64; + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = b0 | (b1 << 16) | (b2 << 32) | (b3 << 48); + dst = dst.add(1); + src = src.add(64); + } + global_offset += 64; + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`. + unsafe { + if let Some((i, b)) = scalar::words(dst, src, bit_len) { + return Some((global_offset + i, b)); + } + } + None +} From 23f042269263b8ccbb8e334f4f79b307aeef2cb5 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 05:21:06 +0000 Subject: [PATCH 16/79] refactor: split binary backends --- .../words_arith/funcs_for_binary_core.rs | 267 +----------------- .../words_arith/funcs_for_binary_core/avx2.rs | 73 +++++ .../words_arith/funcs_for_binary_core/neon.rs | 69 +++++ .../funcs_for_binary_core/scalar.rs | 41 +++ .../words_arith/funcs_for_binary_core/sse2.rs | 71 +++++ 5 files changed, 258 insertions(+), 263 deletions(-) create mode 100644 src/traits/words_arith/funcs_for_binary_core/avx2.rs create mode 100644 src/traits/words_arith/funcs_for_binary_core/neon.rs create mode 100644 src/traits/words_arith/funcs_for_binary_core/scalar.rs create mode 100644 src/traits/words_arith/funcs_for_binary_core/sse2.rs diff --git a/src/traits/words_arith/funcs_for_binary_core.rs b/src/traits/words_arith/funcs_for_binary_core.rs index 6b5a546..6c42bbe 100644 --- a/src/traits/words_arith/funcs_for_binary_core.rs +++ b/src/traits/words_arith/funcs_for_binary_core.rs @@ -98,277 +98,18 @@ unsafe fn dispatch(dst: *mut u64, lhs: *const u64, rhs: *const u64 } #[allow(unused)] -mod scalar { - use super::*; - - #[inline] - fn apply(lhs: u64, rhs: u64) -> u64 { - match OP { - OP_AND => lhs & rhs, - OP_OR => lhs | rhs, - OP_XOR => lhs ^ rhs, - _ => unreachable!("unsupported binary bit operation"), - } - } - - /// Scalar backend for `dst[i] = lhs[i] OP rhs[i]`. - /// - /// Supports `dst == lhs`. - /// - /// # Safety - /// - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. - /// - `dst` must not overlap `rhs`. - #[inline] - pub(super) unsafe fn words( - dst: *mut u64, - lhs: *const u64, - rhs: *const u64, - len: usize, - ) { - for i in 0..len { - // SAFETY: - // - `i < len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == lhs` is safe because both operands are read before writing `dst[i]`. - unsafe { - let lhs_word = lhs.add(i).read(); - let rhs_word = rhs.add(i).read(); - dst.add(i).write(apply::(lhs_word, rhs_word)); - } - } - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_and_si128, _mm_loadu_si128, _mm_or_si128, _mm_storeu_si128, _mm_xor_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_and_si128, _mm_loadu_si128, _mm_or_si128, _mm_storeu_si128, _mm_xor_si128, - }; - - const LANES: usize = 2; - - #[inline] - fn apply(lhs: __m128i, rhs: __m128i) -> __m128i { - match OP { - OP_AND => unsafe { _mm_and_si128(lhs, rhs) }, - OP_OR => unsafe { _mm_or_si128(lhs, rhs) }, - OP_XOR => unsafe { _mm_xor_si128(lhs, rhs) }, - _ => unreachable!("unsupported binary bit operation"), - } - } - - /// SSE2 backend for `dst[i] = lhs[i] OP rhs[i]`. - /// - /// Supports `dst == lhs`. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. - /// - `dst` must not overlap `rhs`. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words( - dst: *mut u64, - lhs: *const u64, - rhs: *const u64, - len: usize, - ) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - Unaligned load/store intrinsics permit unaligned access. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == lhs` is safe because loads happen before the store. - unsafe { - let lhs_vec = _mm_loadu_si128(lhs.add(offset).cast::<__m128i>()); - let rhs_vec = _mm_loadu_si128(rhs.add(offset).cast::<__m128i>()); - let out_vec = apply::(lhs_vec, rhs_vec); - - _mm_storeu_si128(dst.add(offset).cast::<__m128i>(), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); - } - } -} +mod sse2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_and_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_storeu_si256, - _mm256_xor_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_and_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_storeu_si256, - _mm256_xor_si256, - }; - - const LANES: usize = 4; - - #[inline] - fn apply(lhs: __m256i, rhs: __m256i) -> __m256i { - match OP { - OP_AND => unsafe { _mm256_and_si256(lhs, rhs) }, - OP_OR => unsafe { _mm256_or_si256(lhs, rhs) }, - OP_XOR => unsafe { _mm256_xor_si256(lhs, rhs) }, - _ => unreachable!("unsupported binary bit operation"), - } - } - - /// AVX2 backend for `dst[i] = lhs[i] OP rhs[i]`. - /// - /// Supports `dst == lhs`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. - /// - `dst` must not overlap `rhs`. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words( - dst: *mut u64, - lhs: *const u64, - rhs: *const u64, - len: usize, - ) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm256_loadu_si256` and `_mm256_storeu_si256` permit unaligned access. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == lhs` is safe because both loads happen before the store. - unsafe { - let lhs_vec = _mm256_loadu_si256(lhs.add(offset).cast::<__m256i>()); - let rhs_vec = _mm256_loadu_si256(rhs.add(offset).cast::<__m256i>()); - let out_vec = apply::(lhs_vec, rhs_vec); - - _mm256_storeu_si256(dst.add(offset).cast::<__m256i>(), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); - } - } -} +mod avx2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::*; - - use core::arch::aarch64::{uint64x2_t, vandq_u64, veorq_u64, vld1q_u64, vorrq_u64, vst1q_u64}; - - const LANES: usize = 2; - - #[inline] - fn apply(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { - // SAFETY: - // - This helper is only called from `words`, which has - // `#[target_feature(enable = "neon")]`. - // - The dispatch path only reaches `words` when NEON is enabled. - unsafe { - match OP { - OP_AND => vandq_u64(lhs, rhs), - OP_OR => vorrq_u64(lhs, rhs), - OP_XOR => veorq_u64(lhs, rhs), - _ => unreachable!("unsupported binary bit operation"), - } - } - } - - /// NEON backend for `dst[i] = lhs[i] OP rhs[i]`. - /// - /// Supports `dst == lhs`. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. - /// - `dst` must not overlap `rhs`. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words( - dst: *mut u64, - lhs: *const u64, - rhs: *const u64, - len: usize, - ) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - Each NEON vector reads/writes 2 u64 values. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == lhs` is safe because both loads happen before the store. - unsafe { - let lhs_vec = vld1q_u64(lhs.add(offset)); - let rhs_vec = vld1q_u64(rhs.add(offset)); - let out_vec = apply::(lhs_vec, rhs_vec); - - vst1q_u64(dst.add(offset), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); - } - } -} - +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_arith/funcs_for_binary_core/avx2.rs b/src/traits/words_arith/funcs_for_binary_core/avx2.rs new file mode 100644 index 0000000..c8ee3a0 --- /dev/null +++ b/src/traits/words_arith/funcs_for_binary_core/avx2.rs @@ -0,0 +1,73 @@ +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_and_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_storeu_si256, + _mm256_xor_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_and_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_storeu_si256, + _mm256_xor_si256, +}; + +const LANES: usize = 4; + +#[inline] +fn apply(lhs: __m256i, rhs: __m256i) -> __m256i { + match OP { + OP_AND => unsafe { _mm256_and_si256(lhs, rhs) }, + OP_OR => unsafe { _mm256_or_si256(lhs, rhs) }, + OP_XOR => unsafe { _mm256_xor_si256(lhs, rhs) }, + _ => unreachable!("unsupported binary bit operation"), + } +} + +/// AVX2 backend for `dst[i] = lhs[i] OP rhs[i]`. +/// +/// Supports `dst == lhs`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. +/// - `dst` must not overlap `rhs`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words( + dst: *mut u64, + lhs: *const u64, + rhs: *const u64, + len: usize, +) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm256_loadu_si256` and `_mm256_storeu_si256` permit unaligned access. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == lhs` is safe because both loads happen before the store. + unsafe { + let lhs_vec = _mm256_loadu_si256(lhs.add(offset).cast::<__m256i>()); + let rhs_vec = _mm256_loadu_si256(rhs.add(offset).cast::<__m256i>()); + let out_vec = apply::(lhs_vec, rhs_vec); + + _mm256_storeu_si256(dst.add(offset).cast::<__m256i>(), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_binary_core/neon.rs b/src/traits/words_arith/funcs_for_binary_core/neon.rs new file mode 100644 index 0000000..69d12d0 --- /dev/null +++ b/src/traits/words_arith/funcs_for_binary_core/neon.rs @@ -0,0 +1,69 @@ +use super::*; + +use core::arch::aarch64::{uint64x2_t, vandq_u64, veorq_u64, vld1q_u64, vorrq_u64, vst1q_u64}; + +const LANES: usize = 2; + +#[inline] +fn apply(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { + // SAFETY: + // - This helper is only called from `words`, which has + // `#[target_feature(enable = "neon")]`. + // - The dispatch path only reaches `words` when NEON is enabled. + unsafe { + match OP { + OP_AND => vandq_u64(lhs, rhs), + OP_OR => vorrq_u64(lhs, rhs), + OP_XOR => veorq_u64(lhs, rhs), + _ => unreachable!("unsupported binary bit operation"), + } + } +} + +/// NEON backend for `dst[i] = lhs[i] OP rhs[i]`. +/// +/// Supports `dst == lhs`. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. +/// - `dst` must not overlap `rhs`. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words( + dst: *mut u64, + lhs: *const u64, + rhs: *const u64, + len: usize, +) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - Each NEON vector reads/writes 2 u64 values. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == lhs` is safe because both loads happen before the store. + unsafe { + let lhs_vec = vld1q_u64(lhs.add(offset)); + let rhs_vec = vld1q_u64(rhs.add(offset)); + let out_vec = apply::(lhs_vec, rhs_vec); + + vst1q_u64(dst.add(offset), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_binary_core/scalar.rs b/src/traits/words_arith/funcs_for_binary_core/scalar.rs new file mode 100644 index 0000000..cd47d09 --- /dev/null +++ b/src/traits/words_arith/funcs_for_binary_core/scalar.rs @@ -0,0 +1,41 @@ +use super::*; + +#[inline] +fn apply(lhs: u64, rhs: u64) -> u64 { + match OP { + OP_AND => lhs & rhs, + OP_OR => lhs | rhs, + OP_XOR => lhs ^ rhs, + _ => unreachable!("unsupported binary bit operation"), + } +} + +/// Scalar backend for `dst[i] = lhs[i] OP rhs[i]`. +/// +/// Supports `dst == lhs`. +/// +/// # Safety +/// +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. +/// - `dst` must not overlap `rhs`. +#[inline] +pub(super) unsafe fn words( + dst: *mut u64, + lhs: *const u64, + rhs: *const u64, + len: usize, +) { + for i in 0..len { + // SAFETY: + // - `i < len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == lhs` is safe because both operands are read before writing `dst[i]`. + unsafe { + let lhs_word = lhs.add(i).read(); + let rhs_word = rhs.add(i).read(); + dst.add(i).write(apply::(lhs_word, rhs_word)); + } + } +} diff --git a/src/traits/words_arith/funcs_for_binary_core/sse2.rs b/src/traits/words_arith/funcs_for_binary_core/sse2.rs new file mode 100644 index 0000000..999c626 --- /dev/null +++ b/src/traits/words_arith/funcs_for_binary_core/sse2.rs @@ -0,0 +1,71 @@ +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_and_si128, _mm_loadu_si128, _mm_or_si128, _mm_storeu_si128, _mm_xor_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_and_si128, _mm_loadu_si128, _mm_or_si128, _mm_storeu_si128, _mm_xor_si128, +}; + +const LANES: usize = 2; + +#[inline] +fn apply(lhs: __m128i, rhs: __m128i) -> __m128i { + match OP { + OP_AND => unsafe { _mm_and_si128(lhs, rhs) }, + OP_OR => unsafe { _mm_or_si128(lhs, rhs) }, + OP_XOR => unsafe { _mm_xor_si128(lhs, rhs) }, + _ => unreachable!("unsupported binary bit operation"), + } +} + +/// SSE2 backend for `dst[i] = lhs[i] OP rhs[i]`. +/// +/// Supports `dst == lhs`. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. +/// - `dst` must not overlap `rhs`. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words( + dst: *mut u64, + lhs: *const u64, + rhs: *const u64, + len: usize, +) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - Unaligned load/store intrinsics permit unaligned access. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == lhs` is safe because loads happen before the store. + unsafe { + let lhs_vec = _mm_loadu_si128(lhs.add(offset).cast::<__m128i>()); + let rhs_vec = _mm_loadu_si128(rhs.add(offset).cast::<__m128i>()); + let out_vec = apply::(lhs_vec, rhs_vec); + + _mm_storeu_si128(dst.add(offset).cast::<__m128i>(), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); + } +} From 23fb110bb6731dd7bee8973084b7c148cc0db6a4 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 06:51:34 +0000 Subject: [PATCH 17/79] fix: cover binary codegen variants --- scripts/codegen_harness.rs | 12 ++++++++++++ scripts/codegen_roots.toml | 10 ++++++++++ scripts/kernel_inventory.toml | 2 +- scripts/tests/test_codegen.py | 10 ++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/codegen_harness.rs b/scripts/codegen_harness.rs index 6390196..9d171de 100644 --- a/scripts/codegen_harness.rs +++ b/scripts/codegen_harness.rs @@ -22,6 +22,18 @@ pub unsafe fn codegen_bit_string_xor(lhs: *const BitString, rhs: *const BitStrin unsafe { (&*lhs).xor(&*rhs).unwrap() } } +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_and(lhs: *const BitString, rhs: *const BitString) -> BitString { + unsafe { (&*lhs).and(&*rhs).unwrap() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_or(lhs: *const BitString, rhs: *const BitString) -> BitString { + unsafe { (&*lhs).or(&*rhs).unwrap() } +} + #[unsafe(no_mangle)] #[inline(never)] pub unsafe fn codegen_bit_string_not(value: *const BitString) -> BitString { diff --git a/scripts/codegen_roots.toml b/scripts/codegen_roots.toml index 81e3762..fe07bd3 100644 --- a/scripts/codegen_roots.toml +++ b/scripts/codegen_roots.toml @@ -13,6 +13,16 @@ symbol = "codegen_bit_string_xor" artifact = "harness" targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] +[[root]] +symbol = "codegen_bit_string_and" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_string_or" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + [[root]] symbol = "codegen_bit_string_not" artifact = "harness" diff --git a/scripts/kernel_inventory.toml b/scripts/kernel_inventory.toml index 1d796a2..96ee7d8 100644 --- a/scripts/kernel_inventory.toml +++ b/scripts/kernel_inventory.toml @@ -14,7 +14,7 @@ roots = ["codegen_bit_string_from_str"] [[kernel]] path = "src/traits/words_arith/funcs_for_binary_core.rs" coverage = "caller" -roots = ["codegen_bit_string_xor"] +roots = ["codegen_bit_string_and", "codegen_bit_string_or", "codegen_bit_string_xor"] [[kernel]] path = "src/traits/words_arith/funcs_for_not_core.rs" diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 395265a..2ce2bb0 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -102,6 +102,16 @@ def test_inventory_has_root_for_each_kernel(self) -> None: self.assertTrue(kernel["roots"]) self.assertLessEqual(set(kernel["roots"]), roots) + binary = next( + kernel + for kernel in inventory + if kernel["path"] == "src/traits/words_arith/funcs_for_binary_core.rs" + ) + self.assertEqual( + set(binary["roots"]), + {"codegen_bit_string_and", "codegen_bit_string_or", "codegen_bit_string_xor"}, + ) + def test_independent_backend_symbols_are_discovered_from_inventory_modules(self) -> None: function = parse("0000 :\n 0: c3 ret") functions = { From bbecf6639fd0837a18d662a9e7bddb3681193a3c Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 07:05:26 +0000 Subject: [PATCH 18/79] refactor: split not backends --- src/traits/words_arith/funcs_for_not_core.rs | 205 +----------------- .../words_arith/funcs_for_not_core/avx2.rs | 58 +++++ .../words_arith/funcs_for_not_core/neon.rs | 55 +++++ .../words_arith/funcs_for_not_core/scalar.rs | 22 ++ .../words_arith/funcs_for_not_core/sse2.rs | 56 +++++ 5 files changed, 195 insertions(+), 201 deletions(-) create mode 100644 src/traits/words_arith/funcs_for_not_core/avx2.rs create mode 100644 src/traits/words_arith/funcs_for_not_core/neon.rs create mode 100644 src/traits/words_arith/funcs_for_not_core/scalar.rs create mode 100644 src/traits/words_arith/funcs_for_not_core/sse2.rs diff --git a/src/traits/words_arith/funcs_for_not_core.rs b/src/traits/words_arith/funcs_for_not_core.rs index d619286..9f84a10 100644 --- a/src/traits/words_arith/funcs_for_not_core.rs +++ b/src/traits/words_arith/funcs_for_not_core.rs @@ -91,216 +91,19 @@ unsafe fn dispatch(dst: *mut u64, src: *const u64, len: usize) { } #[allow(unused)] -mod scalar { - /// Scalar backend for `dst[i] = !src[i]`. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `src` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[inline] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { - for i in 0..len { - // SAFETY: - // - `i < len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == src` is safe because `src[i]` is read before `dst[i]` is written. - unsafe { - let word = src.add(i).read(); - dst.add(i).write(!word); - } - } - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_setzero_si256, _mm256_storeu_si256, - _mm256_xor_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_setzero_si256, _mm256_storeu_si256, - _mm256_xor_si256, - }; - - const LANES: usize = 4; - - /// AVX2 backend for `dst[i] = !src[i]`. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `src` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm256_loadu_si256` and `_mm256_storeu_si256` permit unaligned access. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == src` is safe because load happens before store. - unsafe { - let zero = _mm256_setzero_si256(); - let all_ones = _mm256_cmpeq_epi8(zero, zero); - let src_vec = _mm256_loadu_si256(src.add(offset).cast::<__m256i>()); - let out_vec = _mm256_xor_si256(src_vec, all_ones); - - _mm256_storeu_si256(dst.add(offset).cast::<__m256i>(), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words(dst.add(done), src.add(done), len - done); - } - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_setzero_si128, _mm_storeu_si128, - _mm_xor_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_setzero_si128, _mm_storeu_si128, - _mm_xor_si128, - }; - - const LANES: usize = 2; - - /// SSE2 backend for `dst[i] = !src[i]`. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `src` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm_loadu_si128` and `_mm_storeu_si128` permit unaligned access. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == src` is safe because load happens before store. - unsafe { - let zero = _mm_setzero_si128(); - let all_ones = _mm_cmpeq_epi8(zero, zero); - let src_vec = _mm_loadu_si128(src.add(offset).cast::<__m128i>()); - let out_vec = _mm_xor_si128(src_vec, all_ones); - - _mm_storeu_si128(dst.add(offset).cast::<__m128i>(), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words(dst.add(done), src.add(done), len - done); - } - } -} +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::scalar; - - use core::arch::aarch64::{uint64x2_t, vdupq_n_u64, veorq_u64, vld1q_u64, vst1q_u64}; - - const LANES: usize = 2; - - #[inline] - fn not_vec(src: uint64x2_t) -> uint64x2_t { - // SAFETY: - // - This helper is only called from `words`, which has - // `#[target_feature(enable = "neon")]`. - // - The dispatch path only reaches `words` when NEON is enabled. - unsafe { veorq_u64(src, vdupq_n_u64(u64::MAX)) } - } - - /// NEON backend for `dst[i] = !src[i]`. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `src` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - Each NEON vector reads/writes 2 u64 values. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == src` is safe because load happens before store. - unsafe { - let src_vec = vld1q_u64(src.add(offset)); - let out_vec = not_vec(src_vec); - - vst1q_u64(dst.add(offset), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words(dst.add(done), src.add(done), len - done); - } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_arith/funcs_for_not_core/avx2.rs b/src/traits/words_arith/funcs_for_not_core/avx2.rs new file mode 100644 index 0000000..9638103 --- /dev/null +++ b/src/traits/words_arith/funcs_for_not_core/avx2.rs @@ -0,0 +1,58 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_setzero_si256, _mm256_storeu_si256, + _mm256_xor_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_setzero_si256, _mm256_storeu_si256, + _mm256_xor_si256, +}; + +const LANES: usize = 4; + +/// AVX2 backend for `dst[i] = !src[i]`. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `src` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm256_loadu_si256` and `_mm256_storeu_si256` permit unaligned access. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == src` is safe because load happens before store. + unsafe { + let zero = _mm256_setzero_si256(); + let all_ones = _mm256_cmpeq_epi8(zero, zero); + let src_vec = _mm256_loadu_si256(src.add(offset).cast::<__m256i>()); + let out_vec = _mm256_xor_si256(src_vec, all_ones); + + _mm256_storeu_si256(dst.add(offset).cast::<__m256i>(), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words(dst.add(done), src.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_not_core/neon.rs b/src/traits/words_arith/funcs_for_not_core/neon.rs new file mode 100644 index 0000000..dddfcef --- /dev/null +++ b/src/traits/words_arith/funcs_for_not_core/neon.rs @@ -0,0 +1,55 @@ +use super::scalar; + +use core::arch::aarch64::{uint64x2_t, vdupq_n_u64, veorq_u64, vld1q_u64, vst1q_u64}; + +const LANES: usize = 2; + +#[inline] +fn not_vec(src: uint64x2_t) -> uint64x2_t { + // SAFETY: + // - This helper is only called from `words`, which has + // `#[target_feature(enable = "neon")]`. + // - The dispatch path only reaches `words` when NEON is enabled. + unsafe { veorq_u64(src, vdupq_n_u64(u64::MAX)) } +} + +/// NEON backend for `dst[i] = !src[i]`. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `src` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - Each NEON vector reads/writes 2 u64 values. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == src` is safe because load happens before store. + unsafe { + let src_vec = vld1q_u64(src.add(offset)); + let out_vec = not_vec(src_vec); + + vst1q_u64(dst.add(offset), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words(dst.add(done), src.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_not_core/scalar.rs b/src/traits/words_arith/funcs_for_not_core/scalar.rs new file mode 100644 index 0000000..dd13c56 --- /dev/null +++ b/src/traits/words_arith/funcs_for_not_core/scalar.rs @@ -0,0 +1,22 @@ +/// Scalar backend for `dst[i] = !src[i]`. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `src` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[inline] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { + for i in 0..len { + // SAFETY: + // - `i < len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == src` is safe because `src[i]` is read before `dst[i]` is written. + unsafe { + let word = src.add(i).read(); + dst.add(i).write(!word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_not_core/sse2.rs b/src/traits/words_arith/funcs_for_not_core/sse2.rs new file mode 100644 index 0000000..82efcdd --- /dev/null +++ b/src/traits/words_arith/funcs_for_not_core/sse2.rs @@ -0,0 +1,56 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_setzero_si128, _mm_storeu_si128, _mm_xor_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_setzero_si128, _mm_storeu_si128, _mm_xor_si128, +}; + +const LANES: usize = 2; + +/// SSE2 backend for `dst[i] = !src[i]`. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `src` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm_loadu_si128` and `_mm_storeu_si128` permit unaligned access. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == src` is safe because load happens before store. + unsafe { + let zero = _mm_setzero_si128(); + let all_ones = _mm_cmpeq_epi8(zero, zero); + let src_vec = _mm_loadu_si128(src.add(offset).cast::<__m128i>()); + let out_vec = _mm_xor_si128(src_vec, all_ones); + + _mm_storeu_si128(dst.add(offset).cast::<__m128i>(), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words(dst.add(done), src.add(done), len - done); + } +} From 8b6604d59f451786b75e273e5cfb3da2d4760cbb Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 07:14:32 +0000 Subject: [PATCH 19/79] refactor: split shl backends --- src/traits/words_arith/funcs_for_shl_core.rs | 309 +----------------- .../words_arith/funcs_for_shl_core/avx2.rs | 88 +++++ .../words_arith/funcs_for_shl_core/neon.rs | 100 ++++++ .../words_arith/funcs_for_shl_core/scalar.rs | 23 ++ .../words_arith/funcs_for_shl_core/sse2.rs | 86 +++++ 5 files changed, 301 insertions(+), 305 deletions(-) create mode 100644 src/traits/words_arith/funcs_for_shl_core/avx2.rs create mode 100644 src/traits/words_arith/funcs_for_shl_core/neon.rs create mode 100644 src/traits/words_arith/funcs_for_shl_core/scalar.rs create mode 100644 src/traits/words_arith/funcs_for_shl_core/sse2.rs diff --git a/src/traits/words_arith/funcs_for_shl_core.rs b/src/traits/words_arith/funcs_for_shl_core.rs index 42ab3dc..e933b47 100644 --- a/src/traits/words_arith/funcs_for_shl_core.rs +++ b/src/traits/words_arith/funcs_for_shl_core.rs @@ -130,320 +130,19 @@ unsafe fn scalar_word(src: *const u64, word_len: usize, dst_index: usize, amount } #[allow(unused)] -mod scalar { - use super::scalar_word; - - /// Scalar backend for word-level left shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - for dst_index in (0..word_len).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - Descending order makes `dst == src` safe for left shift. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - const LANES: usize = 4; - - /// AVX2 backend for word-level left shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in (0..word_len).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_start = word_shift + usize::from(bit_shift != 0); - let mut end = word_len; - - // Process the vectorizable suffix from high to low. - while end >= bulk_start + LANES { - let dst_start = end - LANES; - let src_start = dst_start - word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES <= word_len`. - // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. - // - Unaligned load/store intrinsics permit unaligned access. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from high to low. - unsafe { - let cur = _mm256_loadu_si256(src.add(src_start).cast::<__m256i>()); - let out = if bit_shift == 0 { - cur - } else { - let prev = _mm256_loadu_si256(src.add(src_start - 1).cast::<__m256i>()); - let left_count: __m128i = _mm_cvtsi64_si128(bit_shift as i64); - let right_count: __m128i = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); - - let left = _mm256_sll_epi64(cur, left_count); - let right = _mm256_srl_epi64(prev, right_count); - - _mm256_or_si256(left, right) - }; - - _mm256_storeu_si256(dst.add(dst_start).cast::<__m256i>(), out); - } - - end = dst_start; - } - - // Scalar tail below the vectorized suffix, still high to low. - for dst_index in (0..end).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - const LANES: usize = 2; - - /// SSE2 backend for word-level left shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in (0..word_len).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_start = word_shift + usize::from(bit_shift != 0); - let mut end = word_len; - - while end >= bulk_start + LANES { - let dst_start = end - LANES; - let src_start = dst_start - word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES <= word_len`. - // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. - // - Unaligned load/store intrinsics permit unaligned access. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from high to low. - unsafe { - let cur = _mm_loadu_si128(src.add(src_start).cast::<__m128i>()); - let out = if bit_shift == 0 { - cur - } else { - let prev = _mm_loadu_si128(src.add(src_start - 1).cast::<__m128i>()); - let left_count = _mm_cvtsi64_si128(bit_shift as i64); - let right_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); - - let left = _mm_sll_epi64(cur, left_count); - let right = _mm_srl_epi64(prev, right_count); - - _mm_or_si128(left, right) - }; - - _mm_storeu_si128(dst.add(dst_start).cast::<__m128i>(), out); - } - - end = dst_start; - } - - for dst_index in (0..end).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - use core::arch::aarch64::{ - int64x2_t, uint64x2_t, vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64, - }; - - const LANES: usize = 2; - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn shl_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vshlq_u64(src, vdupq_n_s64(amount as i64) as int64x2_t) } - } - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn shr_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vshlq_u64(src, vdupq_n_s64(-(amount as i64)) as int64x2_t) } - } - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn or_vec(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vorrq_u64(lhs, rhs) } - } - - /// NEON backend for word-level left shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in (0..word_len).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_start = word_shift + usize::from(bit_shift != 0); - let mut end = word_len; - - while end >= bulk_start + LANES { - let dst_start = end - LANES; - let src_start = dst_start - word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES <= word_len`. - // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. - // - `vld1q_u64` reads exactly 2 u64 values. - // - `vst1q_u64` writes exactly 2 u64 values. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from high to low. - unsafe { - let cur = vld1q_u64(src.add(src_start)); - let out = if bit_shift == 0 { - cur - } else { - let prev = vld1q_u64(src.add(src_start - 1)); - let left = shl_vec(cur, bit_shift); - let right = shr_vec(prev, WORD_BITS - bit_shift); - - or_vec(left, right) - }; - - vst1q_u64(dst.add(dst_start), out); - } - - end = dst_start; - } - - for dst_index in (0..end).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_arith/funcs_for_shl_core/avx2.rs b/src/traits/words_arith/funcs_for_shl_core/avx2.rs new file mode 100644 index 0000000..b13adff --- /dev/null +++ b/src/traits/words_arith/funcs_for_shl_core/avx2.rs @@ -0,0 +1,88 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +const LANES: usize = 4; + +/// AVX2 backend for word-level left shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in (0..word_len).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_start = word_shift + usize::from(bit_shift != 0); + let mut end = word_len; + + // Process the vectorizable suffix from high to low. + while end >= bulk_start + LANES { + let dst_start = end - LANES; + let src_start = dst_start - word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES <= word_len`. + // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. + // - Unaligned load/store intrinsics permit unaligned access. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from high to low. + unsafe { + let cur = _mm256_loadu_si256(src.add(src_start).cast::<__m256i>()); + let out = if bit_shift == 0 { + cur + } else { + let prev = _mm256_loadu_si256(src.add(src_start - 1).cast::<__m256i>()); + let left_count: __m128i = _mm_cvtsi64_si128(bit_shift as i64); + let right_count: __m128i = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + + let left = _mm256_sll_epi64(cur, left_count); + let right = _mm256_srl_epi64(prev, right_count); + + _mm256_or_si256(left, right) + }; + + _mm256_storeu_si256(dst.add(dst_start).cast::<__m256i>(), out); + } + + end = dst_start; + } + + // Scalar tail below the vectorized suffix, still high to low. + for dst_index in (0..end).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shl_core/neon.rs b/src/traits/words_arith/funcs_for_shl_core/neon.rs new file mode 100644 index 0000000..f426911 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shl_core/neon.rs @@ -0,0 +1,100 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +use core::arch::aarch64::{ + int64x2_t, uint64x2_t, vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64, +}; + +const LANES: usize = 2; + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn shl_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vshlq_u64(src, vdupq_n_s64(amount as i64) as int64x2_t) } +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn shr_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vshlq_u64(src, vdupq_n_s64(-(amount as i64)) as int64x2_t) } +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn or_vec(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vorrq_u64(lhs, rhs) } +} + +/// NEON backend for word-level left shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in (0..word_len).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_start = word_shift + usize::from(bit_shift != 0); + let mut end = word_len; + + while end >= bulk_start + LANES { + let dst_start = end - LANES; + let src_start = dst_start - word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES <= word_len`. + // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. + // - `vld1q_u64` reads exactly 2 u64 values. + // - `vst1q_u64` writes exactly 2 u64 values. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from high to low. + unsafe { + let cur = vld1q_u64(src.add(src_start)); + let out = if bit_shift == 0 { + cur + } else { + let prev = vld1q_u64(src.add(src_start - 1)); + let left = shl_vec(cur, bit_shift); + let right = shr_vec(prev, WORD_BITS - bit_shift); + + or_vec(left, right) + }; + + vst1q_u64(dst.add(dst_start), out); + } + + end = dst_start; + } + + for dst_index in (0..end).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shl_core/scalar.rs b/src/traits/words_arith/funcs_for_shl_core/scalar.rs new file mode 100644 index 0000000..407d362 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shl_core/scalar.rs @@ -0,0 +1,23 @@ +use super::scalar_word; + +/// Scalar backend for word-level left shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + for dst_index in (0..word_len).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - Descending order makes `dst == src` safe for left shift. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shl_core/sse2.rs b/src/traits/words_arith/funcs_for_shl_core/sse2.rs new file mode 100644 index 0000000..7d74057 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shl_core/sse2.rs @@ -0,0 +1,86 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +const LANES: usize = 2; + +/// SSE2 backend for word-level left shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in (0..word_len).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_start = word_shift + usize::from(bit_shift != 0); + let mut end = word_len; + + while end >= bulk_start + LANES { + let dst_start = end - LANES; + let src_start = dst_start - word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES <= word_len`. + // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. + // - Unaligned load/store intrinsics permit unaligned access. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from high to low. + unsafe { + let cur = _mm_loadu_si128(src.add(src_start).cast::<__m128i>()); + let out = if bit_shift == 0 { + cur + } else { + let prev = _mm_loadu_si128(src.add(src_start - 1).cast::<__m128i>()); + let left_count = _mm_cvtsi64_si128(bit_shift as i64); + let right_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + + let left = _mm_sll_epi64(cur, left_count); + let right = _mm_srl_epi64(prev, right_count); + + _mm_or_si128(left, right) + }; + + _mm_storeu_si128(dst.add(dst_start).cast::<__m128i>(), out); + } + + end = dst_start; + } + + for dst_index in (0..end).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} From 58c7bf358c286a41c90fdb98f08c68b7bf474ded Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 07:30:44 +0000 Subject: [PATCH 20/79] refactor: split shr backends --- src/traits/words_arith/funcs_for_shr_core.rs | 328 +----------------- .../words_arith/funcs_for_shr_core/avx2.rs | 95 +++++ .../words_arith/funcs_for_shr_core/neon.rs | 106 ++++++ .../words_arith/funcs_for_shr_core/scalar.rs | 23 ++ .../words_arith/funcs_for_shr_core/sse2.rs | 92 +++++ 5 files changed, 320 insertions(+), 324 deletions(-) create mode 100644 src/traits/words_arith/funcs_for_shr_core/avx2.rs create mode 100644 src/traits/words_arith/funcs_for_shr_core/neon.rs create mode 100644 src/traits/words_arith/funcs_for_shr_core/scalar.rs create mode 100644 src/traits/words_arith/funcs_for_shr_core/sse2.rs diff --git a/src/traits/words_arith/funcs_for_shr_core.rs b/src/traits/words_arith/funcs_for_shr_core.rs index 6086500..578fe5c 100644 --- a/src/traits/words_arith/funcs_for_shr_core.rs +++ b/src/traits/words_arith/funcs_for_shr_core.rs @@ -128,339 +128,19 @@ unsafe fn scalar_word(src: *const u64, word_len: usize, dst_index: usize, amount } #[allow(unused)] -mod scalar { - use super::scalar_word; - - /// Scalar backend for word-level right shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - for dst_index in 0..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - Ascending order makes `dst == src` safe for right shift. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - const LANES: usize = 4; - - /// AVX2 backend for word-level right shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in 0..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); - let mut start = 0usize; - - // Process the vectorizable prefix from low to high. - while start + LANES <= bulk_end { - let dst_start = start; - let src_start = dst_start + word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len` (since `dst_start + LANES <= bulk_end <= word_len`). - // - `src_start + LANES + 1 <= word_len` (since `bulk_end` guarantees room for the carry word). - // - Unaligned load/store intrinsics permit unaligned access. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from low to high. - unsafe { - let cur = _mm256_loadu_si256(src.add(src_start).cast::<__m256i>()); - let out = if bit_shift == 0 { - cur - } else { - let next = _mm256_loadu_si256(src.add(src_start + 1).cast::<__m256i>()); - let right_count: __m128i = _mm_cvtsi64_si128(bit_shift as i64); - let left_count: __m128i = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); - - let right = _mm256_srl_epi64(cur, right_count); - let left = _mm256_sll_epi64(next, left_count); - - _mm256_or_si256(right, left) - }; - - _mm256_storeu_si256(dst.add(dst_start).cast::<__m256i>(), out); - } - - start = dst_start + LANES; - } - - // Non-zero scalar tail. - for dst_index in start..word_len - word_shift { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - - // Zero-fill suffix: dst_index >= word_len - word_shift. - for dst_index in (word_len - word_shift)..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - const LANES: usize = 2; - - /// SSE2 backend for word-level right shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in 0..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); - let mut start = 0usize; - - while start + LANES <= bulk_end { - let dst_start = start; - let src_start = dst_start + word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES + 1 <= word_len` (guaranteed by `bulk_end`). - // - Unaligned load/store intrinsics permit unaligned access. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from low to high. - unsafe { - let cur = _mm_loadu_si128(src.add(src_start).cast::<__m128i>()); - let out = if bit_shift == 0 { - cur - } else { - let next = _mm_loadu_si128(src.add(src_start + 1).cast::<__m128i>()); - let right_count = _mm_cvtsi64_si128(bit_shift as i64); - let left_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); - - let right = _mm_srl_epi64(cur, right_count); - let left = _mm_sll_epi64(next, left_count); - - _mm_or_si128(right, left) - }; - - _mm_storeu_si128(dst.add(dst_start).cast::<__m128i>(), out); - } - - start = dst_start + LANES; - } - - for dst_index in start..word_len - word_shift { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - - for dst_index in (word_len - word_shift)..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - } -} +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - use core::arch::aarch64::{ - int64x2_t, uint64x2_t, vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64, - }; - - const LANES: usize = 2; - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn shl_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vshlq_u64(src, vdupq_n_s64(amount as i64) as int64x2_t) } - } - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn shr_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vshlq_u64(src, vdupq_n_s64(-(amount as i64)) as int64x2_t) } - } - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn or_vec(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vorrq_u64(lhs, rhs) } - } - - /// NEON backend for word-level right shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in 0..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); - let mut start = 0usize; - - while start + LANES <= bulk_end { - let dst_start = start; - let src_start = dst_start + word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES + 1 <= word_len` (guaranteed by `bulk_end`). - // - `vld1q_u64` reads exactly 2 u64 values. - // - `vst1q_u64` writes exactly 2 u64 values. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from low to high. - unsafe { - let cur = vld1q_u64(src.add(src_start)); - let out = if bit_shift == 0 { - cur - } else { - let next = vld1q_u64(src.add(src_start + 1)); - let right = shr_vec(cur, bit_shift); - let left = shl_vec(next, WORD_BITS - bit_shift); - - or_vec(right, left) - }; - - vst1q_u64(dst.add(dst_start), out); - } - - start = dst_start + LANES; - } - - for dst_index in start..word_len - word_shift { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - - for dst_index in (word_len - word_shift)..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_arith/funcs_for_shr_core/avx2.rs b/src/traits/words_arith/funcs_for_shr_core/avx2.rs new file mode 100644 index 0000000..07a8364 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shr_core/avx2.rs @@ -0,0 +1,95 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +const LANES: usize = 4; + +/// AVX2 backend for word-level right shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in 0..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); + let mut start = 0usize; + + // Process the vectorizable prefix from low to high. + while start + LANES <= bulk_end { + let dst_start = start; + let src_start = dst_start + word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len` (since `dst_start + LANES <= bulk_end <= word_len`). + // - `src_start + LANES + 1 <= word_len` (since `bulk_end` guarantees room for the carry word). + // - Unaligned load/store intrinsics permit unaligned access. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from low to high. + unsafe { + let cur = _mm256_loadu_si256(src.add(src_start).cast::<__m256i>()); + let out = if bit_shift == 0 { + cur + } else { + let next = _mm256_loadu_si256(src.add(src_start + 1).cast::<__m256i>()); + let right_count: __m128i = _mm_cvtsi64_si128(bit_shift as i64); + let left_count: __m128i = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + + let right = _mm256_srl_epi64(cur, right_count); + let left = _mm256_sll_epi64(next, left_count); + + _mm256_or_si256(right, left) + }; + + _mm256_storeu_si256(dst.add(dst_start).cast::<__m256i>(), out); + } + + start = dst_start + LANES; + } + + // Non-zero scalar tail. + for dst_index in start..word_len - word_shift { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } + + // Zero-fill suffix: dst_index >= word_len - word_shift. + for dst_index in (word_len - word_shift)..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } +} diff --git a/src/traits/words_arith/funcs_for_shr_core/neon.rs b/src/traits/words_arith/funcs_for_shr_core/neon.rs new file mode 100644 index 0000000..ae45eb2 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shr_core/neon.rs @@ -0,0 +1,106 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +use core::arch::aarch64::{ + int64x2_t, uint64x2_t, vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64, +}; + +const LANES: usize = 2; + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn shl_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vshlq_u64(src, vdupq_n_s64(amount as i64) as int64x2_t) } +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn shr_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vshlq_u64(src, vdupq_n_s64(-(amount as i64)) as int64x2_t) } +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn or_vec(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vorrq_u64(lhs, rhs) } +} + +/// NEON backend for word-level right shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in 0..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); + let mut start = 0usize; + + while start + LANES <= bulk_end { + let dst_start = start; + let src_start = dst_start + word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES + 1 <= word_len` (guaranteed by `bulk_end`). + // - `vld1q_u64` reads exactly 2 u64 values. + // - `vst1q_u64` writes exactly 2 u64 values. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from low to high. + unsafe { + let cur = vld1q_u64(src.add(src_start)); + let out = if bit_shift == 0 { + cur + } else { + let next = vld1q_u64(src.add(src_start + 1)); + let right = shr_vec(cur, bit_shift); + let left = shl_vec(next, WORD_BITS - bit_shift); + + or_vec(right, left) + }; + + vst1q_u64(dst.add(dst_start), out); + } + + start = dst_start + LANES; + } + + for dst_index in start..word_len - word_shift { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } + + for dst_index in (word_len - word_shift)..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } +} diff --git a/src/traits/words_arith/funcs_for_shr_core/scalar.rs b/src/traits/words_arith/funcs_for_shr_core/scalar.rs new file mode 100644 index 0000000..65936c0 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shr_core/scalar.rs @@ -0,0 +1,23 @@ +use super::scalar_word; + +/// Scalar backend for word-level right shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + for dst_index in 0..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - Ascending order makes `dst == src` safe for right shift. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shr_core/sse2.rs b/src/traits/words_arith/funcs_for_shr_core/sse2.rs new file mode 100644 index 0000000..6c2d8bd --- /dev/null +++ b/src/traits/words_arith/funcs_for_shr_core/sse2.rs @@ -0,0 +1,92 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +const LANES: usize = 2; + +/// SSE2 backend for word-level right shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in 0..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); + let mut start = 0usize; + + while start + LANES <= bulk_end { + let dst_start = start; + let src_start = dst_start + word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES + 1 <= word_len` (guaranteed by `bulk_end`). + // - Unaligned load/store intrinsics permit unaligned access. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from low to high. + unsafe { + let cur = _mm_loadu_si128(src.add(src_start).cast::<__m128i>()); + let out = if bit_shift == 0 { + cur + } else { + let next = _mm_loadu_si128(src.add(src_start + 1).cast::<__m128i>()); + let right_count = _mm_cvtsi64_si128(bit_shift as i64); + let left_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + + let right = _mm_srl_epi64(cur, right_count); + let left = _mm_sll_epi64(next, left_count); + + _mm_or_si128(right, left) + }; + + _mm_storeu_si128(dst.add(dst_start).cast::<__m128i>(), out); + } + + start = dst_start + LANES; + } + + for dst_index in start..word_len - word_shift { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } + + for dst_index in (word_len - word_shift)..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } +} From 50519749ce779ccdaef87c7b49b582dfa419d2a4 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 07:43:43 +0000 Subject: [PATCH 21/79] refactor: split copy_words_shifted backends --- .../funcs_for_copy_words_shifted_core.rs | 143 +----------------- .../funcs_for_copy_words_shifted_core/avx2.rs | 40 +++++ .../funcs_for_copy_words_shifted_core/neon.rs | 31 ++++ .../funcs_for_copy_words_shifted_core/sse2.rs | 40 +++++ 4 files changed, 114 insertions(+), 140 deletions(-) create mode 100644 src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs create mode 100644 src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/neon.rs create mode 100644 src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/sse2.rs diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs index 600af9a..ce241e5 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs @@ -72,152 +72,15 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn copy_words_shifted( - dst: &mut [u64], - src: &[u64], - len: usize, - shift: usize, - ) { - // SAFETY: `#[target_feature(enable = "avx2")]` guarantees the CPU supports AVX2; the `unsafe fn` contract guarantees pointer validity. - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - // SAFETY: Same as above. - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 4 <= len { - // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm256_loadu_si256` uses unaligned loads so alignment is not required. - let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: Same as above. - let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; - // SAFETY: Pure register operations; no memory access. - let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; - // SAFETY: Same as above. - let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; - // SAFETY: Same as above. - let window = unsafe { _mm256_or_si256(lo, hi) }; - // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm256_storeu_si256` uses unaligned stores. - unsafe { _mm256_storeu_si256(dst.as_mut_ptr().add(i).cast::<__m256i>(), window) }; - i += 4; - } - while i < len { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - i += 1; - } - } -} - -// --------------------------------------------------------------------------- -// SSE2 — 2 words per iteration -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - #[target_feature(enable = "sse2")] - pub(super) unsafe fn copy_words_shifted( - dst: &mut [u64], - src: &[u64], - len: usize, - shift: usize, - ) { - // SAFETY: `#[target_feature(enable = "sse2")]` guarantees the CPU supports SSE2; the `unsafe fn` contract guarantees pointer validity. - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - // SAFETY: Same as above. - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 2 <= len { - // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm_loadu_si128` uses unaligned loads so alignment is not required. - let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: Same as above. - let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; - // SAFETY: Pure register operations; no memory access. - let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; - // SAFETY: Same as above. - let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; - // SAFETY: Same as above. - let window = unsafe { _mm_or_si128(lo, hi) }; - // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm_storeu_si128` uses unaligned stores. - unsafe { _mm_storeu_si128(dst.as_mut_ptr().add(i).cast::<__m128i>(), window) }; - i += 2; - } - while i < len { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - i += 1; - } - } -} - -// --------------------------------------------------------------------------- -// NEON — 2 words per iteration -// --------------------------------------------------------------------------- +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use crate::WORD_BITS; - use core::arch::aarch64::{vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64}; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn copy_words_shifted( - dst: &mut [u64], - src: &[u64], - len: usize, - shift: usize, - ) { - // SAFETY: `#[target_feature(enable = "neon")]` guarantees the CPU supports NEON; the `unsafe fn` contract guarantees pointer validity. - let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; - // SAFETY: Same as above. - let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; - - let mut i = 0; - while i + 2 <= len { - // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee). - let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - // SAFETY: Same as above. - let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; - // SAFETY: Pure register operations; no memory access. - let lo = unsafe { vshlq_u64(w0, neg_shift) }; - // SAFETY: Same as above. - let hi = unsafe { vshlq_u64(w1, pos_shift) }; - // SAFETY: Same as above. - let window = unsafe { vorrq_u64(lo, hi) }; - // SAFETY: `dst` pointer is valid for `len` words (caller guarantee). - unsafe { vst1q_u64(dst.as_mut_ptr().add(i), window) }; - i += 2; - } - while i < len { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - i += 1; - } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs new file mode 100644 index 0000000..aa69a99 --- /dev/null +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs @@ -0,0 +1,40 @@ +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn copy_words_shifted(dst: &mut [u64], src: &[u64], len: usize, shift: usize) { + // SAFETY: `#[target_feature(enable = "avx2")]` guarantees the CPU supports AVX2; the `unsafe fn` contract guarantees pointer validity. + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: Same as above. + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 4 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm256_loadu_si256` uses unaligned loads so alignment is not required. + let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: Same as above. + let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; + // SAFETY: Pure register operations; no memory access. + let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + // SAFETY: Same as above. + let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + // SAFETY: Same as above. + let window = unsafe { _mm256_or_si256(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm256_storeu_si256` uses unaligned stores. + unsafe { _mm256_storeu_si256(dst.as_mut_ptr().add(i).cast::<__m256i>(), window) }; + i += 4; + } + while i < len { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + i += 1; + } +} diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/neon.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/neon.rs new file mode 100644 index 0000000..034e35b --- /dev/null +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/neon.rs @@ -0,0 +1,31 @@ +use crate::WORD_BITS; +use core::arch::aarch64::{vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn copy_words_shifted(dst: &mut [u64], src: &[u64], len: usize, shift: usize) { + // SAFETY: `#[target_feature(enable = "neon")]` guarantees the CPU supports NEON; the `unsafe fn` contract guarantees pointer validity. + let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + // SAFETY: Same as above. + let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; + + let mut i = 0; + while i + 2 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee). + let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: Same as above. + let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; + // SAFETY: Pure register operations; no memory access. + let lo = unsafe { vshlq_u64(w0, neg_shift) }; + // SAFETY: Same as above. + let hi = unsafe { vshlq_u64(w1, pos_shift) }; + // SAFETY: Same as above. + let window = unsafe { vorrq_u64(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee). + unsafe { vst1q_u64(dst.as_mut_ptr().add(i), window) }; + i += 2; + } + while i < len { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + i += 1; + } +} diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/sse2.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/sse2.rs new file mode 100644 index 0000000..bd7e23f --- /dev/null +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/sse2.rs @@ -0,0 +1,40 @@ +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +#[target_feature(enable = "sse2")] +pub(super) unsafe fn copy_words_shifted(dst: &mut [u64], src: &[u64], len: usize, shift: usize) { + // SAFETY: `#[target_feature(enable = "sse2")]` guarantees the CPU supports SSE2; the `unsafe fn` contract guarantees pointer validity. + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: Same as above. + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 2 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm_loadu_si128` uses unaligned loads so alignment is not required. + let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: Same as above. + let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; + // SAFETY: Pure register operations; no memory access. + let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; + // SAFETY: Same as above. + let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; + // SAFETY: Same as above. + let window = unsafe { _mm_or_si128(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm_storeu_si128` uses unaligned stores. + unsafe { _mm_storeu_si128(dst.as_mut_ptr().add(i).cast::<__m128i>(), window) }; + i += 2; + } + while i < len { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + i += 1; + } +} From 543b520ec336c0a7b5aa113e8c0ad06930a9ab69 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 07:48:50 +0000 Subject: [PATCH 22/79] fix: normalize anonymous relocation identities --- scripts/compare_codegen.py | 5 +++-- scripts/tests/test_codegen.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index e0d6cc5..bfeb9b4 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -22,6 +22,7 @@ SYMBOL_TARGET = re.compile(r"(?:0x)?([0-9a-f]+)\s+<([^>]+)>") PC_RELATIVE_X86_RELOCATIONS = {"R_X86_64_PC32", "R_X86_64_PLT32"} X86_PC_BIAS = re.compile(r"-0x4$") +ANONYMOUS_SYMBOL_INSTANCE = re.compile(r"(\.Lanon\.[0-9a-f]+)\.\d+$") OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") BACKEND_MODULE = re.compile(r"::(?:scalar|sse2|ssse3|sse41|avx2|neon)::") @@ -64,8 +65,8 @@ def classify(old: Function, new: Function) -> str: def _relocation_target(kind: str, value: str) -> str: value = value.strip() if kind in PC_RELATIVE_X86_RELOCATIONS: - return X86_PC_BIAS.sub("", value) - return value + value = X86_PC_BIAS.sub("", value) + return ANONYMOUS_SYMBOL_INSTANCE.sub(r"\1", value) def _is_call(mnemonic: str, arch: str) -> bool: diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 2ce2bb0..feb0554 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -41,6 +41,21 @@ def test_semantic_relocation_addend_is_preserved(self) -> None: second = parse("0000 :\n 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n 3: R_X86_64_PC32 .rodata+0x40") self.assertNotEqual(first.instructions, second.instructions) + def test_anonymous_relocation_instance_is_ignored(self) -> None: + first = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 \n" + " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.abcdef0123456789.67", + "aarch64", + ) + second = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 \n" + " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.abcdef0123456789.68", + "aarch64", + ) + self.assertEqual(first.instructions, second.instructions) + def test_duplicate_disassembly_symbol_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "duplicate disassembly symbol"): compare_codegen.parse_objdump("0000 :\n 0: c3 ret\n0000 :\n 0: c3 ret", "x86_64") From d7cf6db258dfc08de11e1f3a484875f747a49482 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 07:52:35 +0000 Subject: [PATCH 23/79] fix: handle anonymous relocation variants --- scripts/compare_codegen.py | 2 +- scripts/tests/test_codegen.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index bfeb9b4..73f0d8c 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -22,7 +22,7 @@ SYMBOL_TARGET = re.compile(r"(?:0x)?([0-9a-f]+)\s+<([^>]+)>") PC_RELATIVE_X86_RELOCATIONS = {"R_X86_64_PC32", "R_X86_64_PLT32"} X86_PC_BIAS = re.compile(r"-0x4$") -ANONYMOUS_SYMBOL_INSTANCE = re.compile(r"(\.Lanon\.[0-9a-f]+)\.\d+$") +ANONYMOUS_SYMBOL_INSTANCE = re.compile(r"(\.Lanon\.[^\s]+?)\.\d+(?:[+-]0x[0-9a-f]+)?$") OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") BACKEND_MODULE = re.compile(r"::(?:scalar|sse2|ssse3|sse41|avx2|neon)::") diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index feb0554..c6cb4c9 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -56,6 +56,21 @@ def test_anonymous_relocation_instance_is_ignored(self) -> None: ) self.assertEqual(first.instructions, second.instructions) + def test_anonymous_relocation_instance_with_addend_is_ignored(self) -> None: + first = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 \n" + " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash-with-extra.67+0x20", + "aarch64", + ) + second = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 \n" + " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash-with-extra.68+0x20", + "aarch64", + ) + self.assertEqual(first.instructions, second.instructions) + def test_duplicate_disassembly_symbol_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "duplicate disassembly symbol"): compare_codegen.parse_objdump("0000 :\n 0: c3 ret\n0000 :\n 0: c3 ret", "x86_64") From e780b29e6da3b18fce86e0c35fd1900c16747d44 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 07:54:55 +0000 Subject: [PATCH 24/79] fix: preserve anonymous relocation addends --- scripts/compare_codegen.py | 2 +- scripts/tests/test_codegen.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index 73f0d8c..4aabbdb 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -22,7 +22,7 @@ SYMBOL_TARGET = re.compile(r"(?:0x)?([0-9a-f]+)\s+<([^>]+)>") PC_RELATIVE_X86_RELOCATIONS = {"R_X86_64_PC32", "R_X86_64_PLT32"} X86_PC_BIAS = re.compile(r"-0x4$") -ANONYMOUS_SYMBOL_INSTANCE = re.compile(r"(\.Lanon\.[^\s]+?)\.\d+(?:[+-]0x[0-9a-f]+)?$") +ANONYMOUS_SYMBOL_INSTANCE = re.compile(r"(\.Lanon\.[^\s]+?)\.\d+(?=[+-]0x[0-9a-f]+$|$)") OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") BACKEND_MODULE = re.compile(r"::(?:scalar|sse2|ssse3|sse41|avx2|neon)::") diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index c6cb4c9..d6b95c7 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -70,6 +70,15 @@ def test_anonymous_relocation_instance_with_addend_is_ignored(self) -> None: "aarch64", ) self.assertEqual(first.instructions, second.instructions) + self.assertIn("+0x20", first.instructions[0]) + + different_addend = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 \n" + " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash-with-extra.68+0x40", + "aarch64", + ) + self.assertNotEqual(first.instructions, different_addend.instructions) def test_duplicate_disassembly_symbol_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "duplicate disassembly symbol"): From 3952f5364cb62155d2fe7113682413519417bf54 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 08:01:15 +0000 Subject: [PATCH 25/79] fix: normalize anonymous relocation ordinals --- scripts/compare_codegen.py | 41 ++++++++++++++++++++--- scripts/tests/test_codegen.py | 62 +++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index 4aabbdb..18938bc 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -19,10 +19,14 @@ X86_INSTRUCTION = re.compile(r"^\s*([0-9a-f]+):\s+((?:[0-9a-f]{2}(?:\s+|$))+)(.*?)\s*$") AARCH64_INSTRUCTION = re.compile(r"^\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(.*?)\s*$") RELOCATION = re.compile(r"^\s*([0-9a-f]+):\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") +INLINE_RELOCATION = re.compile(r"\s+([0-9a-f]+):\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") SYMBOL_TARGET = re.compile(r"(?:0x)?([0-9a-f]+)\s+<([^>]+)>") PC_RELATIVE_X86_RELOCATIONS = {"R_X86_64_PC32", "R_X86_64_PLT32"} X86_PC_BIAS = re.compile(r"-0x4$") -ANONYMOUS_SYMBOL_INSTANCE = re.compile(r"(\.Lanon\.[^\s]+?)\.\d+(?=[+-]0x[0-9a-f]+$|$)") +ANONYMOUS_SYMBOL_INSTANCE = re.compile( + r"(?P(?:\.Lanon|anon)\.[^.\s]+)\.(?P\d+)" + r"(?:\.llvm\.[^+\-\s]+)?(?P[+-]0x[0-9a-f]+)?$" +) OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") BACKEND_MODULE = re.compile(r"::(?:scalar|sse2|ssse3|sse41|avx2|neon)::") @@ -62,11 +66,24 @@ def classify(old: Function, new: Function) -> str: return "CODEGEN IDENTICAL" -def _relocation_target(kind: str, value: str) -> str: +def _anonymous_instance(value: str) -> tuple[str, int, str] | None: + match = ANONYMOUS_SYMBOL_INSTANCE.search(value) + if match is None: + return None + key = value[: match.start("stem")] + match.group("stem") + return key, int(match.group("instance")), match.group("addend") or "" + + +def _relocation_target(kind: str, value: str, anonymous_bases: dict[str, int]) -> str: value = value.strip() if kind in PC_RELATIVE_X86_RELOCATIONS: value = X86_PC_BIAS.sub("", value) - return ANONYMOUS_SYMBOL_INSTANCE.sub(r"\1", value) + anonymous = _anonymous_instance(value) + if anonymous is None: + return value + key, instance, addend = anonymous + base = anonymous_bases[key] + return f"{key}.@{instance - base}{addend}" def _is_call(mnemonic: str, arch: str) -> bool: @@ -87,6 +104,14 @@ def _has_control_target(mnemonic: str, arch: str) -> bool: def _normalize(instructions: list[Instruction], arch: str) -> Function: indexes = {instruction.address: index for index, instruction in enumerate(instructions)} + anonymous_bases: dict[str, int] = {} + for instruction in instructions: + for _, target in instruction.relocations: + anonymous = _anonymous_instance(target.strip()) + if anonymous is None: + continue + key, instance, _ = anonymous + anonymous_bases[key] = min(instance, anonymous_bases.get(key, instance)) normalized: list[str] = [] calls = branches = 0 for instruction in instructions: @@ -106,7 +131,8 @@ def _normalize(instructions: list[Instruction], arch: str) -> Function: target = f"@instruction_{indexes[address]}" if address in indexes and "+0x" in symbol else f"<{symbol}>" operands = SYMBOL_TARGET.sub(target, operands) relocations = " ".join( - f"[{kind}:{_relocation_target(kind, target)}]" for kind, target in instruction.relocations + f"[{kind}:{_relocation_target(kind, target, anonymous_bases)}]" + for kind, target in instruction.relocations ) normalized.append(" ".join(f"{mnemonic} {operands} {relocations}".split())) return Function(tuple(normalized), b"".join(item.encoded for item in instructions), calls, branches) @@ -138,9 +164,14 @@ def parse_objdump(output: str, arch: str) -> dict[str, Function]: continue encoded = bytes.fromhex(match.group(2)) assembly = match.group(3).strip() + relocations: list[tuple[str, str]] = [] + inline_relocation = INLINE_RELOCATION.search(assembly) + if inline_relocation is not None: + assembly = assembly[: inline_relocation.start()].rstrip() + relocations.append((inline_relocation.group(2), inline_relocation.group(3))) if not assembly: continue - last_instruction = Instruction(int(match.group(1), 16), encoded, assembly) + last_instruction = Instruction(int(match.group(1), 16), encoded, assembly, relocations) raw[current].append(last_instruction) return {name: _normalize(items, arch) for name, items in raw.items() if items} diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index d6b95c7..5e47723 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -80,6 +80,68 @@ def test_anonymous_relocation_instance_with_addend_is_ignored(self) -> None: ) self.assertNotEqual(first.instructions, different_addend.instructions) + def test_inline_anonymous_relocations_allow_uniform_ordinal_shift(self) -> None: + first = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.67\n" + " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n" + " 8: 90000000 adrp x2, 0 8: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.68\n", + "aarch64", + ) + shifted = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.68\n" + " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.70\n" + " 8: 90000000 adrp x2, 0 8: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n", + "aarch64", + ) + self.assertEqual(first.instructions, shifted.instructions) + + def test_inline_anonymous_relocations_preserve_alias_relationships(self) -> None: + aliased = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.67\n" + " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.67\n" + " 8: 90000000 adrp x2, 0 8: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n", + "aarch64", + ) + no_longer_aliased = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.68\n" + " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n" + " 8: 90000000 adrp x2, 0 8: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.70\n", + "aarch64", + ) + self.assertNotEqual(aliased.instructions, no_longer_aliased.instructions) + + def test_inline_anonymous_relocations_preserve_relative_gaps(self) -> None: + first = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.67\n" + " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n", + "aarch64", + ) + different_gap = parse( + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.68\n" + " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.71\n", + "aarch64", + ) + self.assertNotEqual(first.instructions, different_gap.instructions) + + def test_x86_anonymous_relocations_allow_llvm_suffix_changes(self) -> None: + first = parse( + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax 0: R_X86_64_PC32 anon.hash.67.llvm.123\n" + " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax 7: R_X86_64_PC32 anon.hash.69.llvm.123\n" + ) + shifted = parse( + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax 0: R_X86_64_PC32 anon.hash.68.llvm.456\n" + " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax 7: R_X86_64_PC32 anon.hash.70.llvm.456\n" + ) + self.assertEqual(first.instructions, shifted.instructions) + def test_duplicate_disassembly_symbol_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "duplicate disassembly symbol"): compare_codegen.parse_objdump("0000 :\n 0: c3 ret\n0000 :\n 0: c3 ret", "x86_64") From dc1fe393219ba8a988f45c39f1ecefbdbe6096f5 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 08:18:17 +0000 Subject: [PATCH 26/79] refactor: split eq_words_aligned backends --- .../funcs_for_eq_words_aligned_core.rs | 96 +------------------ .../funcs_for_eq_words_aligned_core/avx2.rs | 30 ++++++ .../funcs_for_eq_words_aligned_core/neon.rs | 27 ++++++ .../funcs_for_eq_words_aligned_core/sse41.rs | 30 ++++++ 4 files changed, 90 insertions(+), 93 deletions(-) create mode 100644 src/traits/words_eq/funcs_for_eq_words_aligned_core/avx2.rs create mode 100644 src/traits/words_eq/funcs_for_eq_words_aligned_core/neon.rs create mode 100644 src/traits/words_eq/funcs_for_eq_words_aligned_core/sse41.rs diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs index 3c2a9cb..856ec9d 100644 --- a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs @@ -60,105 +60,15 @@ pub(super) fn eq_words_aligned(src: &[u64], other: &[u64], count: usize) -> bool #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 4 <= len { - // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. - // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). - let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: same as above; load from `other`. - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: `_mm256_cmpeq_epi64` and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. - let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; - // SAFETY: same as above - if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { - return false; - } - i += 4; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 2 <= len { - // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. - // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). - let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: same as above; load from `other`. - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: `_mm_cmpeq_epi64` and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. - let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; - // SAFETY: same as above - if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { - return false; - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} +mod sse41; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 2 <= len { - // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. - // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). - let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - // SAFETY: same as above; load from `other`. - let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - // SAFETY: `vceqq_u64` and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. - let cmp = unsafe { vceqq_u64(a, b) }; - // SAFETY: same as above - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - return false; - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core/avx2.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/avx2.rs new file mode 100644 index 0000000..2cf67e1 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/avx2.rs @@ -0,0 +1,30 @@ +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 4 <= len { + // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: `_mm256_cmpeq_epi64` and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; + // SAFETY: same as above + if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { + return false; + } + i += 4; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core/neon.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/neon.rs new file mode 100644 index 0000000..412ade4 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/neon.rs @@ -0,0 +1,27 @@ +use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + // SAFETY: `vceqq_u64` and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. + let cmp = unsafe { vceqq_u64(a, b) }; + // SAFETY: same as above + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + return false; + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core/sse41.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/sse41.rs new file mode 100644 index 0000000..5fe6141 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/sse41.rs @@ -0,0 +1,30 @@ +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: `_mm_cmpeq_epi64` and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; + // SAFETY: same as above + if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { + return false; + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true +} From ce974588889998c14e31f042e4b443c2c569793a Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 08:38:02 +0000 Subject: [PATCH 27/79] refactor: split eq_words_unaligned backends --- .../funcs_for_eq_words_unaligned_core.rs | 190 +----------------- .../funcs_for_eq_words_unaligned_core/avx2.rs | 57 ++++++ .../funcs_for_eq_words_unaligned_core/neon.rs | 67 ++++++ .../sse41.rs | 57 ++++++ 4 files changed, 184 insertions(+), 187 deletions(-) create mode 100644 src/traits/words_eq/funcs_for_eq_words_unaligned_core/avx2.rs create mode 100644 src/traits/words_eq/funcs_for_eq_words_unaligned_core/neon.rs create mode 100644 src/traits/words_eq/funcs_for_eq_words_unaligned_core/sse41.rs diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs index 2a9fd69..d56aa60 100644 --- a/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs @@ -66,199 +66,15 @@ pub(super) fn eq_words_unaligned(src: &[u64], other: &[u64], count: usize, shift #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, - _mm256_movemask_pd, _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, - _mm256_movemask_pd, _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn eq_words_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> bool { - // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; AVX2 is enabled by `#[target_feature]`. - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - // SAFETY: same as above - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 4 <= len { - // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. - // Pointers `src` and `other` are valid for `len+1` elements (caller guarantees extra word for window shift). - let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: same as above; load from `src[i+1]`. - let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; - // SAFETY: `_mm256_srl_epi64`, `_mm256_sll_epi64`, `_mm256_or_si256`, `_mm256_cmpeq_epi64`, and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. - let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; - // SAFETY: same as above - let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; - // SAFETY: same as above - let window = unsafe { _mm256_or_si256(lo, hi) }; - // SAFETY: AVX2 is enabled; pointer `other` is valid for `len` elements. - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: `_mm256_cmpeq_epi64` is a pure register operation; AVX2 is enabled by `#[target_feature]`. - let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; - // SAFETY: same as above - if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { - return false; - } - i += 4; - } - while i < len { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - i += 1; - } - true - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn eq_words_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> bool { - // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; SSE4.1 is enabled by `#[target_feature]`. - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - // SAFETY: same as above - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 2 <= len { - // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. - // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees). - let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: same as above; load from `src[i+1]`. - let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; - // SAFETY: `_mm_srl_epi64`, `_mm_sll_epi64`, `_mm_or_si128`, `_mm_cmpeq_epi64`, and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. - let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; - // SAFETY: same as above - let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; - // SAFETY: same as above - let window = unsafe { _mm_or_si128(lo, hi) }; - // SAFETY: SSE4.1 is enabled; pointer `other` is valid for `len` elements. - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: `_mm_cmpeq_epi64` is a pure register operation; SSE4.1 is enabled by `#[target_feature]`. - let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; - // SAFETY: same as above - if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { - return false; - } - i += 2; - } - while i < len { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - i += 1; - } - true - } -} +mod sse41; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use crate::WORD_BITS; - use core::arch::aarch64::{ - vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, - }; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn eq_words_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> bool { - // SAFETY: `shift` is in [1, WORD_BITS); the caller guarantees this - // via the `shift == 0` fast-path in the entry point. - // Both shift vectors fit in i64: - // shift ∈ [1, 63] → -shift ∈ [-63, -1] - // WORD_BITS - shift ∈ [1, 63] - let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; - // SAFETY: same as above - let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; - - // Process 2 lanes (128 bits) per iteration. - let mut i = 0; - while i + 2 <= len { - // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. - // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees extra word for window shift). - // Load [src[i], src[i+1]] and [src[i+1], src[i+2]]. - let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - // SAFETY: same as above; load from `src[i+1]`. - let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; - - // Build the shifted 64-bit window for each lane: - // window[k] = (src[i+k] >> shift) | (src[i+k+1] << (64 - shift)) - // vshlq_u64 with a negative shift amount performs a logical right shift. - // SAFETY: `vshlq_u64`, `vorrq_u64`, `vceqq_u64`, and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. - let lo = unsafe { vshlq_u64(w0, neg_shift) }; - // SAFETY: same as above - let hi = unsafe { vshlq_u64(w1, pos_shift) }; - // SAFETY: same as above - let window = unsafe { vorrq_u64(lo, hi) }; - - // SAFETY: NEON is enabled; pointer `other` is valid for `len` elements. - let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - // SAFETY: `vceqq_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. - let cmp = unsafe { vceqq_u64(window, expected) }; - - // Each lane is all-ones on equality → vgetq_lane_u64 returns u64::MAX. - // SAFETY: `vgetq_lane_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - return false; - } - - i += 2; - } - - // Scalar tail for the last word (when len is odd). - while i < len { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - i += 1; - } - - true - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core/avx2.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/avx2.rs new file mode 100644 index 0000000..fabea42 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/avx2.rs @@ -0,0 +1,57 @@ +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn eq_words_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> bool { + // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; AVX2 is enabled by `#[target_feature]`. + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: same as above + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 4 <= len { + // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. + // Pointers `src` and `other` are valid for `len+1` elements (caller guarantees extra word for window shift). + let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: same as above; load from `src[i+1]`. + let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; + // SAFETY: `_mm256_srl_epi64`, `_mm256_sll_epi64`, `_mm256_or_si256`, `_mm256_cmpeq_epi64`, and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. + let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + // SAFETY: same as above + let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + // SAFETY: same as above + let window = unsafe { _mm256_or_si256(lo, hi) }; + // SAFETY: AVX2 is enabled; pointer `other` is valid for `len` elements. + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: `_mm256_cmpeq_epi64` is a pure register operation; AVX2 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; + // SAFETY: same as above + if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { + return false; + } + i += 4; + } + while i < len { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + i += 1; + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core/neon.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/neon.rs new file mode 100644 index 0000000..390e51c --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/neon.rs @@ -0,0 +1,67 @@ +use crate::WORD_BITS; +use core::arch::aarch64::{ + vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, +}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn eq_words_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> bool { + // SAFETY: `shift` is in [1, WORD_BITS); the caller guarantees this + // via the `shift == 0` fast-path in the entry point. + // Both shift vectors fit in i64: + // shift ∈ [1, 63] → -shift ∈ [-63, -1] + // WORD_BITS - shift ∈ [1, 63] + let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + // SAFETY: same as above + let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; + + // Process 2 lanes (128 bits) per iteration. + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. + // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees extra word for window shift). + // Load [src[i], src[i+1]] and [src[i+1], src[i+2]]. + let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: same as above; load from `src[i+1]`. + let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; + + // Build the shifted 64-bit window for each lane: + // window[k] = (src[i+k] >> shift) | (src[i+k+1] << (64 - shift)) + // vshlq_u64 with a negative shift amount performs a logical right shift. + // SAFETY: `vshlq_u64`, `vorrq_u64`, `vceqq_u64`, and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. + let lo = unsafe { vshlq_u64(w0, neg_shift) }; + // SAFETY: same as above + let hi = unsafe { vshlq_u64(w1, pos_shift) }; + // SAFETY: same as above + let window = unsafe { vorrq_u64(lo, hi) }; + + // SAFETY: NEON is enabled; pointer `other` is valid for `len` elements. + let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + // SAFETY: `vceqq_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. + let cmp = unsafe { vceqq_u64(window, expected) }; + + // Each lane is all-ones on equality → vgetq_lane_u64 returns u64::MAX. + // SAFETY: `vgetq_lane_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + return false; + } + + i += 2; + } + + // Scalar tail for the last word (when len is odd). + while i < len { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + i += 1; + } + + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core/sse41.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/sse41.rs new file mode 100644 index 0000000..1fa33e7 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/sse41.rs @@ -0,0 +1,57 @@ +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi64x, + _mm_sll_epi64, _mm_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi64x, + _mm_sll_epi64, _mm_srl_epi64, +}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn eq_words_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> bool { + // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; SSE4.1 is enabled by `#[target_feature]`. + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: same as above + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. + // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees). + let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: same as above; load from `src[i+1]`. + let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; + // SAFETY: `_mm_srl_epi64`, `_mm_sll_epi64`, `_mm_or_si128`, `_mm_cmpeq_epi64`, and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. + let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; + // SAFETY: same as above + let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; + // SAFETY: same as above + let window = unsafe { _mm_or_si128(lo, hi) }; + // SAFETY: SSE4.1 is enabled; pointer `other` is valid for `len` elements. + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: `_mm_cmpeq_epi64` is a pure register operation; SSE4.1 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; + // SAFETY: same as above + if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { + return false; + } + i += 2; + } + while i < len { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + i += 1; + } + true +} From e39a0b4b454958794d43e28f554b2fc54e3b5b15 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 08:47:20 +0000 Subject: [PATCH 28/79] refactor: split cmp_aligned backends --- .../words_ord/funcs_for_cmp_aligned_core.rs | 101 +----------------- .../funcs_for_cmp_aligned_core/avx2.rs | 31 ++++++ .../funcs_for_cmp_aligned_core/neon.rs | 28 +++++ .../funcs_for_cmp_aligned_core/sse41.rs | 33 ++++++ 4 files changed, 95 insertions(+), 98 deletions(-) create mode 100644 src/traits/words_ord/funcs_for_cmp_aligned_core/avx2.rs create mode 100644 src/traits/words_ord/funcs_for_cmp_aligned_core/neon.rs create mode 100644 src/traits/words_ord/funcs_for_cmp_aligned_core/sse41.rs diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs index d27def7..d6925bc 100644 --- a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs @@ -62,39 +62,7 @@ fn scalar_cmp_aligned(src: &[u64], other: &[u64], count: usize) -> Option Option { - let mut i = 0; - while i + 4 <= len { - let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; - let mask = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; - if mask != 0b1111 { - let lane = mask.trailing_ones() as usize; - return Some(WordOrd::bitwise_cmp(src[i + lane], other[i + lane])); - } - i += 4; - } - while i < len { - if src[i] != other[i] { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - i += 1; - } - None - } -} +mod avx2; // --------------------------------------------------------------------------- // SSE2 — 2 × u64 per iteration @@ -102,41 +70,7 @@ mod avx2 { #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use core::cmp::Ordering; - - use crate::traits::WordOrd; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { - let mut i = 0; - while i + 2 <= len { - let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; - let mask = unsafe { _mm_movemask_epi8(cmp) } as u32; - if mask != 0xFFFF { - // Each lane is 8 bytes → 8 high bits. The first zero byte - // tells us which lane differs. - let lane = mask.trailing_ones() as usize / 8; - return Some(WordOrd::bitwise_cmp(src[i + lane], other[i + lane])); - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - i += 1; - } - None - } -} +mod sse41; // --------------------------------------------------------------------------- // NEON — 2 × u64 per iteration @@ -144,36 +78,7 @@ mod sse41 { #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use core::cmp::Ordering; - - use crate::traits::WordOrd; - use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { - let mut i = 0; - while i + 2 <= len { - let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - let cmp = unsafe { vceqq_u64(a, b) }; - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - return Some(WordOrd::bitwise_cmp(src[i + 1], other[i + 1])); - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - i += 1; - } - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core/avx2.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core/avx2.rs new file mode 100644 index 0000000..4815fff --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core/avx2.rs @@ -0,0 +1,31 @@ +use core::cmp::Ordering; + +use crate::traits::WordOrd; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { + let mut i = 0; + while i + 4 <= len { + let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; + let mask = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; + if mask != 0b1111 { + let lane = mask.trailing_ones() as usize; + return Some(WordOrd::bitwise_cmp(src[i + lane], other[i + lane])); + } + i += 4; + } + while i < len { + if src[i] != other[i] { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core/neon.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core/neon.rs new file mode 100644 index 0000000..e71bee1 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core/neon.rs @@ -0,0 +1,28 @@ +use core::cmp::Ordering; + +use crate::traits::WordOrd; +use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { + let mut i = 0; + while i + 2 <= len { + let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + let cmp = unsafe { vceqq_u64(a, b) }; + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + return Some(WordOrd::bitwise_cmp(src[i + 1], other[i + 1])); + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core/sse41.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core/sse41.rs new file mode 100644 index 0000000..bc52aa4 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core/sse41.rs @@ -0,0 +1,33 @@ +use core::cmp::Ordering; + +use crate::traits::WordOrd; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { + let mut i = 0; + while i + 2 <= len { + let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; + let mask = unsafe { _mm_movemask_epi8(cmp) } as u32; + if mask != 0xFFFF { + // Each lane is 8 bytes → 8 high bits. The first zero byte + // tells us which lane differs. + let lane = mask.trailing_ones() as usize / 8; + return Some(WordOrd::bitwise_cmp(src[i + lane], other[i + lane])); + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + i += 1; + } + None +} From 929e1c78a87cc71c65931818364e55b22e878d38 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 09:09:30 +0000 Subject: [PATCH 29/79] refactor: split cmp_unaligned backends --- .../words_ord/funcs_for_cmp_unaligned_core.rs | 171 +----------------- .../funcs_for_cmp_unaligned_core/avx2.rs | 52 ++++++ .../funcs_for_cmp_unaligned_core/neon.rs | 46 +++++ .../funcs_for_cmp_unaligned_core/sse41.rs | 52 ++++++ 4 files changed, 153 insertions(+), 168 deletions(-) create mode 100644 src/traits/words_ord/funcs_for_cmp_unaligned_core/avx2.rs create mode 100644 src/traits/words_ord/funcs_for_cmp_unaligned_core/neon.rs create mode 100644 src/traits/words_ord/funcs_for_cmp_unaligned_core/sse41.rs diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs index cbb6fa4..35709bc 100644 --- a/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs @@ -74,182 +74,17 @@ fn scalar_cmp_unaligned( None } -// --------------------------------------------------------------------------- -// AVX2 — 4 × u64 per iteration -// --------------------------------------------------------------------------- - #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use core::cmp::Ordering; - - use crate::WORD_BITS; - use crate::traits::WordOrd; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, - _mm256_movemask_pd, _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, - _mm256_movemask_pd, _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn cmp_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> Option { - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 4 <= len { - // Load src[i..i+4] and src[i+1..i+5]. - let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; - let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; - let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; - let window = unsafe { _mm256_or_si256(lo, hi) }; - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; - let mask = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; - if mask != 0b1111 { - let lane = mask.trailing_ones() as usize; - let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); - return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); - } - i += 4; - } - while i < len { - let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - if sw != other[i] { - return Some(WordOrd::bitwise_cmp(sw, other[i])); - } - i += 1; - } - None - } -} - -// --------------------------------------------------------------------------- -// SSE2 — 2 × u64 per iteration -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use core::cmp::Ordering; - - use crate::WORD_BITS; - use crate::traits::WordOrd; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn cmp_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> Option { - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 2 <= len { - // Load src[i..i+2] and src[i+1..i+3]. - let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; - let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; - let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; - let window = unsafe { _mm_or_si128(lo, hi) }; - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; - let mask = unsafe { _mm_movemask_epi8(cmp) } as u32; - if mask != 0xFFFF { - let lane = mask.trailing_ones() as usize / 8; - let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); - return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); - } - i += 2; - } - while i < len { - let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - if sw != other[i] { - return Some(WordOrd::bitwise_cmp(sw, other[i])); - } - i += 1; - } - None - } -} - -// --------------------------------------------------------------------------- -// NEON — 2 × u64 per iteration -// --------------------------------------------------------------------------- +mod sse41; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use core::cmp::Ordering; - - use crate::WORD_BITS; - use crate::traits::WordOrd; - - use core::arch::aarch64::{ - vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, - }; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn cmp_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> Option { - let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; - let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 2 <= len { - let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; - let lo = unsafe { vshlq_u64(w0, neg_shift) }; - let hi = unsafe { vshlq_u64(w1, pos_shift) }; - let window = unsafe { vorrq_u64(lo, hi) }; - let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - let cmp = unsafe { vceqq_u64(window, expected) }; - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { - let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - return Some(WordOrd::bitwise_cmp(sw, other[i])); - } - if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - let sw = (src[i + 1] >> shift) | (src[i + 2] << (WORD_BITS - shift)); - return Some(WordOrd::bitwise_cmp(sw, other[i + 1])); - } - i += 2; - } - while i < len { - let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - if sw != other[i] { - return Some(WordOrd::bitwise_cmp(sw, other[i])); - } - i += 1; - } - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core/avx2.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/avx2.rs new file mode 100644 index 0000000..5f6e39d --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core/avx2.rs @@ -0,0 +1,52 @@ +use core::cmp::Ordering; + +use crate::WORD_BITS; +use crate::traits::WordOrd; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn cmp_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> Option { + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 4 <= len { + // Load src[i..i+4] and src[i+1..i+5]. + let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; + let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + let window = unsafe { _mm256_or_si256(lo, hi) }; + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; + let mask = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; + if mask != 0b1111 { + let lane = mask.trailing_ones() as usize; + let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); + return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); + } + i += 4; + } + while i < len { + let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + if sw != other[i] { + return Some(WordOrd::bitwise_cmp(sw, other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core/neon.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/neon.rs new file mode 100644 index 0000000..b4b3a91 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core/neon.rs @@ -0,0 +1,46 @@ +use core::cmp::Ordering; + +use crate::WORD_BITS; +use crate::traits::WordOrd; + +use core::arch::aarch64::{ + vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, +}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn cmp_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> Option { + let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 2 <= len { + let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; + let lo = unsafe { vshlq_u64(w0, neg_shift) }; + let hi = unsafe { vshlq_u64(w1, pos_shift) }; + let window = unsafe { vorrq_u64(lo, hi) }; + let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + let cmp = unsafe { vceqq_u64(window, expected) }; + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { + let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + return Some(WordOrd::bitwise_cmp(sw, other[i])); + } + if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + let sw = (src[i + 1] >> shift) | (src[i + 2] << (WORD_BITS - shift)); + return Some(WordOrd::bitwise_cmp(sw, other[i + 1])); + } + i += 2; + } + while i < len { + let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + if sw != other[i] { + return Some(WordOrd::bitwise_cmp(sw, other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core/sse41.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/sse41.rs new file mode 100644 index 0000000..e56e920 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core/sse41.rs @@ -0,0 +1,52 @@ +use core::cmp::Ordering; + +use crate::WORD_BITS; +use crate::traits::WordOrd; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi64x, + _mm_sll_epi64, _mm_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi64x, + _mm_sll_epi64, _mm_srl_epi64, +}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn cmp_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> Option { + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 2 <= len { + // Load src[i..i+2] and src[i+1..i+3]. + let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; + let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; + let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; + let window = unsafe { _mm_or_si128(lo, hi) }; + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; + let mask = unsafe { _mm_movemask_epi8(cmp) } as u32; + if mask != 0xFFFF { + let lane = mask.trailing_ones() as usize / 8; + let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); + return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); + } + i += 2; + } + while i < len { + let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + if sw != other[i] { + return Some(WordOrd::bitwise_cmp(sw, other[i])); + } + i += 1; + } + None +} From ef4cca8fbb30945969ee626fbe7a84b59b591952 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 09:30:46 +0000 Subject: [PATCH 30/79] refactor: split contains backends --- .../words_find/funcs_for_contains_core.rs | 291 +----------------- .../funcs_for_contains_core/avx2.rs | 89 ++++++ .../funcs_for_contains_core/neon.rs | 86 ++++++ .../funcs_for_contains_core/sse41.rs | 93 ++++++ 4 files changed, 271 insertions(+), 288 deletions(-) create mode 100644 src/traits/words_find/funcs_for_contains_core/avx2.rs create mode 100644 src/traits/words_find/funcs_for_contains_core/neon.rs create mode 100644 src/traits/words_find/funcs_for_contains_core/sse41.rs diff --git a/src/traits/words_find/funcs_for_contains_core.rs b/src/traits/words_find/funcs_for_contains_core.rs index dea6291..bef09dc 100644 --- a/src/traits/words_find/funcs_for_contains_core.rs +++ b/src/traits/words_find/funcs_for_contains_core.rs @@ -154,302 +154,17 @@ where None } -// --------------------------------------------------------------------------- -// SSE2 — 2 haystack words per iteration -// --------------------------------------------------------------------------- - #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - - const LANES: usize = 2; - - /// SSE4.1 backend: loads 2 consecutive words, computes a sliding - /// window for the current shift, and compares against the broadcast - /// needle word. `movemask` extracts match lanes. - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn find_any( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - word_limit: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = unsafe { _mm_set1_epi64x(needle_first as i64) }; - let mask = unsafe { _mm_set1_epi64x(needle_mask as i64) }; - - for shift in 0..WORD_BITS { - let mut i = 0; - while i + LANES <= word_limit { - if i * WORD_BITS + shift > last_start { - break; - } - - let window = if shift == 0 { - unsafe { _mm_loadu_si128(haystack.as_ptr().add(i).cast::<__m128i>()) } - } else { - let src = haystack.as_ptr(); - let w01 = unsafe { _mm_loadu_si128(src.add(i).cast::<__m128i>()) }; - let w12 = unsafe { _mm_loadu_si128(src.add(i + 1).cast::<__m128i>()) }; - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let lo = unsafe { _mm_srl_epi64(w01, count_lo) }; - let hi = unsafe { _mm_sll_epi64(w12, count_hi) }; - unsafe { _mm_or_si128(lo, hi) } - }; - - let masked = unsafe { _mm_and_si128(window, mask) }; - let cmp = unsafe { _mm_cmpeq_epi64(masked, needle) }; - let hits = unsafe { _mm_movemask_epi8(cmp) } as u32; - - if hits & 0xff != 0 { - let pos = i * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if hits & 0xff00 != 0 { - let pos = (i + 1) * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - - i += LANES; - } - - for j in i..word_limit { - let pos = j * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - haystack[j] - } else { - let w0 = haystack[j]; - let w1 = haystack.get(j + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// AVX2 — 4 haystack words per iteration -// --------------------------------------------------------------------------- +mod sse41; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_and_si256, _mm256_cmpeq_epi64, - _mm256_loadu_si256, _mm256_movemask_pd, _mm256_or_si256, _mm256_set1_epi64x, - _mm256_sll_epi64, _mm256_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_and_si256, _mm256_cmpeq_epi64, - _mm256_loadu_si256, _mm256_movemask_pd, _mm256_or_si256, _mm256_set1_epi64x, - _mm256_sll_epi64, _mm256_srl_epi64, - }; - - const LANES: usize = 4; - - /// AVX2 backend: same as SSE2 but with 4-lane (256-bit) vectors. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn find_any( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - word_limit: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = unsafe { _mm256_set1_epi64x(needle_first as i64) }; - let mask = unsafe { _mm256_set1_epi64x(needle_mask as i64) }; - - for shift in 0..WORD_BITS { - let mut i = 0; - while i + LANES <= word_limit { - if i * WORD_BITS + shift > last_start { - break; - } - - let window = if shift == 0 { - unsafe { _mm256_loadu_si256(haystack.as_ptr().add(i).cast::<__m256i>()) } - } else { - let src = haystack.as_ptr(); - let w0 = unsafe { _mm256_loadu_si256(src.add(i).cast::<__m256i>()) }; - let w1 = unsafe { _mm256_loadu_si256(src.add(i + 1).cast::<__m256i>()) }; - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; - let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; - unsafe { _mm256_or_si256(lo, hi) } - }; - - let masked = unsafe { _mm256_and_si256(window, mask) }; - let cmp = unsafe { _mm256_cmpeq_epi64(masked, needle) }; - let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; - - if hits != 0 { - for k in 0..LANES { - if hits & (1 << k) != 0 { - let pos = (i + k) * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - - i += LANES; - } - - for j in i..word_limit { - let pos = j * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - haystack[j] - } else { - let w0 = haystack[j]; - let w1 = haystack.get(j + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// NEON — 2 haystack words per iteration -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::*; - - use core::arch::aarch64::{ - uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, - }; - - const LANES: usize = 2; - - /// NEON backend: 2-lane comparison for aarch64. Unaligned windows - /// fall back to scalar per-position computation. - #[target_feature(enable = "neon")] - pub(super) unsafe fn find_any( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - word_limit: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = unsafe { vdupq_n_u64(needle_first) }; - let mask = unsafe { vdupq_n_u64(needle_mask) }; - - for shift in 0..WORD_BITS { - let mut i = 0; - while i < word_limit { - if i * WORD_BITS + shift > last_start { - break; - } - - if shift == 0 { - let window = unsafe { vld1q_u64(haystack.as_ptr().add(i)) }; - let masked = unsafe { vandq_u64(window, mask) }; - let cmp = unsafe { vceqq_u64(masked, needle) }; - if unsafe { vgetq_lane_u64(cmp, 0) } != 0 { - let pos = i * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if unsafe { vgetq_lane_u64(cmp, 1) } != 0 { - let pos = (i + 1) * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } else { - for k in 0..LANES { - let pos = (i + k) * WORD_BITS + shift; - if pos > last_start { - break; - } - let w0 = haystack[i + k]; - let w1 = haystack.get(i + k + 1).copied().unwrap_or(0); - let window = (w0 >> shift) | (w1 << (WORD_BITS - shift)); - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - - i += LANES; - } - - for j in i..word_limit { - let pos = j * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - haystack[j] - } else { - let w0 = haystack[j]; - let w1 = haystack.get(j + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_find/funcs_for_contains_core/avx2.rs b/src/traits/words_find/funcs_for_contains_core/avx2.rs new file mode 100644 index 0000000..a4b7d40 --- /dev/null +++ b/src/traits/words_find/funcs_for_contains_core/avx2.rs @@ -0,0 +1,89 @@ +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, + _mm256_movemask_pd, _mm256_or_si256, _mm256_set1_epi64x, _mm256_sll_epi64, _mm256_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, + _mm256_movemask_pd, _mm256_or_si256, _mm256_set1_epi64x, _mm256_sll_epi64, _mm256_srl_epi64, +}; + +const LANES: usize = 4; + +/// AVX2 backend: same as SSE2 but with 4-lane (256-bit) vectors. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn find_any( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + word_limit: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = unsafe { _mm256_set1_epi64x(needle_first as i64) }; + let mask = unsafe { _mm256_set1_epi64x(needle_mask as i64) }; + + for shift in 0..WORD_BITS { + let mut i = 0; + while i + LANES <= word_limit { + if i * WORD_BITS + shift > last_start { + break; + } + + let window = if shift == 0 { + unsafe { _mm256_loadu_si256(haystack.as_ptr().add(i).cast::<__m256i>()) } + } else { + let src = haystack.as_ptr(); + let w0 = unsafe { _mm256_loadu_si256(src.add(i).cast::<__m256i>()) }; + let w1 = unsafe { _mm256_loadu_si256(src.add(i + 1).cast::<__m256i>()) }; + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + unsafe { _mm256_or_si256(lo, hi) } + }; + + let masked = unsafe { _mm256_and_si256(window, mask) }; + let cmp = unsafe { _mm256_cmpeq_epi64(masked, needle) }; + let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; + + if hits != 0 { + for k in 0..LANES { + if hits & (1 << k) != 0 { + let pos = (i + k) * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + + i += LANES; + } + + for j in i..word_limit { + let pos = j * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + haystack[j] + } else { + let w0 = haystack[j]; + let w1 = haystack.get(j + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_contains_core/neon.rs b/src/traits/words_find/funcs_for_contains_core/neon.rs new file mode 100644 index 0000000..629631a --- /dev/null +++ b/src/traits/words_find/funcs_for_contains_core/neon.rs @@ -0,0 +1,86 @@ +use super::*; + +use core::arch::aarch64::{ + uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, +}; + +const LANES: usize = 2; + +/// NEON backend: 2-lane comparison for aarch64. Unaligned windows +/// fall back to scalar per-position computation. +#[target_feature(enable = "neon")] +pub(super) unsafe fn find_any( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + word_limit: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = unsafe { vdupq_n_u64(needle_first) }; + let mask = unsafe { vdupq_n_u64(needle_mask) }; + + for shift in 0..WORD_BITS { + let mut i = 0; + while i < word_limit { + if i * WORD_BITS + shift > last_start { + break; + } + + if shift == 0 { + let window = unsafe { vld1q_u64(haystack.as_ptr().add(i)) }; + let masked = unsafe { vandq_u64(window, mask) }; + let cmp = unsafe { vceqq_u64(masked, needle) }; + if unsafe { vgetq_lane_u64(cmp, 0) } != 0 { + let pos = i * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if unsafe { vgetq_lane_u64(cmp, 1) } != 0 { + let pos = (i + 1) * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } else { + for k in 0..LANES { + let pos = (i + k) * WORD_BITS + shift; + if pos > last_start { + break; + } + let w0 = haystack[i + k]; + let w1 = haystack.get(i + k + 1).copied().unwrap_or(0); + let window = (w0 >> shift) | (w1 << (WORD_BITS - shift)); + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + + i += LANES; + } + + for j in i..word_limit { + let pos = j * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + haystack[j] + } else { + let w0 = haystack[j]; + let w1 = haystack.get(j + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_contains_core/sse41.rs b/src/traits/words_find/funcs_for_contains_core/sse41.rs new file mode 100644 index 0000000..e3fda74 --- /dev/null +++ b/src/traits/words_find/funcs_for_contains_core/sse41.rs @@ -0,0 +1,93 @@ +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, + _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, + _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, +}; + +const LANES: usize = 2; + +/// SSE4.1 backend: loads 2 consecutive words, computes a sliding +/// window for the current shift, and compares against the broadcast +/// needle word. `movemask` extracts match lanes. +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn find_any( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + word_limit: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = unsafe { _mm_set1_epi64x(needle_first as i64) }; + let mask = unsafe { _mm_set1_epi64x(needle_mask as i64) }; + + for shift in 0..WORD_BITS { + let mut i = 0; + while i + LANES <= word_limit { + if i * WORD_BITS + shift > last_start { + break; + } + + let window = if shift == 0 { + unsafe { _mm_loadu_si128(haystack.as_ptr().add(i).cast::<__m128i>()) } + } else { + let src = haystack.as_ptr(); + let w01 = unsafe { _mm_loadu_si128(src.add(i).cast::<__m128i>()) }; + let w12 = unsafe { _mm_loadu_si128(src.add(i + 1).cast::<__m128i>()) }; + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let lo = unsafe { _mm_srl_epi64(w01, count_lo) }; + let hi = unsafe { _mm_sll_epi64(w12, count_hi) }; + unsafe { _mm_or_si128(lo, hi) } + }; + + let masked = unsafe { _mm_and_si128(window, mask) }; + let cmp = unsafe { _mm_cmpeq_epi64(masked, needle) }; + let hits = unsafe { _mm_movemask_epi8(cmp) } as u32; + + if hits & 0xff != 0 { + let pos = i * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if hits & 0xff00 != 0 { + let pos = (i + 1) * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + + i += LANES; + } + + for j in i..word_limit { + let pos = j * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + haystack[j] + } else { + let w0 = haystack[j]; + let w1 = haystack.get(j + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + + None +} From e7a5b4140d334dcd7b3a2a0cc3d9ddcf179a750f Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 11:23:46 +0000 Subject: [PATCH 31/79] fix: compare anonymous relocation targets semantically --- scripts/compare_codegen.py | 319 +++++++++++++++++++++++++++++++--- scripts/tests/test_codegen.py | 158 +++++++++++++++++ 2 files changed, 455 insertions(+), 22 deletions(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index 18938bc..ceffa36 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -23,6 +23,7 @@ SYMBOL_TARGET = re.compile(r"(?:0x)?([0-9a-f]+)\s+<([^>]+)>") PC_RELATIVE_X86_RELOCATIONS = {"R_X86_64_PC32", "R_X86_64_PLT32"} X86_PC_BIAS = re.compile(r"-0x4$") +ADDEND = re.compile(r"^(?P.*?)(?P[+-]0x[0-9a-f]+)?$") ANONYMOUS_SYMBOL_INSTANCE = re.compile( r"(?P(?:\.Lanon|anon)\.[^.\s]+)\.(?P\d+)" r"(?:\.llvm\.[^+\-\s]+)?(?P[+-]0x[0-9a-f]+)?$" @@ -37,6 +38,201 @@ class Instruction: encoded: bytes assembly: str relocations: list[tuple[str, str]] = field(default_factory=list) + member: str = "" + + +@dataclass(frozen=True) +class Relocation: + kind: str + raw_target: str + token: str + anonymous: bool + resolved: bool + instruction_index: int + + +@dataclass(frozen=True) +class SectionRelocation: + offset: int + kind: str + target: str + + +@dataclass(frozen=True) +class SectionData: + member: str + name: str + data: bytes + relocations: tuple[SectionRelocation, ...] + + +@dataclass(frozen=True) +class ResolvedTarget: + token: str + anonymous: bool + resolved: bool + + +class ArtifactSections: + """Small, conservative index of sections needed by anonymous relocations.""" + + SECTION_HEADER = re.compile(r"^Contents of section (.+):$") + FILE_FORMAT = re.compile(r"^(?P.+):\s+file format .+$") + RELOCATION_HEADER = re.compile(r"^RELOCATION RECORDS FOR \[(.+)\]:$") + RELOCATION_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") + + def __init__(self, sections: dict[tuple[str, str], SectionData], arch: str) -> None: + self.sections = sections + self.arch = arch + + @classmethod + def from_objdump(cls, output: str, arch: str) -> ArtifactSections: + member = "" + section_name: str | None = None + section_bytes: dict[tuple[str, str], bytearray] = {} + section_relocations: dict[tuple[str, str], list[SectionRelocation]] = {} + relocation_section: str | None = None + for line in output.splitlines(): + file_format = cls.FILE_FORMAT.match(line) + if file_format: + member = file_format.group("member") + section_name = None + relocation_section = None + continue + section = cls.SECTION_HEADER.match(line) + if section: + section_name = section.group(1) + section_bytes.setdefault((member, section_name), bytearray()) + relocation_section = None + continue + relocation_header = cls.RELOCATION_HEADER.match(line) + if relocation_header: + relocation_section = relocation_header.group(1) + section_relocations.setdefault((member, relocation_section), []) + section_name = None + continue + relocation = cls.RELOCATION_ENTRY.match(line) + if relocation and relocation_section is not None: + section_relocations[(member, relocation_section)].append( + SectionRelocation(int(relocation.group(1), 16), relocation.group(2), relocation.group(3)) + ) + continue + if section_name is None: + continue + fields = line.split() + if len(fields) < 2 or not re.fullmatch(r"[0-9a-f]+", fields[0]): + continue + chunks: list[str] = [] + for field in fields[1:]: + if len(field) in {2, 4, 6, 8} and len(field) % 2 == 0 and re.fullmatch(r"[0-9a-fA-F]+", field): + chunks.append(field) + else: + break + if not chunks: + continue + data = section_bytes[(member, section_name)] + offset = int(fields[0], 16) + encoded = bytes.fromhex("".join(chunks)) + end = offset + len(encoded) + if len(data) < end: + data.extend(b"\0" * (end - len(data))) + data[offset:end] = encoded + sections = { + key: SectionData(key[0], key[1], bytes(data), tuple(section_relocations.get(key, ()))) + for key, data in section_bytes.items() + } + return cls(sections, arch) + + def _find(self, member: str, name: str) -> SectionData | None: + exact = self.sections.get((member, name)) + if exact is not None: + return exact + candidates = [ + section + for (candidate_member, candidate_name), section in self.sections.items() + if candidate_member == member + and (candidate_name == name or candidate_name.endswith(f".{name}")) + ] + if not candidates: + candidates = [ + section + for (candidate_member, candidate_name), section in self.sections.items() + if candidate_name == name or candidate_name.endswith(f".{name}") + ] + return candidates[0] if len(candidates) == 1 else None + + @staticmethod + def _split_target(value: str) -> tuple[str, str]: + match = ADDEND.match(value.strip()) + assert match is not None + return match.group("base"), match.group("addend") or "" + + @staticmethod + def _addend(value: str) -> int: + return int(value, 16) if value.startswith("+0x") else -int(value[3:], 16) + + def _string_target(self, member: str, name: str, addend: str) -> str | None: + if not name.startswith(".rodata.str"): + return None + section = self._find(member, name) + if section is None: + return None + offset = self._addend(addend) if addend else 0 + if offset < 0 or offset >= len(section.data): + return None + value = section.data[offset:] + return value.split(b"\0", 1)[0].hex() + + def _fingerprint(self, section: SectionData, member: str) -> str | None: + data = bytearray(section.data) + descriptors: list[str] = [] + widths = { + "R_X86_64_64": 8, + "R_X86_64_32": 4, + "R_X86_64_PC32": 4, + "R_X86_64_PLT32": 4, + "R_X86_64_GOTPCREL": 4, + "R_AARCH64_ABS64": 8, + "R_AARCH64_ADR_PREL_PG_HI21": 4, + "R_AARCH64_ADD_ABS_LO12_NC": 4, + "R_AARCH64_LDST64_ABS_LO12_NC": 4, + } + for relocation in section.relocations: + width = widths.get(relocation.kind) + if width is None or relocation.offset < 0 or relocation.offset + width > len(data): + return None + data[relocation.offset : relocation.offset + width] = b"\0" * width + target = relocation.target + if target in {"", ".text"}: + descriptor_target = target + else: + target_name, addend = self._split_target(target) + if _anonymous_instance(target_name) is not None: + return None + string = self._string_target(member, target_name, addend) + descriptor_target = f"string:{string}" if string is not None else f"symbol:{target_name}{addend}" + descriptors.append(f"{relocation.offset}:{relocation.kind}:{descriptor_target}") + payload = bytes(data) + b"\0" + "\n".join(descriptors).encode() + return hashlib.sha256(payload).hexdigest() + + def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: + target = value.strip() + if kind in PC_RELATIVE_X86_RELOCATIONS: + target = X86_PC_BIAS.sub("", target) + anonymous = _anonymous_instance(target) + if anonymous is None: + name, addend = self._split_target(target) + string = self._string_target(member, name, addend) + token = f"string:{string}" if string is not None else f"symbol:{name}{addend}" + return ResolvedTarget(token, False, True) + name, addend = self._split_target(target) + section = self._find(member, name) + if section is None: + return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False) + fingerprint = self._fingerprint(section, member) + if fingerprint is None: + return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False) + return ResolvedTarget(f"anonymous:{fingerprint}:{addend}", True, True) @dataclass(frozen=True) @@ -45,10 +241,13 @@ class Function: encoded: bytes calls: int branches: int + executable: tuple[str, ...] = () + relocations: tuple[Relocation, ...] = () + alias_partition: tuple[int, ...] = () @property def digest(self) -> str: - return hashlib.sha256("\n".join(self.instructions).encode()).hexdigest()[:16] + return hashlib.sha256("\n".join(self.executable).encode()).hexdigest()[:16] @dataclass(frozen=True) @@ -59,11 +258,40 @@ class Root: def classify(old: Function, new: Function) -> str: - if old.instructions != new.instructions: + if old.executable != new.executable: return "CODEGEN CHANGED" - if len(old.encoded) != len(new.encoded): + if old.encoded != new.encoded: return "CODEGEN ENCODING CHANGED" - return "CODEGEN IDENTICAL" + if _same_relocations(old.relocations, new.relocations) and old.alias_partition == new.alias_partition: + return "CODEGEN IDENTICAL" + if _metadata_only_relocations(old, new): + return "CODEGEN METADATA CHANGED" + return "CODEGEN CHANGED" + + +def _same_relocations(old: tuple[Relocation, ...], new: tuple[Relocation, ...]) -> bool: + return all( + before.kind == after.kind + and before.token == after.token + and before.anonymous == after.anonymous + and before.resolved == after.resolved + and (not before.anonymous or before.resolved) + for before, after in zip(old, new) + ) and len(old) == len(new) + + +def _metadata_only_relocations(old: Function, new: Function) -> bool: + if old.alias_partition != new.alias_partition or len(old.relocations) != len(new.relocations): + return False + changed = False + for before, after in zip(old.relocations, new.relocations): + if before.kind != after.kind or before.anonymous != after.anonymous: + return False + if before.token != after.token: + if not before.anonymous or not before.resolved or not after.resolved: + return False + changed = True + return changed def _anonymous_instance(value: str) -> tuple[str, int, str] | None: @@ -102,7 +330,7 @@ def _has_control_target(mnemonic: str, arch: str) -> bool: return mnemonic in {"b", "bl", "blr", "br", "cbz", "cbnz", "tbz", "tbnz"} or mnemonic.startswith("b.") -def _normalize(instructions: list[Instruction], arch: str) -> Function: +def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSections | None = None) -> Function: indexes = {instruction.address: index for index, instruction in enumerate(instructions)} anonymous_bases: dict[str, int] = {} for instruction in instructions: @@ -113,8 +341,12 @@ def _normalize(instructions: list[Instruction], arch: str) -> Function: key, instance, _ = anonymous anonymous_bases[key] = min(instance, anonymous_bases.get(key, instance)) normalized: list[str] = [] + executable: list[str] = [] + relocations: list[Relocation] = [] + aliases: dict[str, int] = {} + alias_partition: list[int] = [] calls = branches = 0 - for instruction in instructions: + for instruction_index, instruction in enumerate(instructions): assembly = OBJDUMP_COMMENT.sub("", instruction.assembly).strip() if not assembly: continue @@ -130,20 +362,49 @@ def _normalize(instructions: list[Instruction], arch: str) -> Function: symbol = match.group(2) target = f"@instruction_{indexes[address]}" if address in indexes and "+0x" in symbol else f"<{symbol}>" operands = SYMBOL_TARGET.sub(target, operands) - relocations = " ".join( - f"[{kind}:{_relocation_target(kind, target, anonymous_bases)}]" - for kind, target in instruction.relocations - ) - normalized.append(" ".join(f"{mnemonic} {operands} {relocations}".split())) - return Function(tuple(normalized), b"".join(item.encoded for item in instructions), calls, branches) - - -def parse_objdump(output: str, arch: str) -> dict[str, Function]: + relocation_text: list[str] = [] + for kind, target in instruction.relocations: + if sections is None: + token = _relocation_target(kind, target, anonymous_bases) + anonymous = _anonymous_instance(target) is not None + resolved = True + else: + resolved_target = sections.resolve(instruction.member, kind, target) + token = resolved_target.token + anonymous = resolved_target.anonymous + resolved = resolved_target.resolved + relocation_text.append(f"[{kind}:{token}]") + relocations.append(Relocation(kind, target, token, anonymous, resolved, instruction_index)) + if anonymous: + alias_key = target + if kind in PC_RELATIVE_X86_RELOCATIONS: + alias_key = X86_PC_BIAS.sub("", alias_key) + alias = aliases.setdefault(alias_key, len(aliases)) + alias_partition.append(alias) + executable.append(" ".join(f"{mnemonic} {operands}".split())) + normalized.append(" ".join(f"{mnemonic} {operands} {' '.join(relocation_text)}".split())) + return Function( + tuple(normalized), + b"".join(item.encoded for item in instructions), + calls, + branches, + tuple(executable), + tuple(relocations), + tuple(alias_partition), + ) + + +def parse_objdump(output: str, arch: str, sections: ArtifactSections | None = None) -> dict[str, Function]: instruction_pattern = X86_INSTRUCTION if arch == "x86_64" else AARCH64_INSTRUCTION raw: dict[str, list[Instruction]] = {} current: str | None = None + current_member = "" last_instruction: Instruction | None = None for line in output.splitlines(): + file_format = ArtifactSections.FILE_FORMAT.match(line) + if file_format: + current_member = file_format.group("member") + continue label = FUNCTION.match(line) if label: current = label.group(1) @@ -171,14 +432,16 @@ def parse_objdump(output: str, arch: str) -> dict[str, Function]: relocations.append((inline_relocation.group(2), inline_relocation.group(3))) if not assembly: continue - last_instruction = Instruction(int(match.group(1), 16), encoded, assembly, relocations) + last_instruction = Instruction(int(match.group(1), 16), encoded, assembly, relocations, current_member) raw[current].append(last_instruction) - return {name: _normalize(items, arch) for name, items in raw.items() if items} + return {name: _normalize(items, arch, sections) for name, items in raw.items() if items} def load_functions(artifact: Path, objdump: str, arch: str) -> dict[str, Function]: output = subprocess.check_output([objdump, "-drwC", str(artifact)], text=True, errors="replace") - return parse_objdump(output, arch) + section_output = subprocess.check_output([objdump, "-sr", str(artifact)], text=True, errors="replace") + sections = ArtifactSections.from_objdump(section_output, arch) + return parse_objdump(output, arch, sections) def load_roots(path: Path) -> list[Root]: @@ -226,7 +489,7 @@ def main() -> int: before = load_functions(args.baseline, args.objdump, args.arch) after = load_functions(args.current, args.objdump, args.arch) roots = [root for root in load_roots(args.roots) if root.artifact == args.artifact] - identical = encoding_changed = changed = errors = skipped = 0 + identical = metadata_changed = encoding_changed = changed = errors = skipped = 0 if args.artifact == "library": if args.inventory is None: parser.error("--inventory is required for library artifacts") @@ -250,6 +513,8 @@ def main() -> int: status = classify(old, new) if status == "CODEGEN IDENTICAL": identical += 1 + elif status == "CODEGEN METADATA CHANGED": + metadata_changed += 1 elif status == "CODEGEN ENCODING CHANGED": encoding_changed += 1 else: @@ -260,18 +525,28 @@ def main() -> int: print(f" calls {old.calls} -> {new.calls}") print(f" branches {old.branches} -> {new.branches}") print(f" asm hash {old.digest} -> {new.digest}") + if status == "CODEGEN METADATA CHANGED": + for old_relocation, new_relocation in zip(old.relocations, new.relocations): + if old_relocation.token != new_relocation.token: + print(f" metadata {old_relocation.raw_target} -> {new_relocation.raw_target}") + print(f" semantic {old_relocation.token} -> {new_relocation.token}") if new.calls > old.calls: print(f"WARNING: {root.symbol} call count increased from {old.calls} to {new.calls}", file=sys.stderr) if not same_instructions: - print("\n".join(difflib.unified_diff(old.instructions, new.instructions, "baseline", "current", lineterm=""))) + print( + "\n".join( + difflib.unified_diff(old.instructions, new.instructions, "baseline", "current", lineterm="") + ) + ) print("\nCODEGEN SUMMARY") - print(f"compared : {identical + encoding_changed + changed}") + print(f"compared : {identical + metadata_changed + encoding_changed + changed}") print(f"identical : {identical}") + print(f"metadata changed : {metadata_changed}") print(f"encoding changed : {encoding_changed}") print(f"changed : {changed}") print(f"errors : {errors}") print(f"skipped : {skipped}") - return 0 if not (encoding_changed or changed or errors) else 1 + return 0 if not (metadata_changed or encoding_changed or changed or errors) else 1 if __name__ == "__main__": diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 5e47723..5a25c5d 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -18,7 +18,165 @@ def parse(body: str, arch: str = "x86_64") -> compare_codegen.Function: return compare_codegen.parse_objdump(body, arch)["root"] +def parse_with_sections(body: str, sections: str, arch: str = "x86_64") -> compare_codegen.Function: + index = compare_codegen.ArtifactSections.from_objdump(sections, arch) + return compare_codegen.parse_objdump(body, arch, index)["root"] + + class CodegenParserTests(unittest.TestCase): + SECTION_PREFIX = ( + "fixture.o: file format elf64-x86-64\n" + "Contents of section .rodata.str1.1:\n" + " 0000 66696c65 2e727300 file.rs.\n" + ) + + @classmethod + def anonymous_section(cls, ordinal: int, line: int = 1, column: int = 2) -> str: + line_bytes = line.to_bytes(4, "little").hex() + column_bytes = column.to_bytes(4, "little").hex() + return ( + f"Contents of section .data.rel.ro..Lanon.hash.{ordinal}:\n" + f" 0000 00000000 00000000 {line_bytes} {column_bytes} ........\n" + "RELOCATION RECORDS FOR [.data.rel.ro..Lanon.hash." + f"{ordinal}]:\n" + "OFFSET TYPE VALUE\n" + "0000000000000000 R_X86_64_64 .rodata.str1.1+0x0000000000000000\n" + ) + + def test_anonymous_section_content_ignores_ordinal(self) -> None: + old_sections = self.SECTION_PREFIX + self.anonymous_section(1) + new_sections = self.SECTION_PREFIX + self.anonymous_section(9) + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", + old_sections, + ) + new = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.9-0x4\n", + new_sections, + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") + + def test_anonymous_section_content_change_is_metadata_change(self) -> None: + old_sections = self.SECTION_PREFIX + self.anonymous_section(1, line=1) + new_sections = self.SECTION_PREFIX + self.anonymous_section(9, line=3) + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", + old_sections, + ) + new = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.9-0x4\n", + new_sections, + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN METADATA CHANGED") + + def test_anonymous_section_alias_partition_is_preserved(self) -> None: + sections = self.SECTION_PREFIX + self.anonymous_section(1) + self.anonymous_section(2) + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n" + " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " a: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", + sections, + ) + new = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n" + " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " a: R_X86_64_PC32 .data.rel.ro..Lanon.hash.2-0x4\n", + sections, + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_unresolved_anonymous_section_is_conservative(self) -> None: + sections = "fixture.o: file format elf64-x86-64\n" + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", + sections, + ) + new = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.9-0x4\n", + sections, + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_named_relocation_change_is_not_metadata(self) -> None: + old = parse( + "0000 :\n" + " 0: e8 00 00 00 00 call 5 \n" + " 1: R_X86_64_PLT32 foo-0x4" + ) + new = parse( + "0000 :\n" + " 0: e8 00 00 00 00 call 5 \n" + " 1: R_X86_64_PLT32 bar-0x4" + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_relocation_addend_change_is_not_metadata(self) -> None: + old = parse( + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .rodata+0x20" + ) + new = parse( + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .rodata+0x40" + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_aarch64_anonymous_section_identity_ignores_ordinal(self) -> None: + def sections(ordinal: int) -> str: + return ( + "fixture.o: file format elf64-littleaarch64\n" + "Contents of section .rodata.str1.1:\n" + " 0000 66696c65 2e727300 file.rs.\n" + f"Contents of section .data.rel.ro..Lanon.hash.{ordinal}:\n" + " 0000 00000000 00000000 01000000 02000000 ........\n" + f"RELOCATION RECORDS FOR [.data.rel.ro..Lanon.hash.{ordinal}]:\n" + "OFFSET TYPE VALUE\n" + "0000000000000000 R_AARCH64_ABS64 .rodata.str1.1+0x0000000000000000\n" + ) + + old = parse_with_sections( + "fixture.o: file format elf64-littleaarch64\n" + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 \n" + " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.1\n", + sections(1), + "aarch64", + ) + new = parse_with_sections( + "fixture.o: file format elf64-littleaarch64\n" + "0000000000000000 :\n" + " 0: 90000000 adrp x0, 0 \n" + " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.9\n", + sections(9), + "aarch64", + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") def test_call_target_identity_is_preserved(self) -> None: foo = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 foo-0x4") bar = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 bar-0x4") From bc0b1c94db5ce9d6057f029f5cf2aaf86f7296fd Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 12:24:38 +0000 Subject: [PATCH 32/79] fix: preserve relocation identity during comparison --- scripts/compare_codegen.py | 69 ++++++++++++-------- scripts/tests/test_codegen.py | 115 ++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 25 deletions(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index ceffa36..4facf0d 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -37,18 +37,20 @@ class Instruction: address: int encoded: bytes assembly: str - relocations: list[tuple[str, str]] = field(default_factory=list) + relocations: list[tuple[int, str, str]] = field(default_factory=list) member: str = "" @dataclass(frozen=True) class Relocation: + offset: int kind: str raw_target: str token: str anonymous: bool resolved: bool instruction_index: int + addend: str = "" @dataclass(frozen=True) @@ -71,6 +73,7 @@ class ResolvedTarget: token: str anonymous: bool resolved: bool + addend: str = "" class ArtifactSections: @@ -123,9 +126,13 @@ def from_objdump(cls, output: str, arch: str) -> ArtifactSections: if len(fields) < 2 or not re.fullmatch(r"[0-9a-f]+", fields[0]): continue chunks: list[str] = [] - for field in fields[1:]: - if len(field) in {2, 4, 6, 8} and len(field) % 2 == 0 and re.fullmatch(r"[0-9a-fA-F]+", field): - chunks.append(field) + for chunk_field in fields[1:5]: + if ( + len(chunk_field) in {2, 4, 6, 8} + and len(chunk_field) % 2 == 0 + and re.fullmatch(r"[0-9a-fA-F]+", chunk_field) + ): + chunks.append(chunk_field) else: break if not chunks: @@ -153,12 +160,6 @@ def _find(self, member: str, name: str) -> SectionData | None: if candidate_member == member and (candidate_name == name or candidate_name.endswith(f".{name}")) ] - if not candidates: - candidates = [ - section - for (candidate_member, candidate_name), section in self.sections.items() - if candidate_name == name or candidate_name.endswith(f".{name}") - ] return candidates[0] if len(candidates) == 1 else None @staticmethod @@ -186,6 +187,8 @@ def _string_target(self, member: str, name: str, addend: str) -> str | None: def _fingerprint(self, section: SectionData, member: str) -> str | None: data = bytearray(section.data) descriptors: list[str] = [] + section_identity = _anonymous_instance(section.name) + section_class = section_identity[0] if section_identity is not None else section.name widths = { "R_X86_64_64": 8, "R_X86_64_32": 4, @@ -212,7 +215,7 @@ def _fingerprint(self, section: SectionData, member: str) -> str | None: string = self._string_target(member, target_name, addend) descriptor_target = f"string:{string}" if string is not None else f"symbol:{target_name}{addend}" descriptors.append(f"{relocation.offset}:{relocation.kind}:{descriptor_target}") - payload = bytes(data) + b"\0" + "\n".join(descriptors).encode() + payload = section_class.encode() + b"\0" + bytes(data) + b"\0" + "\n".join(descriptors).encode() return hashlib.sha256(payload).hexdigest() def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: @@ -224,15 +227,15 @@ def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: name, addend = self._split_target(target) string = self._string_target(member, name, addend) token = f"string:{string}" if string is not None else f"symbol:{name}{addend}" - return ResolvedTarget(token, False, True) + return ResolvedTarget(token, False, True, addend) name, addend = self._split_target(target) section = self._find(member, name) if section is None: - return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False) + return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False, addend) fingerprint = self._fingerprint(section, member) if fingerprint is None: - return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False) - return ResolvedTarget(f"anonymous:{fingerprint}:{addend}", True, True) + return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False, addend) + return ResolvedTarget(f"anonymous:{fingerprint}", True, True, addend) @dataclass(frozen=True) @@ -271,10 +274,13 @@ def classify(old: Function, new: Function) -> str: def _same_relocations(old: tuple[Relocation, ...], new: tuple[Relocation, ...]) -> bool: return all( - before.kind == after.kind + before.offset == after.offset + and before.instruction_index == after.instruction_index + and before.kind == after.kind and before.token == after.token and before.anonymous == after.anonymous and before.resolved == after.resolved + and before.addend == after.addend and (not before.anonymous or before.resolved) for before, after in zip(old, new) ) and len(old) == len(new) @@ -285,7 +291,13 @@ def _metadata_only_relocations(old: Function, new: Function) -> bool: return False changed = False for before, after in zip(old.relocations, new.relocations): - if before.kind != after.kind or before.anonymous != after.anonymous: + if ( + before.offset != after.offset + or before.instruction_index != after.instruction_index + or before.kind != after.kind + or before.anonymous != after.anonymous + or before.addend != after.addend + ): return False if before.token != after.token: if not before.anonymous or not before.resolved or not after.resolved: @@ -334,7 +346,7 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec indexes = {instruction.address: index for index, instruction in enumerate(instructions)} anonymous_bases: dict[str, int] = {} for instruction in instructions: - for _, target in instruction.relocations: + for _, _, target in instruction.relocations: anonymous = _anonymous_instance(target.strip()) if anonymous is None: continue @@ -363,23 +375,28 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec target = f"@instruction_{indexes[address]}" if address in indexes and "+0x" in symbol else f"<{symbol}>" operands = SYMBOL_TARGET.sub(target, operands) relocation_text: list[str] = [] - for kind, target in instruction.relocations: + for relocation_offset, kind, target in instruction.relocations: if sections is None: token = _relocation_target(kind, target, anonymous_bases) anonymous = _anonymous_instance(target) is not None resolved = True + addend = _anonymous_instance(target)[2] if anonymous else "" else: resolved_target = sections.resolve(instruction.member, kind, target) token = resolved_target.token anonymous = resolved_target.anonymous resolved = resolved_target.resolved + addend = resolved_target.addend relocation_text.append(f"[{kind}:{token}]") - relocations.append(Relocation(kind, target, token, anonymous, resolved, instruction_index)) + relocations.append( + Relocation(relocation_offset, kind, target, token, anonymous, resolved, instruction_index, addend) + ) if anonymous: - alias_key = target + alias_target = X86_PC_BIAS.sub("", target) if kind in PC_RELATIVE_X86_RELOCATIONS else target + alias_key, _ = ArtifactSections._split_target(alias_target) if kind in PC_RELATIVE_X86_RELOCATIONS: alias_key = X86_PC_BIAS.sub("", alias_key) - alias = aliases.setdefault(alias_key, len(aliases)) + alias = aliases.setdefault(f"{instruction.member}:{alias_key}", len(aliases)) alias_partition.append(alias) executable.append(" ".join(f"{mnemonic} {operands}".split())) normalized.append(" ".join(f"{mnemonic} {operands} {' '.join(relocation_text)}".split())) @@ -418,18 +435,20 @@ def parse_objdump(output: str, arch: str, sections: ArtifactSections | None = No relocation = RELOCATION.match(line) if relocation: if last_instruction is not None: - last_instruction.relocations.append((relocation.group(2), relocation.group(3))) + relocation_offset = int(relocation.group(1), 16) - last_instruction.address + last_instruction.relocations.append((relocation_offset, relocation.group(2), relocation.group(3))) continue match = instruction_pattern.match(line) if not match: continue encoded = bytes.fromhex(match.group(2)) assembly = match.group(3).strip() - relocations: list[tuple[str, str]] = [] + relocations: list[tuple[int, str, str]] = [] inline_relocation = INLINE_RELOCATION.search(assembly) if inline_relocation is not None: assembly = assembly[: inline_relocation.start()].rstrip() - relocations.append((inline_relocation.group(2), inline_relocation.group(3))) + relocation_offset = int(inline_relocation.group(1), 16) - int(match.group(1), 16) + relocations.append((relocation_offset, inline_relocation.group(2), inline_relocation.group(3))) if not assembly: continue last_instruction = Instruction(int(match.group(1), 16), encoded, assembly, relocations, current_member) diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 5a25c5d..5ef396e 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -62,6 +62,34 @@ def test_anonymous_section_content_ignores_ordinal(self) -> None: ) self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") + def test_relocation_site_change_is_not_ignored(self) -> None: + old = parse( + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 foo-0x4\n" + " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax" + ) + new = parse( + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " a: R_X86_64_PC32 foo-0x4" + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_relocation_offset_change_is_not_ignored(self) -> None: + old = parse( + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 foo-0x4" + ) + new = parse( + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 4: R_X86_64_PC32 foo-0x4" + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + def test_anonymous_section_content_change_is_metadata_change(self) -> None: old_sections = self.SECTION_PREFIX + self.anonymous_section(1, line=1) new_sections = self.SECTION_PREFIX + self.anonymous_section(9, line=3) @@ -177,6 +205,93 @@ def sections(ordinal: int) -> str: "aarch64", ) self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") + + def test_anonymous_addend_change_is_not_metadata(self) -> None: + sections = self.SECTION_PREFIX + self.anonymous_section(1) + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1+0x0", + sections, + ) + new = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1+0x8", + sections, + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_alias_identity_excludes_addend(self) -> None: + sections = ( + self.SECTION_PREFIX + + self.anonymous_section(1) + + self.anonymous_section(2) + + self.anonymous_section(3) + ) + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1+0x0\n" + " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " a: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1+0x8", + sections, + ) + new = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.2+0x0\n" + " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " a: R_X86_64_PC32 .data.rel.ro..Lanon.hash.3+0x8", + sections, + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_section_class_is_part_of_anonymous_identity(self) -> None: + old_sections = self.SECTION_PREFIX + self.anonymous_section(1) + new_sections = self.SECTION_PREFIX + self.anonymous_section(9).replace( + ".data.rel.ro..Lanon", ".rodata..Lanon" + ) + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4", + old_sections, + ) + new = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .rodata..Lanon.hash.9-0x4", + new_sections, + ) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN METADATA CHANGED") + + def test_section_lookup_does_not_cross_archive_members(self) -> None: + sections = ( + "member-a.o: file format elf64-x86-64\n" + "member-b.o: file format elf64-x86-64\n" + "Contents of section .data.rel.ro..Lanon.hash.1:\n" + " 0000 00000000 00000000 01000000 02000000 ........\n" + ) + index = compare_codegen.ArtifactSections.from_objdump(sections, "x86_64") + resolved = index.resolve("member-a.o", "R_X86_64_PC32", ".data.rel.ro..Lanon.hash.1-0x4") + self.assertFalse(resolved.resolved) + + def test_section_bytes_ignore_ascii_column_after_four_groups(self) -> None: + output = ( + "fixture.o: file format elf64-x86-64\n" + "Contents of section .rodata:\n" + " 0000 00000000 00000000 00000000 00000000 deadbeef\n" + ) + index = compare_codegen.ArtifactSections.from_objdump(output, "x86_64") + section = index.sections[("fixture.o", ".rodata")] + self.assertEqual(section.data, b"\0" * 16) def test_call_target_identity_is_preserved(self) -> None: foo = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 foo-0x4") bar = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 bar-0x4") From badb700c2ade6bcbed5e8acb27678a2632476eaf Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 12:49:46 +0000 Subject: [PATCH 33/79] fix: resolve one-level nested anonymous targets --- scripts/compare_codegen.py | 123 +++++++++++++++++++++++++++++----- scripts/tests/test_codegen.py | 93 +++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 16 deletions(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index 4facf0d..b274b9e 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -51,6 +51,7 @@ class Relocation: resolved: bool instruction_index: int addend: str = "" + nested_addends: tuple[str, ...] = () @dataclass(frozen=True) @@ -74,6 +75,7 @@ class ResolvedTarget: anonymous: bool resolved: bool addend: str = "" + nested_addends: tuple[str, ...] = () class ArtifactSections: @@ -83,13 +85,25 @@ class ArtifactSections: FILE_FORMAT = re.compile(r"^(?P.+):\s+file format .+$") RELOCATION_HEADER = re.compile(r"^RELOCATION RECORDS FOR \[(.+)\]:$") RELOCATION_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") - - def __init__(self, sections: dict[tuple[str, str], SectionData], arch: str) -> None: + SYMBOL_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+\S+\s+\S+\s+(\S+)\s+([0-9a-f]+)\s+(.+?)\s*$") + + def __init__( + self, + sections: dict[tuple[str, str], SectionData], + arch: str, + symbols: dict[tuple[str, str], tuple[str, int, int]] | None = None, + ) -> None: self.sections = sections self.arch = arch + self.symbols = symbols or {} @classmethod - def from_objdump(cls, output: str, arch: str) -> ArtifactSections: + def from_objdump( + cls, + output: str, + arch: str, + symbols_output: str = "", + ) -> ArtifactSections: member = "" section_name: str | None = None section_bytes: dict[tuple[str, str], bytearray] = {} @@ -148,7 +162,24 @@ def from_objdump(cls, output: str, arch: str) -> ArtifactSections: key: SectionData(key[0], key[1], bytes(data), tuple(section_relocations.get(key, ()))) for key, data in section_bytes.items() } - return cls(sections, arch) + symbols: dict[tuple[str, str], tuple[str, int, int]] = {} + member = "" + for line in symbols_output.splitlines(): + file_format = cls.FILE_FORMAT.match(line) + if file_format: + member = file_format.group("member") + continue + symbol = cls.SYMBOL_ENTRY.match(line) + if symbol and symbol.group(2) not in {"*UND*", "*ABS*"}: + name = symbol.group(4).strip() + if name.startswith(".hidden "): + name = name.removeprefix(".hidden ") + symbols[(member, name)] = ( + symbol.group(2), + int(symbol.group(1), 16), + int(symbol.group(3), 16), + ) + return cls(sections, arch, symbols) def _find(self, member: str, name: str) -> SectionData | None: exact = self.sections.get((member, name)) @@ -160,7 +191,21 @@ def _find(self, member: str, name: str) -> SectionData | None: if candidate_member == member and (candidate_name == name or candidate_name.endswith(f".{name}")) ] - return candidates[0] if len(candidates) == 1 else None + if len(candidates) == 1: + return candidates[0] + symbol = self.symbols.get((member, name)) + if symbol is None: + return None + symbol_section, offset, size = symbol + section = self.sections.get((member, symbol_section)) + if section is None or offset < 0 or offset + size > len(section.data): + return None + relocations = tuple( + SectionRelocation(relocation.offset - offset, relocation.kind, relocation.target) + for relocation in section.relocations + if offset <= relocation.offset < offset + size + ) + return SectionData(member, symbol_section, section.data[offset : offset + size], relocations) @staticmethod def _split_target(value: str) -> tuple[str, str]: @@ -184,7 +229,9 @@ def _string_target(self, member: str, name: str, addend: str) -> str | None: value = section.data[offset:] return value.split(b"\0", 1)[0].hex() - def _fingerprint(self, section: SectionData, member: str) -> str | None: + def _fingerprint_details( + self, section: SectionData, member: str, anonymous_depth: int = 0 + ) -> tuple[str, tuple[str, ...]] | None: data = bytearray(section.data) descriptors: list[str] = [] section_identity = _anonymous_instance(section.name) @@ -200,6 +247,8 @@ def _fingerprint(self, section: SectionData, member: str) -> str | None: "R_AARCH64_ADD_ABS_LO12_NC": 4, "R_AARCH64_LDST64_ABS_LO12_NC": 4, } + anonymous_aliases: dict[str, int] = {} + nested_addends: list[str] = [] for relocation in section.relocations: width = widths.get(relocation.kind) if width is None or relocation.offset < 0 or relocation.offset + width > len(data): @@ -210,13 +259,39 @@ def _fingerprint(self, section: SectionData, member: str) -> str | None: descriptor_target = target else: target_name, addend = self._split_target(target) - if _anonymous_instance(target_name) is not None: - return None - string = self._string_target(member, target_name, addend) - descriptor_target = f"string:{string}" if string is not None else f"symbol:{target_name}{addend}" + anonymous = _anonymous_instance(target_name) + if anonymous is not None: + # Deliberately support exactly one nested anonymous level. + # A nested anonymous target at this level would require a + # recursive object graph, so remain fail-closed. + if anonymous_depth >= 1: + return None + nested = self._find(member, target_name) + if nested is None: + return None + nested_details = self._fingerprint_details(nested, member, anonymous_depth + 1) + if nested_details is None: + return None + nested_fingerprint, nested_child_addends = nested_details + # Keep the concrete target symbol here so distinct + # anonymous objects with the same stem remain distinct. + # Their absolute ordinals are normalized by encounter + # order, which tolerates compiler renumbering. + alias_key = f"{member}:{target_name}" + alias = anonymous_aliases.setdefault(alias_key, len(anonymous_aliases)) + descriptor_target = f"anonymous:{nested_fingerprint}:alias{alias}:{addend}" + nested_addends.append(addend) + nested_addends.extend(nested_child_addends) + else: + string = self._string_target(member, target_name, addend) + descriptor_target = f"string:{string}" if string is not None else f"symbol:{target_name}{addend}" descriptors.append(f"{relocation.offset}:{relocation.kind}:{descriptor_target}") payload = section_class.encode() + b"\0" + bytes(data) + b"\0" + "\n".join(descriptors).encode() - return hashlib.sha256(payload).hexdigest() + return hashlib.sha256(payload).hexdigest(), tuple(nested_addends) + + def _fingerprint(self, section: SectionData, member: str, anonymous_depth: int = 0) -> str | None: + details = self._fingerprint_details(section, member, anonymous_depth) + return details[0] if details is not None else None def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: target = value.strip() @@ -232,10 +307,11 @@ def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: section = self._find(member, name) if section is None: return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False, addend) - fingerprint = self._fingerprint(section, member) - if fingerprint is None: + details = self._fingerprint_details(section, member) + if details is None: return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False, addend) - return ResolvedTarget(f"anonymous:{fingerprint}", True, True, addend) + fingerprint, nested_addends = details + return ResolvedTarget(f"anonymous:{fingerprint}", True, True, addend, nested_addends) @dataclass(frozen=True) @@ -281,6 +357,7 @@ def _same_relocations(old: tuple[Relocation, ...], new: tuple[Relocation, ...]) and before.anonymous == after.anonymous and before.resolved == after.resolved and before.addend == after.addend + and before.nested_addends == after.nested_addends and (not before.anonymous or before.resolved) for before, after in zip(old, new) ) and len(old) == len(new) @@ -297,6 +374,7 @@ def _metadata_only_relocations(old: Function, new: Function) -> bool: or before.kind != after.kind or before.anonymous != after.anonymous or before.addend != after.addend + or before.nested_addends != after.nested_addends ): return False if before.token != after.token: @@ -381,15 +459,27 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec anonymous = _anonymous_instance(target) is not None resolved = True addend = _anonymous_instance(target)[2] if anonymous else "" + nested_addends = () else: resolved_target = sections.resolve(instruction.member, kind, target) token = resolved_target.token anonymous = resolved_target.anonymous resolved = resolved_target.resolved addend = resolved_target.addend + nested_addends = resolved_target.nested_addends relocation_text.append(f"[{kind}:{token}]") relocations.append( - Relocation(relocation_offset, kind, target, token, anonymous, resolved, instruction_index, addend) + Relocation( + relocation_offset, + kind, + target, + token, + anonymous, + resolved, + instruction_index, + addend, + nested_addends, + ) ) if anonymous: alias_target = X86_PC_BIAS.sub("", target) if kind in PC_RELATIVE_X86_RELOCATIONS else target @@ -459,7 +549,8 @@ def parse_objdump(output: str, arch: str, sections: ArtifactSections | None = No def load_functions(artifact: Path, objdump: str, arch: str) -> dict[str, Function]: output = subprocess.check_output([objdump, "-drwC", str(artifact)], text=True, errors="replace") section_output = subprocess.check_output([objdump, "-sr", str(artifact)], text=True, errors="replace") - sections = ArtifactSections.from_objdump(section_output, arch) + symbol_output = subprocess.check_output([objdump, "-t", str(artifact)], text=True, errors="replace") + sections = ArtifactSections.from_objdump(section_output, arch, symbol_output) return parse_objdump(output, arch, sections) diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 5ef396e..7b95e18 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -62,6 +62,99 @@ def test_anonymous_section_content_ignores_ordinal(self) -> None: ) self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") + @staticmethod + def nested_sections( + outer_ordinal: int, + nested_ordinal: int, + nested_value: int = 1, + nested_addend: str = "+0x0", + ) -> str: + nested_name = f".data.rel.ro..Lanon.nested.{nested_ordinal}" + outer_name = f".data.rel.ro..Lanon.outer.{outer_ordinal}" + result = ( + "fixture.o: file format elf64-x86-64\n" + f"Contents of section {nested_name}:\n" + f" 0000 {nested_value & 0xffffffff:08x} 00000000 ........\n" + f"Contents of section {outer_name}:\n" + " 0000 00000000 00000000 ........\n" + f"RELOCATION RECORDS FOR [{outer_name}]:\n" + "OFFSET TYPE VALUE\n" + f"0000000000000000 R_X86_64_64 {nested_name}{nested_addend}\n" + ) + return result + + def nested_root(self, outer_ordinal: int, addend: str = "+0x0") -> str: + return ( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + f" 3: R_X86_64_PC32 .data.rel.ro..Lanon.outer.{outer_ordinal}{addend}\n" + ) + + def test_one_level_nested_ordinal_change_is_identical(self) -> None: + old = parse_with_sections(self.nested_root(1), self.nested_sections(1, 2)) + new = parse_with_sections(self.nested_root(9), self.nested_sections(9, 8)) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") + + def test_nested_semantic_content_change_is_metadata_change(self) -> None: + old = parse_with_sections(self.nested_root(1), self.nested_sections(1, 2, nested_value=1)) + new = parse_with_sections(self.nested_root(9), self.nested_sections(9, 8, nested_value=2)) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN METADATA CHANGED") + + def test_nested_addend_change_is_not_metadata(self) -> None: + old = parse_with_sections(self.nested_root(1), self.nested_sections(1, 2, nested_addend="+0x0")) + new = parse_with_sections(self.nested_root(9), self.nested_sections(9, 8, nested_addend="+0x8")) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_nested_alias_partition_is_preserved(self) -> None: + def sections(outer_ordinal: int, first: int, second: int) -> str: + outer = f".data.rel.ro..Lanon.outer.{outer_ordinal}" + first_name = f".data.rel.ro..Lanon.nested.{first}" + second_name = f".data.rel.ro..Lanon.nested.{second}" + return ( + "fixture.o: file format elf64-x86-64\n" + f"Contents of section {first_name}:\n" + " 0000 0100000000000000 ........\n" + + ( + f"Contents of section {second_name}:\n" + " 0000 0100000000000000 ........\n" + if second != first + else "" + ) + + f"Contents of section {outer}:\n" + " 0000 0000000000000000 0000000000000000 ................\n" + f"RELOCATION RECORDS FOR [{outer}]:\n" + "OFFSET TYPE VALUE\n" + f"0000000000000000 R_X86_64_64 {first_name}+0x0\n" + f"0000000000000008 R_X86_64_64 {second_name}+0x0\n" + ) + + old = parse_with_sections(self.nested_root(1), sections(1, 2, 2)) + new = parse_with_sections(self.nested_root(9), sections(9, 8, 9)) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + + def test_second_level_nested_anonymous_is_conservative(self) -> None: + inner = ".data.rel.ro..Lanon.inner.3" + nested = ".data.rel.ro..Lanon.nested.2" + outer = ".data.rel.ro..Lanon.outer.1" + sections = ( + "fixture.o: file format elf64-x86-64\n" + f"Contents of section {inner}:\n" + " 0000 0000000000000000 ........\n" + f"Contents of section {nested}:\n" + " 0000 0000000000000000 ........\n" + f"RELOCATION RECORDS FOR [{nested}]:\n" + "OFFSET TYPE VALUE\n" + f"0000000000000000 R_X86_64_64 {inner}+0x0\n" + f"Contents of section {outer}:\n" + " 0000 0000000000000000 ........\n" + f"RELOCATION RECORDS FOR [{outer}]:\n" + "OFFSET TYPE VALUE\n" + f"0000000000000000 R_X86_64_64 {nested}+0x0\n" + ) + old = parse_with_sections(self.nested_root(1), sections) + self.assertEqual(compare_codegen.classify(old, old), "CODEGEN CHANGED") + def test_relocation_site_change_is_not_ignored(self) -> None: old = parse( "0000 :\n" From 832b296e0685661bf0f72dd40f2a518c8909f5f8 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 13:04:43 +0000 Subject: [PATCH 34/79] fix: preserve nested anonymous alias topology --- scripts/compare_codegen.py | 27 ++++++++++++++++++----- scripts/tests/test_codegen.py | 40 +++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index b274b9e..2f2b102 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -52,6 +52,7 @@ class Relocation: instruction_index: int addend: str = "" nested_addends: tuple[str, ...] = () + nested_alias_partition: tuple[int, ...] = () @dataclass(frozen=True) @@ -76,6 +77,7 @@ class ResolvedTarget: resolved: bool addend: str = "" nested_addends: tuple[str, ...] = () + nested_alias_partition: tuple[int, ...] = () class ArtifactSections: @@ -231,7 +233,7 @@ def _string_target(self, member: str, name: str, addend: str) -> str | None: def _fingerprint_details( self, section: SectionData, member: str, anonymous_depth: int = 0 - ) -> tuple[str, tuple[str, ...]] | None: + ) -> tuple[str, tuple[str, ...], tuple[int, ...]] | None: data = bytearray(section.data) descriptors: list[str] = [] section_identity = _anonymous_instance(section.name) @@ -249,6 +251,7 @@ def _fingerprint_details( } anonymous_aliases: dict[str, int] = {} nested_addends: list[str] = [] + nested_alias_partition: list[int] = [] for relocation in section.relocations: width = widths.get(relocation.kind) if width is None or relocation.offset < 0 or relocation.offset + width > len(data): @@ -272,7 +275,7 @@ def _fingerprint_details( nested_details = self._fingerprint_details(nested, member, anonymous_depth + 1) if nested_details is None: return None - nested_fingerprint, nested_child_addends = nested_details + nested_fingerprint, nested_child_addends, nested_child_aliases = nested_details # Keep the concrete target symbol here so distinct # anonymous objects with the same stem remain distinct. # Their absolute ordinals are normalized by encounter @@ -282,12 +285,14 @@ def _fingerprint_details( descriptor_target = f"anonymous:{nested_fingerprint}:alias{alias}:{addend}" nested_addends.append(addend) nested_addends.extend(nested_child_addends) + nested_alias_partition.append(alias) + nested_alias_partition.extend(nested_child_aliases) else: string = self._string_target(member, target_name, addend) descriptor_target = f"string:{string}" if string is not None else f"symbol:{target_name}{addend}" descriptors.append(f"{relocation.offset}:{relocation.kind}:{descriptor_target}") payload = section_class.encode() + b"\0" + bytes(data) + b"\0" + "\n".join(descriptors).encode() - return hashlib.sha256(payload).hexdigest(), tuple(nested_addends) + return hashlib.sha256(payload).hexdigest(), tuple(nested_addends), tuple(nested_alias_partition) def _fingerprint(self, section: SectionData, member: str, anonymous_depth: int = 0) -> str | None: details = self._fingerprint_details(section, member, anonymous_depth) @@ -310,8 +315,15 @@ def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: details = self._fingerprint_details(section, member) if details is None: return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False, addend) - fingerprint, nested_addends = details - return ResolvedTarget(f"anonymous:{fingerprint}", True, True, addend, nested_addends) + fingerprint, nested_addends, nested_alias_partition = details + return ResolvedTarget( + f"anonymous:{fingerprint}", + True, + True, + addend, + nested_addends, + nested_alias_partition, + ) @dataclass(frozen=True) @@ -358,6 +370,7 @@ def _same_relocations(old: tuple[Relocation, ...], new: tuple[Relocation, ...]) and before.resolved == after.resolved and before.addend == after.addend and before.nested_addends == after.nested_addends + and before.nested_alias_partition == after.nested_alias_partition and (not before.anonymous or before.resolved) for before, after in zip(old, new) ) and len(old) == len(new) @@ -375,6 +388,7 @@ def _metadata_only_relocations(old: Function, new: Function) -> bool: or before.anonymous != after.anonymous or before.addend != after.addend or before.nested_addends != after.nested_addends + or before.nested_alias_partition != after.nested_alias_partition ): return False if before.token != after.token: @@ -460,6 +474,7 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec resolved = True addend = _anonymous_instance(target)[2] if anonymous else "" nested_addends = () + nested_alias_partition = () else: resolved_target = sections.resolve(instruction.member, kind, target) token = resolved_target.token @@ -467,6 +482,7 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec resolved = resolved_target.resolved addend = resolved_target.addend nested_addends = resolved_target.nested_addends + nested_alias_partition = resolved_target.nested_alias_partition relocation_text.append(f"[{kind}:{token}]") relocations.append( Relocation( @@ -479,6 +495,7 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec instruction_index, addend, nested_addends, + nested_alias_partition, ) ) if anonymous: diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 7b95e18..ddebb38 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -114,15 +114,15 @@ def sections(outer_ordinal: int, first: int, second: int) -> str: return ( "fixture.o: file format elf64-x86-64\n" f"Contents of section {first_name}:\n" - " 0000 0100000000000000 ........\n" + " 0000 01000000 00000000 ........\n" + ( f"Contents of section {second_name}:\n" - " 0000 0100000000000000 ........\n" + " 0000 01000000 00000000 ........\n" if second != first else "" ) + f"Contents of section {outer}:\n" - " 0000 0000000000000000 0000000000000000 ................\n" + " 0000 00000000 00000000 00000000 00000000 ................\n" f"RELOCATION RECORDS FOR [{outer}]:\n" "OFFSET TYPE VALUE\n" f"0000000000000000 R_X86_64_64 {first_name}+0x0\n" @@ -131,8 +131,17 @@ def sections(outer_ordinal: int, first: int, second: int) -> str: old = parse_with_sections(self.nested_root(1), sections(1, 2, 2)) new = parse_with_sections(self.nested_root(9), sections(9, 8, 9)) + self.assertTrue(old.relocations[0].resolved) + self.assertTrue(new.relocations[0].resolved) + self.assertEqual(old.relocations[0].nested_alias_partition, (0, 0)) + self.assertEqual(new.relocations[0].nested_alias_partition, (0, 1)) self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") + same_alias = parse_with_sections(self.nested_root(9), sections(9, 8, 8)) + self.assertTrue(same_alias.relocations[0].resolved) + self.assertEqual(same_alias.relocations[0].nested_alias_partition, (0, 0)) + self.assertEqual(compare_codegen.classify(old, same_alias), "CODEGEN IDENTICAL") + def test_second_level_nested_anonymous_is_conservative(self) -> None: inner = ".data.rel.ro..Lanon.inner.3" nested = ".data.rel.ro..Lanon.nested.2" @@ -140,21 +149,40 @@ def test_second_level_nested_anonymous_is_conservative(self) -> None: sections = ( "fixture.o: file format elf64-x86-64\n" f"Contents of section {inner}:\n" - " 0000 0000000000000000 ........\n" + " 0000 00000000 00000000 ........\n" f"Contents of section {nested}:\n" - " 0000 0000000000000000 ........\n" + " 0000 00000000 00000000 ........\n" f"RELOCATION RECORDS FOR [{nested}]:\n" "OFFSET TYPE VALUE\n" f"0000000000000000 R_X86_64_64 {inner}+0x0\n" f"Contents of section {outer}:\n" - " 0000 0000000000000000 ........\n" + " 0000 00000000 00000000 ........\n" f"RELOCATION RECORDS FOR [{outer}]:\n" "OFFSET TYPE VALUE\n" f"0000000000000000 R_X86_64_64 {nested}+0x0\n" ) old = parse_with_sections(self.nested_root(1), sections) + self.assertFalse(old.relocations[0].resolved) self.assertEqual(compare_codegen.classify(old, old), "CODEGEN CHANGED") + def test_nested_target_does_not_cross_archive_members(self) -> None: + outer = ".data.rel.ro..Lanon.outer.1" + nested = ".data.rel.ro..Lanon.nested.2" + sections = ( + "member-a.o: file format elf64-x86-64\n" + f"Contents of section {outer}:\n" + " 0000 00000000 00000000 ........\n" + f"RELOCATION RECORDS FOR [{outer}]:\n" + "OFFSET TYPE VALUE\n" + f"0000000000000000 R_X86_64_64 {nested}+0x0\n" + "member-b.o: file format elf64-x86-64\n" + f"Contents of section {nested}:\n" + " 0000 01000000 00000000 ........\n" + ) + index = compare_codegen.ArtifactSections.from_objdump(sections, "x86_64") + resolved = index.resolve("member-a.o", "R_X86_64_PC32", outer) + self.assertFalse(resolved.resolved) + def test_relocation_site_change_is_not_ignored(self) -> None: old = parse( "0000 :\n" From 7a4cf2c0251a6c6d8b491a3b301b08ffccc46bf8 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 13:16:22 +0000 Subject: [PATCH 35/79] fix: make metadata changes non-blocking by default --- scripts/compare_codegen.py | 6 ++- scripts/run_codegen_comparison.py | 21 +++++++-- scripts/tests/test_codegen.py | 72 +++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 4 deletions(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index 2f2b102..b3ad6be 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -612,6 +612,7 @@ def main() -> int: parser.add_argument("--target", required=True) parser.add_argument("--arch", choices=("x86_64", "aarch64"), required=True) parser.add_argument("--objdump", default="objdump") + parser.add_argument("--strict-metadata", action="store_true") args = parser.parse_args() before = load_functions(args.baseline, args.objdump, args.arch) after = load_functions(args.current, args.objdump, args.arch) @@ -647,6 +648,8 @@ def main() -> int: else: changed += 1 print(f"{root.symbol}: {status}") + if status == "CODEGEN METADATA CHANGED": + print(f"WARNING: {root.symbol} has metadata-only codegen changes", file=sys.stderr) print(f" text bytes {len(old.encoded)} -> {len(new.encoded)}") print(f" instructions {len(old.instructions)} -> {len(new.instructions)}") print(f" calls {old.calls} -> {new.calls}") @@ -673,7 +676,8 @@ def main() -> int: print(f"changed : {changed}") print(f"errors : {errors}") print(f"skipped : {skipped}") - return 0 if not (metadata_changed or encoding_changed or changed or errors) else 1 + failed = encoding_changed or changed or errors or (args.strict_metadata and metadata_changed) + return 0 if not failed else 1 if __name__ == "__main__": diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py index 1dadaa0..d2d7e7d 100755 --- a/scripts/run_codegen_comparison.py +++ b/scripts/run_codegen_comparison.py @@ -60,7 +60,15 @@ def build_harness(root: Path, library: Path, output: Path, rustflags: str, revis run(command, root, os.environ.copy()) -def compare(root: Path, baseline: Path, current: Path, artifact: str, target: str, arch: str) -> int: +def compare( + root: Path, + baseline: Path, + current: Path, + artifact: str, + target: str, + arch: str, + strict_metadata: bool, +) -> int: command = [ sys.executable, str(root / "scripts" / "compare_codegen.py"), @@ -77,6 +85,8 @@ def compare(root: Path, baseline: Path, current: Path, artifact: str, target: st "--arch", arch, ] + if strict_metadata: + command.append("--strict-metadata") return subprocess.run(command, cwd=root, check=False).returncode @@ -84,6 +94,7 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--baseline", default=BASELINE_SHA) parser.add_argument("--config", choices=sorted(CODEGEN_CONFIGS), action="append") + parser.add_argument("--strict-metadata", action="store_true") args = parser.parse_args() root = Path(__file__).resolve().parents[1] current_revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() @@ -118,8 +129,12 @@ def cleanup_worktree() -> None: current_harness = config_dir / "current-harness.o" build_harness(root, baseline_library, baseline_harness, rustflags, current_revision) build_harness(root, current_library, current_harness, rustflags, current_revision) - failed |= compare(root, baseline_harness, current_harness, "harness", config, arch) != 0 - failed |= compare(root, baseline_library, current_library, "library", config, arch) != 0 + failed |= compare( + root, baseline_harness, current_harness, "harness", config, arch, args.strict_metadata + ) != 0 + failed |= compare( + root, baseline_library, current_library, "library", config, arch, args.strict_metadata + ) != 0 return int(failed) diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index ddebb38..38eaa75 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -12,6 +12,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import compare_codegen +import run_codegen_comparison def parse(body: str, arch: str = "x86_64") -> compare_codegen.Function: @@ -587,6 +588,77 @@ def test_expected_missing_symbol_fails(self) -> None: ): self.assertNotEqual(compare_codegen.main(), 0) + def test_metadata_policy_exit_codes(self) -> None: + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", + self.SECTION_PREFIX + self.anonymous_section(1, line=1), + ) + metadata = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.9-0x4\n", + self.SECTION_PREFIX + self.anonymous_section(9, line=2), + ) + changed = parse("0000 :\n 0: 90 nop\n 1: c3 ret") + old_encoding = parse("0000 :\n 0: 90 nop\n 1: c3 ret") + new_encoding = parse("0000 :\n 0: 66 90 nop\n 2: c3 ret") + + def run_main(before: compare_codegen.Function, after: compare_codegen.Function, strict: bool) -> int: + with tempfile.TemporaryDirectory() as directory: + roots = Path(directory) / "roots.toml" + roots.write_text('[[root]]\nsymbol="root"\nartifact="harness"\ntargets=["x86_64-avx2"]\n') + argv = [ + "compare_codegen.py", + "baseline.o", + "current.o", + "--roots", + str(roots), + "--artifact", + "harness", + "--target", + "x86_64-avx2", + "--arch", + "x86_64", + ] + if strict: + argv.append("--strict-metadata") + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object( + compare_codegen, + "load_functions", + side_effect=[{"root": before}, {"root": after}], + ), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + return compare_codegen.main() + + self.assertEqual(run_main(old, metadata, False), 0) + self.assertNotEqual(run_main(old, metadata, True), 0) + self.assertNotEqual(run_main(changed, old, False), 0) + self.assertNotEqual(run_main(changed, old, True), 0) + self.assertNotEqual(run_main(old_encoding, new_encoding, False), 0) + self.assertNotEqual(run_main(old_encoding, new_encoding, True), 0) + + def test_runner_forwards_strict_metadata(self) -> None: + with mock.patch.object(run_codegen_comparison.subprocess, "run", return_value=mock.Mock(returncode=0)) as run: + result = run_codegen_comparison.compare( + Path("."), + Path("baseline.o"), + Path("current.o"), + "harness", + "x86_64-avx2", + "x86_64", + True, + ) + self.assertEqual(result, 0) + self.assertIn("--strict-metadata", run.call_args.args[0]) + def test_inventory_has_root_for_each_kernel(self) -> None: scripts = Path(__file__).resolve().parents[1] roots = {root.symbol for root in compare_codegen.load_roots(scripts / "codegen_roots.toml")} From f8365de46c0528f0beaa38f6cb3e2b590921d692 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 13:37:20 +0000 Subject: [PATCH 36/79] refactor: split find_core backends --- src/traits/words_find/funcs_for_find_core.rs | 232 +----------------- .../words_find/funcs_for_find_core/avx2.rs | 75 ++++++ .../words_find/funcs_for_find_core/neon.rs | 69 ++++++ .../words_find/funcs_for_find_core/sse41.rs | 77 ++++++ 4 files changed, 224 insertions(+), 229 deletions(-) create mode 100644 src/traits/words_find/funcs_for_find_core/avx2.rs create mode 100644 src/traits/words_find/funcs_for_find_core/neon.rs create mode 100644 src/traits/words_find/funcs_for_find_core/sse41.rs diff --git a/src/traits/words_find/funcs_for_find_core.rs b/src/traits/words_find/funcs_for_find_core.rs index f2d3674..7062ce6 100644 --- a/src/traits/words_find/funcs_for_find_core.rs +++ b/src/traits/words_find/funcs_for_find_core.rs @@ -96,243 +96,17 @@ where None } -// --------------------------------------------------------------------------- -// SSE2 — same loop as scalar but checks 2 consecutive shifts at once -// --------------------------------------------------------------------------- - #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, - _mm_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, - _mm_set1_epi64x, - }; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn find( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = _mm_set1_epi64x(needle_first as i64); - let mask = _mm_set1_epi64x(needle_mask as i64); - - for i in 0..haystack.len() { - let base = i * WORD_BITS; - if base > last_start { - break; - } - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - - let mut s = 0; - while s < WORD_BITS { - if base + s > last_start { - break; - } - // Build two consecutive windows manually, pack into __m128i. - let win0 = if s == 0 { - w0 - } else { - (w0 >> s) | (w1 << (WORD_BITS - s)) - }; - let win1 = if s + 1 >= WORD_BITS { - 0 - } else { - (w0 >> (s + 1)) | (w1 << (WORD_BITS - (s + 1))) - }; - let windows = unsafe { _mm_loadu_si128([win0, win1].as_ptr().cast::<__m128i>()) }; - let m = _mm_and_si128(windows, mask); - let c = unsafe { _mm_cmpeq_epi64(m, needle) }; - let hits = _mm_movemask_epi8(c) as u32; - - if hits & 0xff != 0 { - let pos = base + s; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if hits & 0xff00 != 0 { - let pos = base + s + 1; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - - s += 2; - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// AVX2 — checks 4 consecutive shifts at once -// --------------------------------------------------------------------------- +mod sse41; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, - _mm256_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, - _mm256_set1_epi64x, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn find( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = _mm256_set1_epi64x(needle_first as i64); - let mask = _mm256_set1_epi64x(needle_mask as i64); - - for i in 0..haystack.len() { - let base = i * WORD_BITS; - if base > last_start { - break; - } - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - - let mut s = 0; - while s < WORD_BITS { - if base + s > last_start { - break; - } - let end = WORD_BITS.min(s + 4); - let mut wins = [0u64; 4]; - for k in 0..(end - s) { - let shift = s + k; - wins[k] = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - } - let windows = unsafe { _mm256_loadu_si256(wins.as_ptr().cast::<__m256i>()) }; - let m = _mm256_and_si256(windows, mask); - let c = _mm256_cmpeq_epi64(m, needle); - let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(c)) } as u32; - if hits != 0 { - for k in 0..(end - s) { - if hits & (1 << k) != 0 { - let pos = base + s + k; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - - s += 4; - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// NEON -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::*; - - use core::arch::aarch64::{ - uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, - }; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn find( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = vdupq_n_u64(needle_first); - let mask = vdupq_n_u64(needle_mask); - - for i in 0..haystack.len() { - let base = i * WORD_BITS; - if base > last_start { - break; - } - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - - let mut s = 0; - while s < WORD_BITS { - if base + s > last_start { - break; - } - let end = WORD_BITS.min(s + 2); - let mut wins = [0u64; 2]; - for k in 0..(end - s) { - let shift = s + k; - wins[k] = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - } - let windows = unsafe { vld1q_u64(wins.as_ptr()) }; - let m = vandq_u64(windows, mask); - let c = vceqq_u64(m, needle); - if vgetq_lane_u64(c, 0) != 0 { - let pos = base + s; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if vgetq_lane_u64(c, 1) != 0 && s + 1 < end { - let pos = base + s + 1; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - - s += 2; - } - } - - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_find/funcs_for_find_core/avx2.rs b/src/traits/words_find/funcs_for_find_core/avx2.rs new file mode 100644 index 0000000..f929da4 --- /dev/null +++ b/src/traits/words_find/funcs_for_find_core/avx2.rs @@ -0,0 +1,75 @@ +// --------------------------------------------------------------------------- +// AVX2 — checks 4 consecutive shifts at once +// --------------------------------------------------------------------------- + +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_set1_epi64x, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_set1_epi64x, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn find( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = _mm256_set1_epi64x(needle_first as i64); + let mask = _mm256_set1_epi64x(needle_mask as i64); + + for i in 0..haystack.len() { + let base = i * WORD_BITS; + if base > last_start { + break; + } + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + + let mut s = 0; + while s < WORD_BITS { + if base + s > last_start { + break; + } + let end = WORD_BITS.min(s + 4); + let mut wins = [0u64; 4]; + for k in 0..(end - s) { + let shift = s + k; + wins[k] = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + } + let windows = unsafe { _mm256_loadu_si256(wins.as_ptr().cast::<__m256i>()) }; + let m = _mm256_and_si256(windows, mask); + let c = _mm256_cmpeq_epi64(m, needle); + let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(c)) } as u32; + if hits != 0 { + for k in 0..(end - s) { + if hits & (1 << k) != 0 { + let pos = base + s + k; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + + s += 4; + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_find_core/neon.rs b/src/traits/words_find/funcs_for_find_core/neon.rs new file mode 100644 index 0000000..5ebbef8 --- /dev/null +++ b/src/traits/words_find/funcs_for_find_core/neon.rs @@ -0,0 +1,69 @@ +// --------------------------------------------------------------------------- +// NEON +// --------------------------------------------------------------------------- + +use super::*; + +use core::arch::aarch64::{ + uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, +}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn find( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = vdupq_n_u64(needle_first); + let mask = vdupq_n_u64(needle_mask); + + for i in 0..haystack.len() { + let base = i * WORD_BITS; + if base > last_start { + break; + } + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + + let mut s = 0; + while s < WORD_BITS { + if base + s > last_start { + break; + } + let end = WORD_BITS.min(s + 2); + let mut wins = [0u64; 2]; + for k in 0..(end - s) { + let shift = s + k; + wins[k] = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + } + let windows = unsafe { vld1q_u64(wins.as_ptr()) }; + let m = vandq_u64(windows, mask); + let c = vceqq_u64(m, needle); + if vgetq_lane_u64(c, 0) != 0 { + let pos = base + s; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if vgetq_lane_u64(c, 1) != 0 && s + 1 < end { + let pos = base + s + 1; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + + s += 2; + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_find_core/sse41.rs b/src/traits/words_find/funcs_for_find_core/sse41.rs new file mode 100644 index 0000000..cf9db33 --- /dev/null +++ b/src/traits/words_find/funcs_for_find_core/sse41.rs @@ -0,0 +1,77 @@ +// --------------------------------------------------------------------------- +// SSE2 — same loop as scalar but checks 2 consecutive shifts at once +// --------------------------------------------------------------------------- + +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, +}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn find( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = _mm_set1_epi64x(needle_first as i64); + let mask = _mm_set1_epi64x(needle_mask as i64); + + for i in 0..haystack.len() { + let base = i * WORD_BITS; + if base > last_start { + break; + } + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + + let mut s = 0; + while s < WORD_BITS { + if base + s > last_start { + break; + } + // Build two consecutive windows manually, pack into __m128i. + let win0 = if s == 0 { + w0 + } else { + (w0 >> s) | (w1 << (WORD_BITS - s)) + }; + let win1 = if s + 1 >= WORD_BITS { + 0 + } else { + (w0 >> (s + 1)) | (w1 << (WORD_BITS - (s + 1))) + }; + let windows = unsafe { _mm_loadu_si128([win0, win1].as_ptr().cast::<__m128i>()) }; + let m = _mm_and_si128(windows, mask); + let c = unsafe { _mm_cmpeq_epi64(m, needle) }; + let hits = _mm_movemask_epi8(c) as u32; + + if hits & 0xff != 0 { + let pos = base + s; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if hits & 0xff00 != 0 { + let pos = base + s + 1; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + + s += 2; + } + } + + None +} From e01e3c8d1f792eeaf06d732bed5c0f3fa9bc10a0 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 13:49:50 +0000 Subject: [PATCH 37/79] refactor: split rfind_core backends --- src/traits/words_find/funcs_for_rfind_core.rs | 234 +----------------- .../words_find/funcs_for_rfind_core/avx2.rs | 77 ++++++ .../words_find/funcs_for_rfind_core/neon.rs | 70 ++++++ .../words_find/funcs_for_rfind_core/sse41.rs | 76 ++++++ 4 files changed, 226 insertions(+), 231 deletions(-) create mode 100644 src/traits/words_find/funcs_for_rfind_core/avx2.rs create mode 100644 src/traits/words_find/funcs_for_rfind_core/neon.rs create mode 100644 src/traits/words_find/funcs_for_rfind_core/sse41.rs diff --git a/src/traits/words_find/funcs_for_rfind_core.rs b/src/traits/words_find/funcs_for_rfind_core.rs index 68ecbe1..75a1ae0 100644 --- a/src/traits/words_find/funcs_for_rfind_core.rs +++ b/src/traits/words_find/funcs_for_rfind_core.rs @@ -102,245 +102,17 @@ where None } -// --------------------------------------------------------------------------- -// SSE2 — 2 consecutive shifts at once, reverse -// --------------------------------------------------------------------------- - #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, - _mm_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, - _mm_set1_epi64x, - }; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn rfind( - haystack: &[u64], - needle_key: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = _mm_set1_epi64x(needle_key as i64); - let mask = _mm_set1_epi64x(needle_mask as i64); - - let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); - for i in (0..=start_word).rev() { - let base = i * WORD_BITS; - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - let max_shift = WORD_BITS.min(last_start - base + 1); - - // Round up to a multiple of 2 so the SIMD loop - // processes shifts in 2-lane pairs. Out-of-range - // positions are guarded by `pos <= last_start`. - let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); - while s > 0 { - s -= 2; - - let win0 = if s == 0 { - w0 - } else { - (w0 >> s) | (w1 << (WORD_BITS - s)) - }; - let win1 = if s + 1 >= WORD_BITS { - 0 - } else { - (w0 >> (s + 1)) | (w1 << (WORD_BITS - (s + 1))) - }; - let windows = unsafe { _mm_loadu_si128([win0, win1].as_ptr().cast::<__m128i>()) }; - let m = _mm_and_si128(windows, mask); - let c = unsafe { _mm_cmpeq_epi64(m, needle) }; - let hits = _mm_movemask_epi8(c) as u32; - - // Check higher shift (lane 1) first for rightmost. - if hits & 0xff00 != 0 { - let pos = base + s + 1; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if hits & 0xff != 0 { - let pos = base + s; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// AVX2 — 4 consecutive shifts at once, reverse -// --------------------------------------------------------------------------- +mod sse41; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, - _mm256_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, - _mm256_set1_epi64x, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn rfind( - haystack: &[u64], - needle_key: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = _mm256_set1_epi64x(needle_key as i64); - let mask = _mm256_set1_epi64x(needle_mask as i64); - - let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); - for i in (0..=start_word).rev() { - let base = i * WORD_BITS; - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - let max_shift = WORD_BITS.min(last_start - base + 1); - - // Round up to a multiple of 4 so the SIMD loop - // processes shifts in 4-lane groups. Out-of-range - // positions are guarded by `pos <= last_start`. - let mut s = max_shift.next_multiple_of(4).min(WORD_BITS); - while s > 0 { - s -= 4; - - let end = WORD_BITS.min(s + 4); - let mut wins = [0u64; 4]; - for k in 0..(end - s) { - let shift = s + k; - wins[k] = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - } - - let windows = unsafe { _mm256_loadu_si256(wins.as_ptr().cast::<__m256i>()) }; - let m = _mm256_and_si256(windows, mask); - let c = unsafe { _mm256_cmpeq_epi64(m, needle) }; - let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(c)) } as u32; - - // Check higher shifts first for rightmost. - if hits != 0 { - for k in (0..(end - s)).rev() { - if hits & (1 << k) != 0 { - let pos = base + s + k; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// NEON — 2 consecutive shifts at once, reverse -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::*; - - use core::arch::aarch64::{ - uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, - }; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn rfind( - haystack: &[u64], - needle_key: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = vdupq_n_u64(needle_key); - let mask = vdupq_n_u64(needle_mask); - - let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); - for i in (0..=start_word).rev() { - let base = i * WORD_BITS; - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - let max_shift = WORD_BITS.min(last_start - base + 1); - - // Round up to a multiple of 2 so the SIMD loop - // processes shifts in 2-lane pairs. Out-of-range - // positions are guarded by `pos <= last_start`. - let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); - while s > 0 { - s -= 2; - - let end = WORD_BITS.min(s + 2); - let mut wins = [0u64; 2]; - for k in 0..(end - s) { - let shift = s + k; - wins[k] = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - } - let windows = unsafe { vld1q_u64(wins.as_ptr()) }; - let m = vandq_u64(windows, mask); - let c = vceqq_u64(m, needle); - - // Check higher shift (lane 1) first for rightmost. - if vgetq_lane_u64(c, 1) != 0 { - let pos = base + s + 1; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if vgetq_lane_u64(c, 0) != 0 { - let pos = base + s; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_find/funcs_for_rfind_core/avx2.rs b/src/traits/words_find/funcs_for_rfind_core/avx2.rs new file mode 100644 index 0000000..d7c894b --- /dev/null +++ b/src/traits/words_find/funcs_for_rfind_core/avx2.rs @@ -0,0 +1,77 @@ +// --------------------------------------------------------------------------- +// AVX2 — 4 consecutive shifts at once, reverse +// --------------------------------------------------------------------------- + +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_set1_epi64x, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_set1_epi64x, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn rfind( + haystack: &[u64], + needle_key: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = _mm256_set1_epi64x(needle_key as i64); + let mask = _mm256_set1_epi64x(needle_mask as i64); + + let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); + for i in (0..=start_word).rev() { + let base = i * WORD_BITS; + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + let max_shift = WORD_BITS.min(last_start - base + 1); + + // Round up to a multiple of 4 so the SIMD loop + // processes shifts in 4-lane groups. Out-of-range + // positions are guarded by `pos <= last_start`. + let mut s = max_shift.next_multiple_of(4).min(WORD_BITS); + while s > 0 { + s -= 4; + + let end = WORD_BITS.min(s + 4); + let mut wins = [0u64; 4]; + for k in 0..(end - s) { + let shift = s + k; + wins[k] = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + } + + let windows = unsafe { _mm256_loadu_si256(wins.as_ptr().cast::<__m256i>()) }; + let m = _mm256_and_si256(windows, mask); + let c = _mm256_cmpeq_epi64(m, needle); + let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(c)) } as u32; + + // Check higher shifts first for rightmost. + if hits != 0 { + for k in (0..(end - s)).rev() { + if hits & (1 << k) != 0 { + let pos = base + s + k; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_rfind_core/neon.rs b/src/traits/words_find/funcs_for_rfind_core/neon.rs new file mode 100644 index 0000000..9c6991a --- /dev/null +++ b/src/traits/words_find/funcs_for_rfind_core/neon.rs @@ -0,0 +1,70 @@ +// --------------------------------------------------------------------------- +// NEON — 2 consecutive shifts at once, reverse +// --------------------------------------------------------------------------- + +use super::*; + +use core::arch::aarch64::{ + uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, +}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn rfind( + haystack: &[u64], + needle_key: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = vdupq_n_u64(needle_key); + let mask = vdupq_n_u64(needle_mask); + + let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); + for i in (0..=start_word).rev() { + let base = i * WORD_BITS; + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + let max_shift = WORD_BITS.min(last_start - base + 1); + + // Round up to a multiple of 2 so the SIMD loop + // processes shifts in 2-lane pairs. Out-of-range + // positions are guarded by `pos <= last_start`. + let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); + while s > 0 { + s -= 2; + + let end = WORD_BITS.min(s + 2); + let mut wins = [0u64; 2]; + for k in 0..(end - s) { + let shift = s + k; + wins[k] = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + } + let windows = unsafe { vld1q_u64(wins.as_ptr()) }; + let m = vandq_u64(windows, mask); + let c = vceqq_u64(m, needle); + + // Check higher shift (lane 1) first for rightmost. + if vgetq_lane_u64(c, 1) != 0 { + let pos = base + s + 1; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if vgetq_lane_u64(c, 0) != 0 { + let pos = base + s; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_rfind_core/sse41.rs b/src/traits/words_find/funcs_for_rfind_core/sse41.rs new file mode 100644 index 0000000..e334bc7 --- /dev/null +++ b/src/traits/words_find/funcs_for_rfind_core/sse41.rs @@ -0,0 +1,76 @@ +// --------------------------------------------------------------------------- +// SSE2 — 2 consecutive shifts at once, reverse +// --------------------------------------------------------------------------- + +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, +}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn rfind( + haystack: &[u64], + needle_key: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = _mm_set1_epi64x(needle_key as i64); + let mask = _mm_set1_epi64x(needle_mask as i64); + + let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); + for i in (0..=start_word).rev() { + let base = i * WORD_BITS; + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + let max_shift = WORD_BITS.min(last_start - base + 1); + + // Round up to a multiple of 2 so the SIMD loop + // processes shifts in 2-lane pairs. Out-of-range + // positions are guarded by `pos <= last_start`. + let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); + while s > 0 { + s -= 2; + + let win0 = if s == 0 { + w0 + } else { + (w0 >> s) | (w1 << (WORD_BITS - s)) + }; + let win1 = if s + 1 >= WORD_BITS { + 0 + } else { + (w0 >> (s + 1)) | (w1 << (WORD_BITS - (s + 1))) + }; + let windows = unsafe { _mm_loadu_si128([win0, win1].as_ptr().cast::<__m128i>()) }; + let m = _mm_and_si128(windows, mask); + let c = unsafe { _mm_cmpeq_epi64(m, needle) }; + let hits = _mm_movemask_epi8(c) as u32; + + // Check higher shift (lane 1) first for rightmost. + if hits & 0xff00 != 0 { + let pos = base + s + 1; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if hits & 0xff != 0 { + let pos = base + s; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + + None +} From a7e14fc51e3e9159578fb13099e325c1b80f73b4 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 14:03:58 +0000 Subject: [PATCH 38/79] refactor: split leading_core backends --- .../funcs_for_ends/funcs_for_leading_core.rs | 273 +----------------- .../funcs_for_leading_core/avx2.rs | 142 +++++++++ .../funcs_for_leading_core/neon.rs | 53 ++++ .../funcs_for_leading_core/sse2.rs | 68 +++++ 4 files changed, 266 insertions(+), 270 deletions(-) create mode 100644 src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/avx2.rs create mode 100644 src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/neon.rs create mode 100644 src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/sse2.rs diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs index a53f490..b0df422 100644 --- a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs @@ -137,281 +137,14 @@ pub(crate) fn leading( scanned.min(bit_len) } -// ═══════════════════════════════════════════════════════════════════════ -// AVX2 backend — 256-bit / 4-lane, unaligned loads only. -// ═══════════════════════════════════════════════════════════════════════ - #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, - _mm256_testz_si256, _mm256_xor_si256, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, - _mm256_testz_si256, _mm256_xor_si256, - }; - - const LANES: usize = 4; - const STRIDE: usize = 16; // 4 × LANES for unrolled iteration - const ALIGN_THRESHOLD: usize = 128; - - /// AVX2 forward scan: advances `p` past all-FILL 256-bit chunks. - /// - /// For large inputs (≥ ALIGN_THRESHOLD words), aligns `p` to a - /// 32-byte boundary and uses `_mm256_load_si256` (aligned) for the - /// hot loop. Smaller inputs use `_mm256_loadu_si256` (unaligned) - /// to avoid the alignment prefix overhead. - /// - /// # Safety - /// - /// Caller must ensure AVX2 is enabled for the compilation target. - /// `p` through `end` must be valid for u64 reads. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn leading_scan( - mut p: *const u64, - end: *const u64, - base: *const u64, - total: usize, - ) -> *const u64 { - // SAFETY: AVX2 is enabled for this compilation target. All pointer - // arithmetic stays within bounds. - unsafe { - if total >= ALIGN_THRESHOLD { - // Distance in words to the next 32-byte boundary. - let words_to_align = (4usize.wrapping_sub((base as usize / 8) % 4)) % 4; - if words_to_align > 0 { - let prefix_end = base.add(words_to_align); - while p < prefix_end { - if *p != FILL { - return p; - } - p = p.add(1); - } - } - let mut iters = - (end as usize - p as usize) / (STRIDE * core::mem::size_of::()); - while iters > 0 { - if FILL == 0 { - let d0 = _mm256_load_si256(p.cast::<__m256i>()); - let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); - let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); - let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); - let any01 = _mm256_or_si256(d0, d1); - let any23 = _mm256_or_si256(d2, d3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d0 = _mm256_load_si256(p.cast::<__m256i>()); - let x0 = _mm256_xor_si256(d0, fill_vec); - let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); - let x1 = _mm256_xor_si256(d1, fill_vec); - let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); - let x2 = _mm256_xor_si256(d2, fill_vec); - let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); - let x3 = _mm256_xor_si256(d3, fill_vec); - let any01 = _mm256_or_si256(x0, x1); - let any23 = _mm256_or_si256(x2, x3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } - p = p.add(STRIDE); - iters -= 1; - } - } else { - // 4×-unrolled unaligned path. - let mut iters = total / STRIDE; - while iters > 0 { - if FILL == 0 { - let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); - let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); - let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); - let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); - let any01 = _mm256_or_si256(d0, d1); - let any23 = _mm256_or_si256(d2, d3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); - let x0 = _mm256_xor_si256(d0, fill_vec); - let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); - let x1 = _mm256_xor_si256(d1, fill_vec); - let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); - let x2 = _mm256_xor_si256(d2, fill_vec); - let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); - let x3 = _mm256_xor_si256(d3, fill_vec); - let any01 = _mm256_or_si256(x0, x1); - let any23 = _mm256_or_si256(x2, x3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } - p = p.add(STRIDE); - iters -= 1; - } - } - // Single-chunk remainder (LANES = 4 words). - let limit = end.sub(LANES); - while p <= limit { - if FILL == 0 { - let d = _mm256_loadu_si256(p.cast::<__m256i>()); - if _mm256_testz_si256(d, d) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d = _mm256_loadu_si256(p.cast::<__m256i>()); - let x = _mm256_xor_si256(d, fill_vec); - if _mm256_testz_si256(x, x) == 0 { - break; - } - } - p = p.add(LANES); - } - p - } - } -} - -// ═══════════════════════════════════════════════════════════════════════ -// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). -// ═══════════════════════════════════════════════════════════════════════ +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - _mm_setzero_si128, _mm_xor_si128, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - _mm_setzero_si128, _mm_xor_si128, - }; - - const LANES: usize = 2; - const LANES_2X: usize = LANES * 2; - - #[inline(always)] - unsafe fn chunk_eq(ptr: *const u64) -> bool { - // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and - // SSE2 is available. - unsafe { - let data = _mm_loadu_si128(ptr.cast::<__m128i>()); - if FILL == 0 { - let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); - _mm_movemask_epi8(cmp) == 0xFFFF - } else { - let fill_vec = _mm_set1_epi64x(FILL as i64); - let xor = _mm_xor_si128(data, fill_vec); - let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); - _mm_movemask_epi8(cmp) == 0xFFFF - } - } - } - - /// SSE2 forward scan: advances `p` past all-FILL chunks. - /// - /// # Safety - /// - /// Caller must ensure SSE2 is available (baseline on x86-64). - /// `p` through `end` must be valid for u64 reads. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn leading_scan( - mut p: *const u64, - end: *const u64, - total: usize, - ) -> *const u64 { - // SAFETY: SSE2 is enabled for this compilation target. - unsafe { - let mut iters = total / LANES_2X; - while iters > 0 { - if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { - return p; - } - p = p.add(LANES_2X); - iters -= 1; - } - let limit = end.sub(LANES); - while p <= limit { - if !chunk_eq::(p) { - break; - } - p = p.add(LANES); - } - p - } - } -} - -// ═══════════════════════════════════════════════════════════════════════ -// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). -// ═══════════════════════════════════════════════════════════════════════ +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use core::arch::aarch64::{uint64x2_t, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; - - const LANES: usize = 2; - const LANES_2X: usize = LANES * 2; - - #[inline(always)] - unsafe fn chunk_eq(ptr: *const u64) -> bool { - // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and - // NEON is available. - unsafe { - let data = vld1q_u64(ptr); - let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); - vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 - } - } - - /// NEON forward scan: advances `p` past all-FILL chunks. - /// - /// # Safety - /// - /// Caller must ensure NEON is available. - /// `p` through `end` must be valid for u64 reads. - #[target_feature(enable = "neon")] - pub(super) unsafe fn leading_scan( - mut p: *const u64, - end: *const u64, - total: usize, - ) -> *const u64 { - // SAFETY: only callable when NEON is available. All pointer - // arithmetic stays within `[p, end)`. - unsafe { - let mut iters = total / LANES_2X; - while iters > 0 { - if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { - return p; - } - p = p.add(LANES_2X); - iters -= 1; - } - let limit = end.sub(LANES); - while p <= limit { - if !chunk_eq::(p) { - break; - } - p = p.add(LANES); - } - p - } - } -} +mod neon; diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/avx2.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/avx2.rs new file mode 100644 index 0000000..019acdc --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/avx2.rs @@ -0,0 +1,142 @@ +// ═══════════════════════════════════════════════════════════════════════ +// AVX2 backend — 256-bit / 4-lane, unaligned loads only. +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, + _mm256_testz_si256, _mm256_xor_si256, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, + _mm256_testz_si256, _mm256_xor_si256, +}; + +const LANES: usize = 4; +const STRIDE: usize = 16; // 4 × LANES for unrolled iteration +const ALIGN_THRESHOLD: usize = 128; + +/// AVX2 forward scan: advances `p` past all-FILL 256-bit chunks. +/// +/// For large inputs (≥ ALIGN_THRESHOLD words), aligns `p` to a +/// 32-byte boundary and uses `_mm256_load_si256` (aligned) for the +/// hot loop. Smaller inputs use `_mm256_loadu_si256` (unaligned) +/// to avoid the alignment prefix overhead. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is enabled for the compilation target. +/// `p` through `end` must be valid for u64 reads. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + base: *const u64, + total: usize, +) -> *const u64 { + // SAFETY: AVX2 is enabled for this compilation target. All pointer + // arithmetic stays within bounds. + unsafe { + if total >= ALIGN_THRESHOLD { + // Distance in words to the next 32-byte boundary. + let words_to_align = (4usize.wrapping_sub((base as usize / 8) % 4)) % 4; + if words_to_align > 0 { + let prefix_end = base.add(words_to_align); + while p < prefix_end { + if *p != FILL { + return p; + } + p = p.add(1); + } + } + let mut iters = (end as usize - p as usize) / (STRIDE * core::mem::size_of::()); + while iters > 0 { + if FILL == 0 { + let d0 = _mm256_load_si256(p.cast::<__m256i>()); + let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); + let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); + let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d0 = _mm256_load_si256(p.cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } + p = p.add(STRIDE); + iters -= 1; + } + } else { + // 4×-unrolled unaligned path. + let mut iters = total / STRIDE; + while iters > 0 { + if FILL == 0 { + let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); + let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); + let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); + let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } + p = p.add(STRIDE); + iters -= 1; + } + } + // Single-chunk remainder (LANES = 4 words). + let limit = end.sub(LANES); + while p <= limit { + if FILL == 0 { + let d = _mm256_loadu_si256(p.cast::<__m256i>()); + if _mm256_testz_si256(d, d) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d = _mm256_loadu_si256(p.cast::<__m256i>()); + let x = _mm256_xor_si256(d, fill_vec); + if _mm256_testz_si256(x, x) == 0 { + break; + } + } + p = p.add(LANES); + } + p + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/neon.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/neon.rs new file mode 100644 index 0000000..f65d698 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/neon.rs @@ -0,0 +1,53 @@ +// ═══════════════════════════════════════════════════════════════════════ +// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +use core::arch::aarch64::{uint64x2_t, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; + +const LANES: usize = 2; +const LANES_2X: usize = LANES * 2; + +#[inline(always)] +unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // NEON is available. + unsafe { + let data = vld1q_u64(ptr); + let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); + vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 + } +} + +/// NEON forward scan: advances `p` past all-FILL chunks. +/// +/// # Safety +/// +/// Caller must ensure NEON is available. +/// `p` through `end` must be valid for u64 reads. +#[target_feature(enable = "neon")] +pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + total: usize, +) -> *const u64 { + // SAFETY: only callable when NEON is available. All pointer + // arithmetic stays within `[p, end)`. + unsafe { + let mut iters = total / LANES_2X; + while iters > 0 { + if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { + return p; + } + p = p.add(LANES_2X); + iters -= 1; + } + let limit = end.sub(LANES); + while p <= limit { + if !chunk_eq::(p) { + break; + } + p = p.add(LANES); + } + p + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/sse2.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/sse2.rs new file mode 100644 index 0000000..8e36b87 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/sse2.rs @@ -0,0 +1,68 @@ +// ═══════════════════════════════════════════════════════════════════════ +// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, +}; + +const LANES: usize = 2; +const LANES_2X: usize = LANES * 2; + +#[inline(always)] +unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // SSE2 is available. + unsafe { + let data = _mm_loadu_si128(ptr.cast::<__m128i>()); + if FILL == 0 { + let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } else { + let fill_vec = _mm_set1_epi64x(FILL as i64); + let xor = _mm_xor_si128(data, fill_vec); + let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } + } +} + +/// SSE2 forward scan: advances `p` past all-FILL chunks. +/// +/// # Safety +/// +/// Caller must ensure SSE2 is available (baseline on x86-64). +/// `p` through `end` must be valid for u64 reads. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + total: usize, +) -> *const u64 { + // SAFETY: SSE2 is enabled for this compilation target. + unsafe { + let mut iters = total / LANES_2X; + while iters > 0 { + if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { + return p; + } + p = p.add(LANES_2X); + iters -= 1; + } + let limit = end.sub(LANES); + while p <= limit { + if !chunk_eq::(p) { + break; + } + p = p.add(LANES); + } + p + } +} From bc61492f9c1d561ff1f831e977bf4e68c606d2fc Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 14:31:00 +0000 Subject: [PATCH 39/79] fix: cover leading codegen specializations --- scripts/codegen_harness.rs | 18 ++++++++++++++++++ scripts/codegen_roots.toml | 15 +++++++++++++++ scripts/kernel_inventory.toml | 7 ++++++- scripts/tests/test_codegen.py | 15 +++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/codegen_harness.rs b/scripts/codegen_harness.rs index 9d171de..c4e6c08 100644 --- a/scripts/codegen_harness.rs +++ b/scripts/codegen_harness.rs @@ -113,6 +113,24 @@ pub unsafe fn codegen_bit_string_leading_zeros(value: *const BitString) -> usize unsafe { (&*value).leading_zeros() } } +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_leading_ones(value: *const BitString) -> usize { + unsafe { (&*value).leading_ones() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_str_leading_zeros(value: *const BitString, start: usize) -> usize { + unsafe { (&*value).as_bit_str().slice_from(start).leading_zeros() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_str_leading_ones(value: *const BitString, start: usize) -> usize { + unsafe { (&*value).as_bit_str().slice_from(start).leading_ones() } +} + #[unsafe(no_mangle)] #[inline(never)] pub unsafe fn codegen_bit_string_trailing_zeros(value: *const BitString) -> usize { diff --git a/scripts/codegen_roots.toml b/scripts/codegen_roots.toml index fe07bd3..a7f52dc 100644 --- a/scripts/codegen_roots.toml +++ b/scripts/codegen_roots.toml @@ -78,6 +78,21 @@ symbol = "codegen_bit_string_leading_zeros" artifact = "harness" targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] +[[root]] +symbol = "codegen_bit_string_leading_ones" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_str_leading_zeros" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_str_leading_ones" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + [[root]] symbol = "codegen_bit_string_trailing_zeros" artifact = "harness" diff --git a/scripts/kernel_inventory.toml b/scripts/kernel_inventory.toml index 96ee7d8..42ce048 100644 --- a/scripts/kernel_inventory.toml +++ b/scripts/kernel_inventory.toml @@ -79,7 +79,12 @@ roots = ["codegen_bit_string_count_ones"] [[kernel]] path = "src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs" coverage = "caller" -roots = ["codegen_bit_string_leading_zeros"] +roots = [ + "codegen_bit_string_leading_zeros", + "codegen_bit_string_leading_ones", + "codegen_bit_str_leading_zeros", + "codegen_bit_str_leading_ones", +] [[kernel]] path = "src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs" diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 38eaa75..13c52f7 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -679,6 +679,21 @@ def test_inventory_has_root_for_each_kernel(self) -> None: {"codegen_bit_string_and", "codegen_bit_string_or", "codegen_bit_string_xor"}, ) + leading = next( + kernel + for kernel in inventory + if kernel["path"] == "src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs" + ) + self.assertEqual( + set(leading["roots"]), + { + "codegen_bit_string_leading_zeros", + "codegen_bit_string_leading_ones", + "codegen_bit_str_leading_zeros", + "codegen_bit_str_leading_ones", + }, + ) + def test_independent_backend_symbols_are_discovered_from_inventory_modules(self) -> None: function = parse("0000 :\n 0: c3 ret") functions = { From 9aef41e489817fe4c146bb2424b94b1db5489874 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 15:01:37 +0000 Subject: [PATCH 40/79] fix: cover trailing codegen specializations --- scripts/codegen_harness.rs | 18 ++++++++++++++++++ scripts/codegen_roots.toml | 15 +++++++++++++++ scripts/kernel_inventory.toml | 7 ++++++- scripts/tests/test_codegen.py | 15 +++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/codegen_harness.rs b/scripts/codegen_harness.rs index c4e6c08..d662f21 100644 --- a/scripts/codegen_harness.rs +++ b/scripts/codegen_harness.rs @@ -136,3 +136,21 @@ pub unsafe fn codegen_bit_str_leading_ones(value: *const BitString, start: usize pub unsafe fn codegen_bit_string_trailing_zeros(value: *const BitString) -> usize { unsafe { (&*value).trailing_zeros() } } + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_string_trailing_ones(value: *const BitString) -> usize { + unsafe { (&*value).trailing_ones() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_str_trailing_zeros(value: *const BitString, start: usize) -> usize { + unsafe { (&*value).as_bit_str().slice_from(start).trailing_zeros() } +} + +#[unsafe(no_mangle)] +#[inline(never)] +pub unsafe fn codegen_bit_str_trailing_ones(value: *const BitString, start: usize) -> usize { + unsafe { (&*value).as_bit_str().slice_from(start).trailing_ones() } +} diff --git a/scripts/codegen_roots.toml b/scripts/codegen_roots.toml index a7f52dc..fe16537 100644 --- a/scripts/codegen_roots.toml +++ b/scripts/codegen_roots.toml @@ -98,6 +98,21 @@ symbol = "codegen_bit_string_trailing_zeros" artifact = "harness" targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] +[[root]] +symbol = "codegen_bit_string_trailing_ones" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_str_trailing_zeros" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + +[[root]] +symbol = "codegen_bit_str_trailing_ones" +artifact = "harness" +targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] + [[root]] symbol = "bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words" artifact = "library" diff --git a/scripts/kernel_inventory.toml b/scripts/kernel_inventory.toml index 42ce048..714aa1b 100644 --- a/scripts/kernel_inventory.toml +++ b/scripts/kernel_inventory.toml @@ -89,4 +89,9 @@ roots = [ [[kernel]] path = "src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs" coverage = "caller" -roots = ["codegen_bit_string_trailing_zeros"] +roots = [ + "codegen_bit_string_trailing_zeros", + "codegen_bit_string_trailing_ones", + "codegen_bit_str_trailing_zeros", + "codegen_bit_str_trailing_ones", +] diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 13c52f7..4478a2e 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -694,6 +694,21 @@ def test_inventory_has_root_for_each_kernel(self) -> None: }, ) + trailing = next( + kernel + for kernel in inventory + if kernel["path"] == "src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs" + ) + self.assertEqual( + set(trailing["roots"]), + { + "codegen_bit_string_trailing_zeros", + "codegen_bit_string_trailing_ones", + "codegen_bit_str_trailing_zeros", + "codegen_bit_str_trailing_ones", + }, + ) + def test_independent_backend_symbols_are_discovered_from_inventory_modules(self) -> None: function = parse("0000 :\n 0: c3 ret") functions = { From 189ea51f611bf019add4231e5154c3578730124b Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 30 Aug 2026 15:20:44 +0000 Subject: [PATCH 41/79] refactor: split trailing_core backends --- .../funcs_for_ends/funcs_for_trailing_core.rs | 246 +----------------- .../funcs_for_trailing_core/avx2.rs | 94 +++++++ .../funcs_for_trailing_core/neon.rs | 55 ++++ .../funcs_for_trailing_core/sse2.rs | 70 +++++ 4 files changed, 231 insertions(+), 234 deletions(-) create mode 100644 src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/avx2.rs create mode 100644 src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/neon.rs create mode 100644 src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/sse2.rs diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs index 884d8c2..3750bbb 100644 --- a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs @@ -7,6 +7,18 @@ use super::count_matching; use crate::{SMALL_WORDS, WORD_BITS}; +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2; + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod sse2; + +#[allow(unused)] +#[cfg(target_arch = "aarch64")] +mod neon; + // ── Dispatch ─────────────────────────────────────────────────────────── #[inline] @@ -140,237 +152,3 @@ pub(crate) fn trailing( scanned.min(bit_len) } - -// ═══════════════════════════════════════════════════════════════════════ -// AVX2 backend — 256-bit / 4-lane reverse scan. -// ═══════════════════════════════════════════════════════════════════════ - -#[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, - _mm256_xor_si256, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, - _mm256_xor_si256, - }; - - const LANES: usize = 4; - const STRIDE: usize = 16; - - /// AVX2 reverse scan: scans backwards from `wi_end` and advances `done` - /// past all-FILL 256-bit chunks. - /// - /// Returns the updated `done` count (total words consumed from right). - /// - /// # Safety - /// - /// Caller must ensure AVX2 is enabled for the compilation target. - /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn trailing_scan( - ptr: *const u64, - wi_end: usize, - mut done: usize, - total_words: usize, - ) -> usize { - // SAFETY: AVX2 is enabled for this compilation target. All pointer - // arithmetic stays within bounds. - unsafe { - // Four vectors are combined before testing. This keeps the - // all-matching hot path to one branch per 16 words, matching - // the forward scanner's throughput-oriented layout. - while done + STRIDE <= total_words { - let chunk_start = wi_end + 1 - (done + STRIDE); - if FILL == 0 { - let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); - let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); - let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); - let any01 = _mm256_or_si256(d0, d1); - let any23 = _mm256_or_si256(d2, d3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - let x0 = _mm256_xor_si256(d0, fill_vec); - let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); - let x1 = _mm256_xor_si256(d1, fill_vec); - let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); - let x2 = _mm256_xor_si256(d2, fill_vec); - let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); - let x3 = _mm256_xor_si256(d3, fill_vec); - let any01 = _mm256_or_si256(x0, x1); - let any23 = _mm256_or_si256(x2, x3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } - done += STRIDE; - } - // Single-chunk remainder - while done + LANES <= total_words { - let chunk_start = wi_end + 1 - (done + LANES); - if FILL == 0 { - let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - if _mm256_testz_si256(d, d) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - let x = _mm256_xor_si256(d, fill_vec); - if _mm256_testz_si256(x, x) == 0 { - break; - } - } - done += LANES; - } - - done - } - } -} - -// ═══════════════════════════════════════════════════════════════════════ -// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). -// ═══════════════════════════════════════════════════════════════════════ - -#[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - _mm_setzero_si128, _mm_xor_si128, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - _mm_setzero_si128, _mm_xor_si128, - }; - - const LANES: usize = 2; - const LANES_2X: usize = LANES * 2; - - #[inline(always)] - unsafe fn chunk_eq(ptr: *const u64) -> bool { - // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and - // SSE2 is available. - unsafe { - let data = _mm_loadu_si128(ptr.cast::<__m128i>()); - if FILL == 0 { - let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); - _mm_movemask_epi8(cmp) == 0xFFFF - } else { - let fill_vec = _mm_set1_epi64x(FILL as i64); - let xor = _mm_xor_si128(data, fill_vec); - let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); - _mm_movemask_epi8(cmp) == 0xFFFF - } - } - } - - /// SSE2 reverse scan: advances `done` past all-FILL chunks from the right. - /// - /// # Safety - /// - /// Caller must ensure SSE2 is available (baseline on x86-64). - /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn trailing_scan( - ptr: *const u64, - wi_end: usize, - mut done: usize, - total_words: usize, - ) -> usize { - // SAFETY: SSE2 is enabled for this compilation target. - unsafe { - while done + LANES_2X <= total_words { - let chunk_start = wi_end + 1 - (done + LANES_2X); - if !chunk_eq::(ptr.add(chunk_start)) - || !chunk_eq::(ptr.add(chunk_start + LANES)) - { - return done; - } - done += LANES_2X; - } - while done + LANES <= total_words { - let chunk_start = wi_end + 1 - (done + LANES); - if !chunk_eq::(ptr.add(chunk_start)) { - break; - } - done += LANES; - } - done - } - } -} - -// ═══════════════════════════════════════════════════════════════════════ -// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). -// ═══════════════════════════════════════════════════════════════════════ - -#[allow(unused)] -#[cfg(target_arch = "aarch64")] -mod neon { - use core::arch::aarch64::{vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; - - const LANES: usize = 2; - const LANES_2X: usize = LANES * 2; - - #[inline(always)] - unsafe fn chunk_eq(ptr: *const u64) -> bool { - // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and - // NEON is available. - unsafe { - let data = vld1q_u64(ptr); - let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); - vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 - } - } - - /// NEON reverse scan: advances `done` past all-FILL chunks from the right. - /// - /// # Safety - /// - /// Caller must ensure NEON is available. - /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. - #[target_feature(enable = "neon")] - pub(super) unsafe fn trailing_scan( - ptr: *const u64, - wi_end: usize, - mut done: usize, - total_words: usize, - ) -> usize { - // SAFETY: only callable when NEON is available. All pointer - // arithmetic stays within bounds. - unsafe { - while done + LANES_2X <= total_words { - let chunk_start = wi_end + 1 - (done + LANES_2X); - if !chunk_eq::(ptr.add(chunk_start)) - || !chunk_eq::(ptr.add(chunk_start + LANES)) - { - return done; - } - done += LANES_2X; - } - while done + LANES <= total_words { - let chunk_start = wi_end + 1 - (done + LANES); - if !chunk_eq::(ptr.add(chunk_start)) { - break; - } - done += LANES; - } - done - } - } -} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/avx2.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/avx2.rs new file mode 100644 index 0000000..802ea69 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/avx2.rs @@ -0,0 +1,94 @@ +// ═══════════════════════════════════════════════════════════════════════ +// AVX2 backend — 256-bit / 4-lane reverse scan. +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, + _mm256_xor_si256, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, + _mm256_xor_si256, +}; + +const LANES: usize = 4; +const STRIDE: usize = 16; + +/// AVX2 reverse scan: scans backwards from `wi_end` and advances `done` +/// past all-FILL 256-bit chunks. +/// +/// Returns the updated `done` count (total words consumed from right). +/// +/// # Safety +/// +/// Caller must ensure AVX2 is enabled for the compilation target. +/// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, +) -> usize { + // SAFETY: AVX2 is enabled for this compilation target. All pointer + // arithmetic stays within bounds. + unsafe { + // Four vectors are combined before testing. This keeps the + // all-matching hot path to one branch per 16 words, matching + // the forward scanner's throughput-oriented layout. + while done + STRIDE <= total_words { + let chunk_start = wi_end + 1 - (done + STRIDE); + if FILL == 0 { + let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); + let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); + let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } + done += STRIDE; + } + // Single-chunk remainder + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if FILL == 0 { + let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + if _mm256_testz_si256(d, d) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let x = _mm256_xor_si256(d, fill_vec); + if _mm256_testz_si256(x, x) == 0 { + break; + } + } + done += LANES; + } + + done + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/neon.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/neon.rs new file mode 100644 index 0000000..c5274f4 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/neon.rs @@ -0,0 +1,55 @@ +// ═══════════════════════════════════════════════════════════════════════ +// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +use core::arch::aarch64::{vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; + +const LANES: usize = 2; +const LANES_2X: usize = LANES * 2; + +#[inline(always)] +unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // NEON is available. + unsafe { + let data = vld1q_u64(ptr); + let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); + vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 + } +} + +/// NEON reverse scan: advances `done` past all-FILL chunks from the right. +/// +/// # Safety +/// +/// Caller must ensure NEON is available. +/// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. +#[target_feature(enable = "neon")] +pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, +) -> usize { + // SAFETY: only callable when NEON is available. All pointer + // arithmetic stays within bounds. + unsafe { + while done + LANES_2X <= total_words { + let chunk_start = wi_end + 1 - (done + LANES_2X); + if !chunk_eq::(ptr.add(chunk_start)) + || !chunk_eq::(ptr.add(chunk_start + LANES)) + { + return done; + } + done += LANES_2X; + } + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if !chunk_eq::(ptr.add(chunk_start)) { + break; + } + done += LANES; + } + done + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/sse2.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/sse2.rs new file mode 100644 index 0000000..267a4b0 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/sse2.rs @@ -0,0 +1,70 @@ +// ═══════════════════════════════════════════════════════════════════════ +// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, +}; + +const LANES: usize = 2; +const LANES_2X: usize = LANES * 2; + +#[inline(always)] +unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // SSE2 is available. + unsafe { + let data = _mm_loadu_si128(ptr.cast::<__m128i>()); + if FILL == 0 { + let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } else { + let fill_vec = _mm_set1_epi64x(FILL as i64); + let xor = _mm_xor_si128(data, fill_vec); + let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } + } +} + +/// SSE2 reverse scan: advances `done` past all-FILL chunks from the right. +/// +/// # Safety +/// +/// Caller must ensure SSE2 is available (baseline on x86-64). +/// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, +) -> usize { + // SAFETY: SSE2 is enabled for this compilation target. + unsafe { + while done + LANES_2X <= total_words { + let chunk_start = wi_end + 1 - (done + LANES_2X); + if !chunk_eq::(ptr.add(chunk_start)) + || !chunk_eq::(ptr.add(chunk_start + LANES)) + { + return done; + } + done += LANES_2X; + } + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if !chunk_eq::(ptr.add(chunk_start)) { + break; + } + done += LANES; + } + done + } +} From 7971ed45ed019dbfa8efdb925282ed1c4488fa80 Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 31 Aug 2026 13:50:47 +0000 Subject: [PATCH 42/79] fix: refine codegen topology comparison --- scripts/compare_codegen.py | 109 ++++++++++++++++-- scripts/tests/test_codegen.py | 211 +++++++++++++++++++++++++++++++++- 2 files changed, 310 insertions(+), 10 deletions(-) diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py index b3ad6be..c8f3666 100755 --- a/scripts/compare_codegen.py +++ b/scripts/compare_codegen.py @@ -28,6 +28,7 @@ r"(?P(?:\.Lanon|anon)\.[^.\s]+)\.(?P\d+)" r"(?:\.llvm\.[^+\-\s]+)?(?P[+-]0x[0-9a-f]+)?$" ) +LOCAL_OBJECT_SYMBOL = re.compile(r"(?P\.LCPI\d+_\d+)(?P[+-]0x[0-9a-f]+)?$") OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") BACKEND_MODULE = re.compile(r"::(?:scalar|sse2|ssse3|sse41|avx2|neon)::") @@ -53,6 +54,7 @@ class Relocation: addend: str = "" nested_addends: tuple[str, ...] = () nested_alias_partition: tuple[int, ...] = () + metadata: bool = False @dataclass(frozen=True) @@ -78,6 +80,7 @@ class ResolvedTarget: addend: str = "" nested_addends: tuple[str, ...] = () nested_alias_partition: tuple[int, ...] = () + metadata: bool = False class ArtifactSections: @@ -88,6 +91,7 @@ class ArtifactSections: RELOCATION_HEADER = re.compile(r"^RELOCATION RECORDS FOR \[(.+)\]:$") RELOCATION_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") SYMBOL_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+\S+\s+\S+\s+(\S+)\s+([0-9a-f]+)\s+(.+?)\s*$") + LOCAL_SYMBOL_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+l\s+(\S+)\s+([0-9a-f]+)\s+(\.LCPI\d+_\d+)\s*$") def __init__( self, @@ -181,6 +185,14 @@ def from_objdump( int(symbol.group(1), 16), int(symbol.group(3), 16), ) + continue + local_symbol = cls.LOCAL_SYMBOL_ENTRY.match(line) + if local_symbol: + symbols[(member, local_symbol.group(4))] = ( + local_symbol.group(2), + int(local_symbol.group(1), 16), + int(local_symbol.group(3), 16), + ) return cls(sections, arch, symbols) def _find(self, member: str, name: str) -> SectionData | None: @@ -200,7 +212,20 @@ def _find(self, member: str, name: str) -> SectionData | None: return None symbol_section, offset, size = symbol section = self.sections.get((member, symbol_section)) - if section is None or offset < 0 or offset + size > len(section.data): + if section is None or offset < 0 or offset > len(section.data): + return None + if size == 0 and _local_object_instance(name) is not None: + next_offsets = [ + candidate_offset + for (candidate_member, candidate_name), (candidate_section, candidate_offset, _) in self.symbols.items() + if candidate_member == member + and candidate_section == symbol_section + and candidate_name != name + and _local_object_instance(candidate_name) is not None + and candidate_offset > offset + ] + size = min(next_offsets, default=len(section.data)) - offset + if offset + size > len(section.data): return None relocations = tuple( SectionRelocation(relocation.offset - offset, relocation.kind, relocation.target) @@ -263,7 +288,8 @@ def _fingerprint_details( else: target_name, addend = self._split_target(target) anonymous = _anonymous_instance(target_name) - if anonymous is not None: + local_object = _local_object_instance(target_name) + if anonymous is not None or local_object is not None: # Deliberately support exactly one nested anonymous level. # A nested anonymous target at this level would require a # recursive object graph, so remain fail-closed. @@ -303,7 +329,8 @@ def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: if kind in PC_RELATIVE_X86_RELOCATIONS: target = X86_PC_BIAS.sub("", target) anonymous = _anonymous_instance(target) - if anonymous is None: + local_object = _local_object_instance(target) + if anonymous is None and local_object is None: name, addend = self._split_target(target) string = self._string_target(member, name, addend) token = f"string:{string}" if string is not None else f"symbol:{name}{addend}" @@ -311,10 +338,16 @@ def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: name, addend = self._split_target(target) section = self._find(member, name) if section is None: - return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False, addend) + object_name = anonymous[0] if anonymous is not None else name + return ResolvedTarget( + f"anonymous-unresolved:{object_name}", True, False, addend, metadata=anonymous is not None + ) details = self._fingerprint_details(section, member) if details is None: - return ResolvedTarget(f"anonymous-unresolved:{anonymous[0]}", True, False, addend) + object_name = anonymous[0] if anonymous is not None else name + return ResolvedTarget( + f"anonymous-unresolved:{object_name}", True, False, addend, metadata=anonymous is not None + ) fingerprint, nested_addends, nested_alias_partition = details return ResolvedTarget( f"anonymous:{fingerprint}", @@ -323,6 +356,7 @@ def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: addend, nested_addends, nested_alias_partition, + anonymous is not None, ) @@ -371,6 +405,7 @@ def _same_relocations(old: tuple[Relocation, ...], new: tuple[Relocation, ...]) and before.addend == after.addend and before.nested_addends == after.nested_addends and before.nested_alias_partition == after.nested_alias_partition + and before.metadata == after.metadata and (not before.anonymous or before.resolved) for before, after in zip(old, new) ) and len(old) == len(new) @@ -387,12 +422,16 @@ def _metadata_only_relocations(old: Function, new: Function) -> bool: or before.kind != after.kind or before.anonymous != after.anonymous or before.addend != after.addend - or before.nested_addends != after.nested_addends - or before.nested_alias_partition != after.nested_alias_partition + or before.metadata != after.metadata + ): + return False + if ( + (before.nested_addends != after.nested_addends or before.nested_alias_partition != after.nested_alias_partition) + and not _anonymous_wrapper_representation_changed(before, after) ): return False if before.token != after.token: - if not before.anonymous or not before.resolved or not after.resolved: + if not before.anonymous or not before.resolved or not after.resolved or not before.metadata: return False changed = True return changed @@ -406,6 +445,31 @@ def _anonymous_instance(value: str) -> tuple[str, int, str] | None: return key, int(match.group("instance")), match.group("addend") or "" +def _anonymous_wrapper_representation_changed(before: Relocation, after: Relocation) -> bool: + """Allow rustc to flatten a single anonymous metadata wrapper.""" + + def style(value: str) -> str: + value = value.strip() + if value.startswith("anon."): + return "llvm" + if ".Lanon." in value: + return "section" + return "other" + + return ( + {style(before.raw_target), style(after.raw_target)} == {"llvm", "section"} + and {before.nested_addends, after.nested_addends} == {(), ("",)} + and {before.nested_alias_partition, after.nested_alias_partition} == {(), (0,)} + ) + + +def _local_object_instance(value: str) -> tuple[str, str] | None: + match = LOCAL_OBJECT_SYMBOL.fullmatch(value.strip()) + if match is None: + return None + return match.group("name"), match.group("addend") or "" + + def _relocation_target(kind: str, value: str, anonymous_bases: dict[str, int]) -> str: value = value.strip() if kind in PC_RELATIVE_X86_RELOCATIONS: @@ -475,6 +539,7 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec addend = _anonymous_instance(target)[2] if anonymous else "" nested_addends = () nested_alias_partition = () + metadata = anonymous else: resolved_target = sections.resolve(instruction.member, kind, target) token = resolved_target.token @@ -483,6 +548,7 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec addend = resolved_target.addend nested_addends = resolved_target.nested_addends nested_alias_partition = resolved_target.nested_alias_partition + metadata = resolved_target.metadata relocation_text.append(f"[{kind}:{token}]") relocations.append( Relocation( @@ -496,6 +562,7 @@ def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSec addend, nested_addends, nested_alias_partition, + metadata, ) ) if anonymous: @@ -594,6 +661,17 @@ def load_kernel_modules(path: Path) -> tuple[str, ...]: return tuple(modules) +def load_kernel_caller_modules(path: Path) -> tuple[str, ...]: + data = tomllib.loads(path.read_text(encoding="utf-8")) + modules = [] + for kernel in data.get("kernel", []): + if kernel.get("coverage") != "caller": + continue + source = Path(kernel["path"]) + modules.append("bit_string::" + "::".join(source.with_suffix("").parts[1:])) + return tuple(modules) + + def independent_backend_symbols(functions: dict[str, Function], modules: tuple[str, ...]) -> set[str]: return { symbol @@ -618,10 +696,13 @@ def main() -> int: after = load_functions(args.current, args.objdump, args.arch) roots = [root for root in load_roots(args.roots) if root.artifact == args.artifact] identical = metadata_changed = encoding_changed = changed = errors = skipped = 0 + topology_changed = 0 + caller_covered_modules: tuple[str, ...] = () if args.artifact == "library": if args.inventory is None: parser.error("--inventory is required for library artifacts") modules = load_kernel_modules(args.inventory) + caller_covered_modules = load_kernel_caller_modules(args.inventory) registered = {root.symbol for root in roots if args.target in root.targets} discovered = independent_backend_symbols(before, modules) | independent_backend_symbols(after, modules) for symbol in sorted(discovered - registered): @@ -633,6 +714,17 @@ def main() -> int: continue old, new = before.get(root.symbol), after.get(root.symbol) if old is None or new is None: + if ( + args.artifact == "library" + and old is not None + and new is None + and BACKEND_MODULE.search(root.symbol) + and any(root.symbol.startswith(f"{module}::") for module in caller_covered_modules) + ): + topology_changed += 1 + print(f"{root.symbol}: CODEGEN TOPOLOGY CHANGED") + print(" independent backend symbol disappeared; caller coverage remains required") + continue errors += 1 where = "both" if old is None and new is None else "baseline" if old is None else "current" print(f"ERROR: expected symbol missing in {where}: {root.symbol}", file=sys.stderr) @@ -674,6 +766,7 @@ def main() -> int: print(f"metadata changed : {metadata_changed}") print(f"encoding changed : {encoding_changed}") print(f"changed : {changed}") + print(f"topology changed : {topology_changed}") print(f"errors : {errors}") print(f"skipped : {skipped}") failed = encoding_changed or changed or errors or (args.strict_metadata and metadata_changed) diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py index 4478a2e..c63a7cd 100644 --- a/scripts/tests/test_codegen.py +++ b/scripts/tests/test_codegen.py @@ -19,8 +19,10 @@ def parse(body: str, arch: str = "x86_64") -> compare_codegen.Function: return compare_codegen.parse_objdump(body, arch)["root"] -def parse_with_sections(body: str, sections: str, arch: str = "x86_64") -> compare_codegen.Function: - index = compare_codegen.ArtifactSections.from_objdump(sections, arch) +def parse_with_sections( + body: str, sections: str, arch: str = "x86_64", symbols: str = "" +) -> compare_codegen.Function: + index = compare_codegen.ArtifactSections.from_objdump(sections, arch, symbols) return compare_codegen.parse_objdump(body, arch, index)["root"] @@ -414,6 +416,99 @@ def test_section_bytes_ignore_ascii_column_after_four_groups(self) -> None: index = compare_codegen.ArtifactSections.from_objdump(output, "x86_64") section = index.sections[("fixture.o", ".rodata")] self.assertEqual(section.data, b"\0" * 16) + + def test_lcpi_symbol_ordinal_change_uses_object_content(self) -> None: + def sections(symbol: str, value: str) -> tuple[str, str]: + body = ( + "fixture.o: file format elf64-x86-64\n" + "Contents of section .rodata.cst16:\n" + f" 0000 {value} 00000000 00000000 00000000 ................\n" + ) + symbols = ( + "fixture.o: file format elf64-x86-64\n\n" + "SYMBOL TABLE:\n" + f"0000000000000000 l .rodata.cst16\t0000000000000010 {symbol}\n" + ) + return body, symbols + + old_sections, old_symbols = sections(".LCPI13_0", "01020304") + new_sections, new_symbols = sections(".LCPI11_0", "01020304") + old = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .LCPI13_0-0x4\n", + old_sections, + symbols=old_symbols, + ) + new = parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .LCPI11_0-0x4\n", + new_sections, + symbols=new_symbols, + ) + self.assertTrue(old.relocations[0].resolved) + self.assertTrue(new.relocations[0].resolved) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") + + def test_lcpi_content_change_is_not_ignored(self) -> None: + def make(value: str) -> compare_codegen.Function: + sections = ( + "fixture.o: file format elf64-x86-64\n" + "Contents of section .rodata.cst16:\n" + f" 0000 {value} 00000000 00000000 00000000 ................\n" + ) + symbols = ( + "fixture.o: file format elf64-x86-64\n\n" + "SYMBOL TABLE:\n" + "0000000000000000 l .rodata.cst16\t0000000000000010 .LCPI13_0\n" + ) + return parse_with_sections( + "fixture.o: file format elf64-x86-64\n" + "0000 :\n" + " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" + " 3: R_X86_64_PC32 .LCPI13_0-0x4\n", + sections, + symbols=symbols, + ) + + self.assertEqual(compare_codegen.classify(make("01020304"), make("05060708")), "CODEGEN CHANGED") + + def test_rustc_anonymous_representation_change_is_metadata_only(self) -> None: + def function( + raw_target: str, + token: str, + nested_addends: tuple[str, ...], + aliases: tuple[int, ...], + ) -> compare_codegen.Function: + relocation = compare_codegen.Relocation( + 3, + "R_X86_64_PC32", + raw_target, + token, + True, + True, + 0, + "-0x4", + nested_addends, + aliases, + True, + ) + return compare_codegen.Function( + ("lea %rax",), + b"\x48\x8d\x05\0\0\0\0", + 0, + 0, + ("lea %rax",), + (relocation,), + (0,), + ) + + old = function("anon.hash.20.llvm.123-0x4", "anonymous:old", ("",), (0,)) + new = function(".data.rel.ro..Lanon.hash.20-0x4", "anonymous:new", (), ()) + self.assertEqual(compare_codegen.classify(old, new), "CODEGEN METADATA CHANGED") def test_call_target_identity_is_preserved(self) -> None: foo = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 foo-0x4") bar = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 bar-0x4") @@ -588,6 +683,118 @@ def test_expected_missing_symbol_fails(self) -> None: ): self.assertNotEqual(compare_codegen.main(), 0) + def test_covered_backend_disappearing_is_topology_change(self) -> None: + function = parse("0000 :\n 0: c3 ret") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + roots = root / "roots.toml" + roots.write_text( + '[[root]]\nsymbol="bit_string::foo::avx2::words"\n' + 'artifact="library"\ntargets=["x86_64-avx2"]\n' + ) + inventory = root / "inventory.toml" + inventory.write_text( + '[[kernel]]\npath="src/foo.rs"\ncoverage="caller"\nroots=["caller"]\n' + ) + argv = [ + "compare_codegen.py", + "baseline.o", + "current.o", + "--roots", + str(roots), + "--artifact", + "library", + "--inventory", + str(inventory), + "--target", + "x86_64-avx2", + "--arch", + "x86_64", + ] + stdout = io.StringIO() + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object( + compare_codegen, + "load_functions", + side_effect=[{"bit_string::foo::avx2::words": function}, {}], + ), + contextlib.redirect_stdout(stdout), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(compare_codegen.main(), 0) + self.assertIn("CODEGEN TOPOLOGY CHANGED", stdout.getvalue()) + + def test_caller_root_disappearing_still_fails(self) -> None: + function = parse("0000 :\n 0: c3 ret") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + roots = root / "roots.toml" + roots.write_text( + '[[root]]\nsymbol="caller"\nartifact="harness"\ntargets=["x86_64-avx2"]\n' + ) + argv = [ + "compare_codegen.py", + "baseline.o", + "current.o", + "--roots", + str(roots), + "--artifact", + "harness", + "--target", + "x86_64-avx2", + "--arch", + "x86_64", + ] + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object( + compare_codegen, + "load_functions", + side_effect=[{"caller": function}, {}], + ), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertNotEqual(compare_codegen.main(), 0) + + def test_new_unregistered_backend_still_fails(self) -> None: + function = parse("0000 :\n 0: c3 ret") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + roots = root / "roots.toml" + roots.write_text('[[root]]\nsymbol="caller"\nartifact="library"\ntargets=["x86_64-avx2"]\n') + inventory = root / "inventory.toml" + inventory.write_text( + '[[kernel]]\npath="src/foo.rs"\ncoverage="caller"\nroots=["caller"]\n' + ) + argv = [ + "compare_codegen.py", + "baseline.o", + "current.o", + "--roots", + str(roots), + "--artifact", + "library", + "--inventory", + str(inventory), + "--target", + "x86_64-avx2", + "--arch", + "x86_64", + ] + with ( + mock.patch.object(sys, "argv", argv), + mock.patch.object( + compare_codegen, + "load_functions", + side_effect=[{}, {"bit_string::foo::avx2::words": function}], + ), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertNotEqual(compare_codegen.main(), 0) + def test_metadata_policy_exit_codes(self) -> None: old = parse_with_sections( "fixture.o: file format elf64-x86-64\n" From 66ab326dcaeb090c7887414e327bbbbb0f7fbf73 Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 31 Aug 2026 14:04:26 +0000 Subject: [PATCH 43/79] refactor: split words_eq scalar backends --- .../funcs_for_eq_words_aligned_core.rs | 19 ++++----------- .../funcs_for_eq_words_aligned_core/scalar.rs | 8 +++++++ .../funcs_for_eq_words_unaligned_core.rs | 23 ++++--------------- .../scalar.rs | 12 ++++++++++ 4 files changed, 30 insertions(+), 32 deletions(-) create mode 100644 src/traits/words_eq/funcs_for_eq_words_aligned_core/scalar.rs create mode 100644 src/traits/words_eq/funcs_for_eq_words_unaligned_core/scalar.rs diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs index 856ec9d..eea0d97 100644 --- a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs @@ -14,12 +14,7 @@ use crate::SMALL_WORDS; #[inline] pub(super) fn eq_words_aligned(src: &[u64], other: &[u64], count: usize) -> bool { if count < SMALL_WORDS { - for i in 0..count { - if src[i] != other[i] { - return false; - } - } - return true; + return scalar::eq_words(src, other, count); } #[cfg(all( @@ -48,16 +43,12 @@ pub(super) fn eq_words_aligned(src: &[u64], other: &[u64], count: usize) -> bool } #[allow(unused)] - { - for i in 0..count { - if src[i] != other[i] { - return false; - } - } - true - } + scalar::eq_words(src, other, count) } +#[allow(unused)] +mod scalar; + #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] mod avx2; diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core/scalar.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/scalar.rs new file mode 100644 index 0000000..a7c502c --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/scalar.rs @@ -0,0 +1,8 @@ +pub(super) fn eq_words(src: &[u64], other: &[u64], count: usize) -> bool { + for i in 0..count { + if src[i] != other[i] { + return false; + } + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs index d56aa60..a83f76f 100644 --- a/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs @@ -16,14 +16,7 @@ pub(super) fn eq_words_unaligned(src: &[u64], other: &[u64], count: usize, shift debug_assert!(shift > 0 && shift < WORD_BITS); if count < SMALL_WORDS { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - } - return true; + return scalar::eq_words(src, other, count, shift); } #[cfg(all( @@ -52,18 +45,12 @@ pub(super) fn eq_words_unaligned(src: &[u64], other: &[u64], count: usize, shift } #[allow(unused)] - { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - } - true - } + scalar::eq_words(src, other, count, shift) } +#[allow(unused)] +mod scalar; + #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] mod avx2; diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core/scalar.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/scalar.rs new file mode 100644 index 0000000..eff8f02 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/scalar.rs @@ -0,0 +1,12 @@ +use crate::WORD_BITS; + +pub(super) fn eq_words(src: &[u64], other: &[u64], count: usize, shift: usize) -> bool { + for i in 0..count { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + } + true +} From 9b7c10a32edf05d6585fe0b8681b78ed4b2ba2ec Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 31 Aug 2026 14:18:38 +0000 Subject: [PATCH 44/79] refactor: split words_ord scalar backends --- .../words_ord/funcs_for_cmp_aligned_core.rs | 16 ++++--------- .../funcs_for_cmp_aligned_core/scalar.rs | 13 ++++++++++ .../words_ord/funcs_for_cmp_unaligned_core.rs | 24 ++++--------------- .../funcs_for_cmp_unaligned_core/scalar.rs | 22 +++++++++++++++++ 4 files changed, 43 insertions(+), 32 deletions(-) create mode 100644 src/traits/words_ord/funcs_for_cmp_aligned_core/scalar.rs create mode 100644 src/traits/words_ord/funcs_for_cmp_unaligned_core/scalar.rs diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs index d6925bc..b475daa 100644 --- a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs @@ -1,7 +1,6 @@ use core::cmp::Ordering; use crate::SMALL_WORDS; -use crate::traits::WordOrd; /// Returns `Some(Ordering)` if the first `count` aligned words of `src` /// and `other` differ, otherwise `None` (all equal). @@ -11,7 +10,7 @@ use crate::traits::WordOrd; #[inline] pub(super) fn cmp_aligned_words(src: &[u64], other: &[u64], count: usize) -> Option { if count < SMALL_WORDS { - return scalar_cmp_aligned(src, other, count); + return scalar::cmp_aligned(src, other, count); } #[cfg(all( @@ -43,18 +42,11 @@ pub(super) fn cmp_aligned_words(src: &[u64], other: &[u64], count: usize) -> Opt } #[allow(unreachable_code)] - scalar_cmp_aligned(src, other, count) + scalar::cmp_aligned(src, other, count) } -#[inline] -fn scalar_cmp_aligned(src: &[u64], other: &[u64], count: usize) -> Option { - for i in 0..count { - if src[i] != other[i] { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - } - None -} +#[allow(unused)] +mod scalar; // --------------------------------------------------------------------------- // AVX2 — 4 × u64 per iteration diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core/scalar.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core/scalar.rs new file mode 100644 index 0000000..28cb150 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core/scalar.rs @@ -0,0 +1,13 @@ +use core::cmp::Ordering; + +use crate::traits::WordOrd; + +#[inline] +pub(super) fn cmp_aligned(src: &[u64], other: &[u64], count: usize) -> Option { + for i in 0..count { + if src[i] != other[i] { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs index 35709bc..2563f7c 100644 --- a/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs @@ -6,7 +6,6 @@ use core::cmp::Ordering; -use crate::traits::WordOrd; use crate::{SMALL_WORDS, WORD_BITS}; /// Returns `Some(Ordering)` at the first differing word, or `None` when all @@ -21,7 +20,7 @@ pub(super) fn cmp_unaligned_words( debug_assert!(shift > 0 && shift < WORD_BITS); if count < SMALL_WORDS { - return scalar_cmp_unaligned(src, other, count, shift); + return scalar::cmp_unaligned(src, other, count, shift); } #[cfg(all( @@ -53,26 +52,11 @@ pub(super) fn cmp_unaligned_words( } #[allow(unreachable_code)] - scalar_cmp_unaligned(src, other, count, shift) + scalar::cmp_unaligned(src, other, count, shift) } -#[inline] -fn scalar_cmp_unaligned( - src: &[u64], - other: &[u64], - count: usize, - shift: usize, -) -> Option { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - let window = (w0 >> shift) | (w1 << (WORD_BITS - shift)); - if window != other[i] { - return Some(WordOrd::bitwise_cmp(window, other[i])); - } - } - None -} +#[allow(unused)] +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core/scalar.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/scalar.rs new file mode 100644 index 0000000..e4aa31c --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core/scalar.rs @@ -0,0 +1,22 @@ +use core::cmp::Ordering; + +use crate::WORD_BITS; +use crate::traits::WordOrd; + +#[inline] +pub(super) fn cmp_unaligned( + src: &[u64], + other: &[u64], + count: usize, + shift: usize, +) -> Option { + for i in 0..count { + let w0 = src[i]; + let w1 = src[i + 1]; + let window = (w0 >> shift) | (w1 << (WORD_BITS - shift)); + if window != other[i] { + return Some(WordOrd::bitwise_cmp(window, other[i])); + } + } + None +} From e9891998a4cc5fb80b4e0320662513c68d507c0c Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 31 Aug 2026 14:28:13 +0000 Subject: [PATCH 45/79] refactor: split contains scalar backend --- .../words_find/funcs_for_contains_core.rs | 43 ++----------------- .../funcs_for_contains_core/scalar.rs | 35 +++++++++++++++ 2 files changed, 39 insertions(+), 39 deletions(-) create mode 100644 src/traits/words_find/funcs_for_contains_core/scalar.rs diff --git a/src/traits/words_find/funcs_for_contains_core.rs b/src/traits/words_find/funcs_for_contains_core.rs index bef09dc..b57220d 100644 --- a/src/traits/words_find/funcs_for_contains_core.rs +++ b/src/traits/words_find/funcs_for_contains_core.rs @@ -40,7 +40,7 @@ where let word_limit = (max_word + 1).min(haystack.len()); if haystack.len() < SMALL_WORDS { - return scalar( + return scalar::scalar( haystack, needle_first, needle_mask, @@ -106,7 +106,7 @@ where } #[allow(unused)] - scalar( + scalar::scalar( haystack, needle_first, needle_mask, @@ -116,43 +116,8 @@ where ) } -// --------------------------------------------------------------------------- -// Scalar fallback -// --------------------------------------------------------------------------- - -/// Word-by-word scan: for each shift, check every word pair in -/// `[0, word_limit)` for a matching window. -fn scalar( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - word_limit: usize, - verify: &mut F, -) -> Option -where - F: FnMut(usize) -> bool, -{ - for shift in 0..WORD_BITS { - for i in 0..word_limit { - let pos = i * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - haystack[i] - } else { - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - None -} +#[allow(unused)] +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] diff --git a/src/traits/words_find/funcs_for_contains_core/scalar.rs b/src/traits/words_find/funcs_for_contains_core/scalar.rs new file mode 100644 index 0000000..b9307a8 --- /dev/null +++ b/src/traits/words_find/funcs_for_contains_core/scalar.rs @@ -0,0 +1,35 @@ +use crate::WORD_BITS; + +/// Word-by-word scan: for each shift, check every word pair in +/// `[0, word_limit)` for a matching window. +pub(super) fn scalar( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + word_limit: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + for shift in 0..WORD_BITS { + for i in 0..word_limit { + let pos = i * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + haystack[i] + } else { + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + None +} From ec60aaa26f46d852eaa84f7276b413e6d8ad9c53 Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 31 Aug 2026 14:34:01 +0000 Subject: [PATCH 46/79] refactor: split find scalar backend --- src/traits/words_find/funcs_for_find_core.rs | 40 ++----------------- .../words_find/funcs_for_find_core/scalar.rs | 32 +++++++++++++++ 2 files changed, 36 insertions(+), 36 deletions(-) create mode 100644 src/traits/words_find/funcs_for_find_core/scalar.rs diff --git a/src/traits/words_find/funcs_for_find_core.rs b/src/traits/words_find/funcs_for_find_core.rs index 7062ce6..b1e70d4 100644 --- a/src/traits/words_find/funcs_for_find_core.rs +++ b/src/traits/words_find/funcs_for_find_core.rs @@ -26,7 +26,7 @@ where let needle_mask = low_mask(needle_bit_len.min(WORD_BITS)); let last_start = haystack_bit_len - needle_bit_len; if haystack.len() < SMALL_WORDS { - return scalar_find(haystack, needle_first, needle_mask, last_start, verify); + return scalar::scalar_find(haystack, needle_first, needle_mask, last_start, verify); } #[cfg(all( @@ -58,43 +58,11 @@ where } #[allow(unused)] - scalar_find(haystack, needle_first, needle_mask, last_start, verify) + scalar::scalar_find(haystack, needle_first, needle_mask, last_start, verify) } -// --------------------------------------------------------------------------- -// Scalar -// --------------------------------------------------------------------------- - -fn scalar_find( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, -) -> Option -where - F: FnMut(usize) -> bool, -{ - for i in 0..haystack.len() { - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - for shift in 0..WORD_BITS { - let pos = i * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - None -} +#[allow(unused)] +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] diff --git a/src/traits/words_find/funcs_for_find_core/scalar.rs b/src/traits/words_find/funcs_for_find_core/scalar.rs new file mode 100644 index 0000000..55d40fe --- /dev/null +++ b/src/traits/words_find/funcs_for_find_core/scalar.rs @@ -0,0 +1,32 @@ +use crate::WORD_BITS; + +pub(super) fn scalar_find( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + for i in 0..haystack.len() { + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + for shift in 0..WORD_BITS { + let pos = i * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + None +} From 436a59eda52f07ebf8718c2fe06faa893f36835c Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 31 Aug 2026 14:41:16 +0000 Subject: [PATCH 47/79] refactor: split rfind scalar backend --- src/traits/words_find/funcs_for_rfind_core.rs | 44 ++----------------- .../words_find/funcs_for_rfind_core/scalar.rs | 36 +++++++++++++++ 2 files changed, 40 insertions(+), 40 deletions(-) create mode 100644 src/traits/words_find/funcs_for_rfind_core/scalar.rs diff --git a/src/traits/words_find/funcs_for_rfind_core.rs b/src/traits/words_find/funcs_for_rfind_core.rs index 75a1ae0..8abb87a 100644 --- a/src/traits/words_find/funcs_for_rfind_core.rs +++ b/src/traits/words_find/funcs_for_rfind_core.rs @@ -28,7 +28,7 @@ where let last_start = haystack_bit_len - needle_bit_len; if haystack.len() < SMALL_WORDS { - return scalar_rfind(haystack, needle_key, needle_mask, last_start, verify); + return scalar::scalar_rfind(haystack, needle_key, needle_mask, last_start, verify); } #[cfg(all( @@ -60,47 +60,11 @@ where } #[allow(unused)] - scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) + scalar::scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) } -// --------------------------------------------------------------------------- -// Scalar -// --------------------------------------------------------------------------- - -fn scalar_rfind( - haystack: &[u64], - needle_key: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, -) -> Option -where - F: FnMut(usize) -> bool, -{ - let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); - for i in (0..=start_word).rev() { - let base = i * WORD_BITS; - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - // Note: the SIMD backends compute max_shift differently — - // `WORD_BITS.min(last_start - base + 1)` — to process - // shifts in SIMD-sized chunks (2 or 4), relying on - // `pos <= last_start` to skip out-of-range positions. - let max_shift = (last_start - base).min(WORD_BITS - 1); - for shift in (0..=max_shift).rev() { - let pos = base + shift; - let window = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_key && verify(pos) { - return Some(pos); - } - } - } - None -} +#[allow(unused)] +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] diff --git a/src/traits/words_find/funcs_for_rfind_core/scalar.rs b/src/traits/words_find/funcs_for_rfind_core/scalar.rs new file mode 100644 index 0000000..0b60ee2 --- /dev/null +++ b/src/traits/words_find/funcs_for_rfind_core/scalar.rs @@ -0,0 +1,36 @@ +use crate::WORD_BITS; + +pub(super) fn scalar_rfind( + haystack: &[u64], + needle_key: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); + for i in (0..=start_word).rev() { + let base = i * WORD_BITS; + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + // Note: the SIMD backends compute max_shift differently — + // `WORD_BITS.min(last_start - base + 1)` — to process + // shifts in SIMD-sized chunks (2 or 4), relying on + // `pos <= last_start` to skip out-of-range positions. + let max_shift = (last_start - base).min(WORD_BITS - 1); + for shift in (0..=max_shift).rev() { + let pos = base + shift; + let window = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_key && verify(pos) { + return Some(pos); + } + } + } + None +} From 3396aca391697c931ba9e2e835b236fc03b39127 Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 31 Aug 2026 14:48:39 +0000 Subject: [PATCH 48/79] docs: remove obsolete architecture notes --- CLAUDE.md | 66 ------------------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index fa0dd9a..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,66 +0,0 @@ -# bit-string - -A `no_std` Rust crate providing a compact owned bit string type with editing, matching, and bitwise operations. - -通用约定参见父级：`../CLAUDE.md` - -## Architecture - -``` -src/ - lib.rs # #![no_std], re-exports BitString and errors - bit_string.rs # BitString struct definition (Box<[u64]> + len) - errors.rs # ParseBitStringError, BitStringLenMismatch - funcs_for_share.rs # Shared low-level helpers: bit_at, set_bit, read/write_chunk, - # mask_unused_bits, last_word_mask, word_len, etc. - impls_for_construction.rs # new, try_from, from_bool_iter, from_words, zeros, repeat - impls_for_access.rs # get, len, is_empty, to_string - impls_for_editing.rs # push, pop, insert, remove, set, extend, truncate, slice, etc. - impls_for_matching.rs # bits_equal_at + submodule declarations - impls_for_matching/ - impls_for_matches_at.rs # matches_at, starts_with, ends_with - impls_for_matches_at/ - funcs_for_starts_with_core.rs # SIMD starts_with word equality - funcs_for_ends_with_core.rs # SIMD ends_with (aligned + unaligned) - impls_for_find.rs # contains, find, rfind - impls_for_find/ - funcs_for_contains_core.rs # SIMD find_any_candidate (shift-outer) - funcs_for_find_core.rs # SIMD find_first_word (word-outer) - funcs_for_rfind_core.rs # SIMD find_last_word (word-reverse) - impls_for_strip.rs # strip_prefix, strip_suffix - impls_for_iter.rs # Iterator impl (yields bools) - impls_for_fmt.rs # Display, Debug (outputs "1010…" string) - impls_for_bit_ops/ - impls_for_not.rs # not, not_assign, not_into - impls_for_count_ones.rs # count_ones, count_zeros - impls_for_shl.rs # shl, shl_assign, shl_into (left shift with zero fill) - funcs_for_binary_core.rs # Generic binary-op dispatch: AND, OR, XOR - impls_for_and.rs # and, and_assign, and_into - impls_for_or.rs # or, or_assign, or_into - impls_for_xor.rs # xor, xor_assign, xor_into - impls_for_shr.rs # shr, shr_assign, shr_into (right shift with zero fill) -``` - -### SIMD Backend Dispatch Pattern - -Each bit operation follows the same multi-backend architecture: - -1. **Public API** methods (`shl`, `shl_assign`, `shl_into`) delegate to a `funcs_*_core` module. -2. The **core module** provides `owned()` (allocate new) and `assign()` (in-place) entry points. -3. A **`dispatch`** function uses `#[cfg]` gates to select the best available SIMD backend at compile time: - - `avx2` (x86/x86_64, 256-bit, 4×u64 lanes) - - `sse2` (x86/x86_64, 128-bit, 2×u64 lanes) — gated with `not(target_feature = "avx2")` - - `neon` (aarch64, 128-bit, 2×u64 lanes) - - `scalar` — fallback, supports all targets -4. Each backend lives in its own `mod` annotated with `#[target_feature(enable = "...")]`. -5. **Backend equivalence tests** run the scalar backend as oracle and assert every available SIMD backend produces identical output for a matrix of inputs and amounts. - -### Core Data Structure - -`BitString` stores bits in a `Box<[u64]>` array. The logical bit length (`len: usize`) is tracked separately. The last `u64` word is masked via `last_word_mask` to zero bits beyond `len`. Internal word size: `WORD_BITS = u64::BITS = 64`. - -## Key Dependencies - -- `int-interval` — interval type for editing operations (slice, remove, replace) -- `codspeed-divan-compat` — benchmarking framework -- `bitvec_simd` — dev-dependency for benchmark comparisons From 089fac0939219c7601ebe98137c1546d2c89f309 Mon Sep 17 00:00:00 2001 From: juncheng Date: Mon, 31 Aug 2026 14:54:33 +0000 Subject: [PATCH 49/79] refactor: move bool construction impls to parent --- src/bit_string/impls_for_construction.rs | 2 ++ .../impls_for_construction/funcs_for_pack_bools_core.rs | 3 --- .../impls_for_from_bool_iter.rs | 2 +- .../impls_for_from_bool_iter/tests_for_from_bool_iter.rs | 0 .../impls_for_from_bool_slice.rs | 4 ++-- 5 files changed, 5 insertions(+), 6 deletions(-) rename src/bit_string/impls_for_construction/{funcs_for_pack_bools_core => }/impls_for_from_bool_iter.rs (90%) rename src/bit_string/impls_for_construction/{funcs_for_pack_bools_core => }/impls_for_from_bool_iter/tests_for_from_bool_iter.rs (100%) rename src/bit_string/impls_for_construction/{funcs_for_pack_bools_core => }/impls_for_from_bool_slice.rs (85%) diff --git a/src/bit_string/impls_for_construction.rs b/src/bit_string/impls_for_construction.rs index 1b0fc74..6571ad5 100644 --- a/src/bit_string/impls_for_construction.rs +++ b/src/bit_string/impls_for_construction.rs @@ -1,4 +1,6 @@ mod funcs_for_pack_bools_core; +mod impls_for_from_bool_iter; +mod impls_for_from_bool_slice; mod impls_for_from_str; mod impls_for_from_words; mod impls_for_new; diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs index 42edde1..7969596 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs @@ -84,8 +84,5 @@ mod sse2; #[cfg(target_arch = "aarch64")] mod neon; -mod impls_for_from_bool_iter; -mod impls_for_from_bool_slice; - #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs b/src/bit_string/impls_for_construction/impls_for_from_bool_iter.rs similarity index 90% rename from src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs rename to src/bit_string/impls_for_construction/impls_for_from_bool_iter.rs index 0a60129..bc3e0fa 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_bool_iter.rs @@ -13,7 +13,7 @@ impl BitString { let bit_len = bytes.len(); let src = bytes.as_ptr(); Self { - words: super::bools_core(src, bit_len), + words: super::funcs_for_pack_bools_core::bools_core(src, bit_len), bit_len, } } diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter/tests_for_from_bool_iter.rs b/src/bit_string/impls_for_construction/impls_for_from_bool_iter/tests_for_from_bool_iter.rs similarity index 100% rename from src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter/tests_for_from_bool_iter.rs rename to src/bit_string/impls_for_construction/impls_for_from_bool_iter/tests_for_from_bool_iter.rs diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_slice.rs b/src/bit_string/impls_for_construction/impls_for_from_bool_slice.rs similarity index 85% rename from src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_slice.rs rename to src/bit_string/impls_for_construction/impls_for_from_bool_slice.rs index 53aed80..b284912 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_slice.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_bool_slice.rs @@ -10,7 +10,7 @@ impl From<&[bool]> for BitString { let src = values.as_ptr() as *const u8; let len = values.len(); Self { - words: super::bools_core(src, len), + words: super::funcs_for_pack_bools_core::bools_core(src, len), bit_len: len, } } @@ -25,7 +25,7 @@ impl From<[bool; N]> for BitString { // - Valid bool values are 0x00 (false) or 0x01 (true). let src = values.as_ptr() as *const u8; Self { - words: super::bools_core(src, N), + words: super::funcs_for_pack_bools_core::bools_core(src, N), bit_len: N, } } From dd21158788fe28e6dfb860bf6656d20cb15f4568 Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 1 Sep 2026 13:32:48 +0000 Subject: [PATCH 50/79] refactor: flatten words scan ends module --- .../words_scan/{funcs_for_ends/mod.rs => funcs_for_ends.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/traits/words_scan/{funcs_for_ends/mod.rs => funcs_for_ends.rs} (100%) diff --git a/src/traits/words_scan/funcs_for_ends/mod.rs b/src/traits/words_scan/funcs_for_ends.rs similarity index 100% rename from src/traits/words_scan/funcs_for_ends/mod.rs rename to src/traits/words_scan/funcs_for_ends.rs From 6db8c3f7bb8c933fd5bb0b6eea8262d37e56a8e2 Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 1 Sep 2026 13:33:11 +0000 Subject: [PATCH 51/79] refactor: extract shifted-copy scalar backend --- .../bits_copied/funcs_for_copy_words_shifted_core.rs | 11 +++++------ .../funcs_for_copy_words_shifted_core/scalar.rs | 9 +++++++++ .../tests_for_backend_equivalence.rs | 10 +--------- 3 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/scalar.rs diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs index ce241e5..d019c6d 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs @@ -24,9 +24,7 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi debug_assert!(shift > 0 && shift < WORD_BITS); if count < SMALL_WORDS { - for i in 0..count { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - } + scalar::copy_words_shifted(dst, src, count, shift); return; } @@ -60,9 +58,7 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi #[allow(unused)] { - for i in 0..count { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - } + scalar::copy_words_shifted(dst, src, count, shift); } } @@ -70,6 +66,9 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi // AVX2 — 4 words per iteration // --------------------------------------------------------------------------- +#[allow(unused)] +mod scalar; + #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] mod avx2; diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/scalar.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/scalar.rs new file mode 100644 index 0000000..6e8853a --- /dev/null +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/scalar.rs @@ -0,0 +1,9 @@ +use crate::WORD_BITS; + +/// Scalar backend for shifted 64-bit windows. +#[inline] +pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shift: usize) { + for i in 0..count { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + } +} diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs index dcfb3f0..cf04373 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs @@ -12,14 +12,6 @@ fn config() -> ProptestConfig { } } -fn scalar_copy(dst: &mut [u64], src: &[u64], count: usize, shift: usize) { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - dst[i] = (w0 >> shift) | (w1 << (64 - shift)); - } -} - proptest! { #![proptest_config(config())] @@ -34,7 +26,7 @@ proptest! { let mut dst_scalar = vec![0u64; count + extra]; super::copy_words_shifted(&mut dst_simd, &src, count, shift); - scalar_copy(&mut dst_scalar, &src, count, shift); + super::scalar::copy_words_shifted(&mut dst_scalar, &src, count, shift); assert_eq!(&dst_simd[..count], &dst_scalar[..count]); } From 9646d6312ba956d80b80eb12a82edd1af79aa8dc Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 1 Sep 2026 13:35:31 +0000 Subject: [PATCH 52/79] docs: align repository documentation with current APIs --- README.md | 51 ++++++++++--------- src/consts_for_bits.rs | 5 +- src/traits/word_ord.rs | 2 +- src/traits/words_eq.rs | 22 ++++---- .../funcs_for_contains_core/avx2.rs | 2 +- .../tests_for_backend_equivalence.rs | 2 +- .../words_find/funcs_for_find_core/sse41.rs | 2 +- .../words_find/funcs_for_rfind_core/sse41.rs | 2 +- .../tests_for_backend_equivalence.rs | 2 +- src/traits/words_ord.rs | 14 ++--- .../words_ord/funcs_for_cmp_aligned_core.rs | 2 +- src/traits/words_scan/funcs_for_count_ones.rs | 4 +- 12 files changed, 58 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index ebf2e78..2801608 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/gh/jcfangc/bit-string) [![Coverage](https://codecov.io/gh/jcfangc/bit-string/branch/main/graph/badge.svg)](https://codecov.io/gh/jcfangc/bit-string) -A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero-copy view, with construction, editing, matching, comparison, and bitwise operations — accelerated by compile-time-selected SIMD backends (AVX2, SSSE3, NEON). +A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero-copy view, with construction, editing, matching, comparison, and bitwise operations — accelerated by compile-time-selected SIMD backends (AVX2, SSE2, SSE4.1, SSSE3, NEON). ## Quick start @@ -14,9 +14,9 @@ A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero- use bit_string::BitString; // Parse from a binary string -let bits = BitString::try_from("1010_0011").unwrap(); +let bits = BitString::try_from("10100011").unwrap(); assert_eq!(bits.to_string(), "10100011"); -assert_eq!(bits.len(), 8); +assert_eq!(bits.bit_len(), 8); assert_eq!(bits.count_ones(), 4); // Build programmatically @@ -38,7 +38,7 @@ assert_eq!((!a).to_string(), "0101"); | Type | Role | Size | `Copy` | |------|------|------|--------| -| `BitString` | Owned, `Box<[u64]>` backing | 4×usize | No | +| `BitString` | Owned, `Vec` backing | 4×usize | No | | `BitStr<'bs>` | Zero-copy borrowed view | 3×usize | **Yes** | Bits are packed little-endian into `u64` words. Unused high bits in the last word are always zero. @@ -52,7 +52,7 @@ use bit_string::BitString; // From a binary string literal let a = BitString::try_from("0101").unwrap(); // "0101" -let b = BitString::try_from("0101_1110").unwrap(); // "01011110" (underscores ignored) +let b = BitString::try_from("01011110").unwrap(); // "01011110" // Pre-allocated let z = BitString::zeros(100); // 100 zero bits @@ -70,7 +70,7 @@ let v: BitString = (0..8).map(|i| i % 2 == 0).collect(); use bit_string::BitString; use int_interval::UsizeCO; -let bits = BitString::try_from("1100_1010").unwrap(); +let bits = BitString::try_from("11001010").unwrap(); // Full view let view = bits.as_bit_str(); // &BitStr = "11001010" @@ -78,7 +78,7 @@ assert_eq!(view.bit_len(), 8); // Sub-slice — zero-copy, O(1) let sub = view.slice(UsizeCO::try_new(2, 6).unwrap()); // "0010" -assert!(sub.starts_with(&BitString::try_from("00").unwrap().as_bit_str())); +assert!(sub.starts_with_str(BitString::try_from("00").unwrap().as_bit_str())); // Convert back to owned let owned = sub.to_bit_string(); // BitString = "0010" @@ -103,7 +103,7 @@ assert_eq!(bits.trailing_ones(), 2); // ends with "11" // Bit-level matching let pattern = BitString::try_from("10").unwrap(); -assert!(bits.starts_with(pattern.as_bit_str())); +assert!(bits.starts_with_str(pattern.as_bit_str())); ``` ### Editing @@ -125,29 +125,27 @@ bits.split_off(2); // → BitString "01", bits = "10" // Range operations use int_interval::UsizeCO; let interval = UsizeCO::try_new(1, 3).unwrap(); -bits.replace_interval(interval, &BitString::ones(2)); // "111" -bits.remove(UsizeCO::try_new(0, 2).unwrap()); // "1" +let replaced = bits.replace_interval(interval, &BitString::ones(2)); // "111" +let drained = replaced.drain_interval(UsizeCO::try_new(0, 2).unwrap()); // "1" ``` ### Matching & searching ```rust -let haystack = BitString::try_from("0010_1100").unwrap(); +let haystack = BitString::try_from("00101100").unwrap(); let needle = BitString::try_from("01").unwrap(); -let np = needle.as_bit_str(); - // Fixed-end checks -assert!(haystack.ends_with(np)); -assert!(!haystack.starts_with(np)); +assert!(haystack.ends_with_string(&needle)); +assert!(!haystack.starts_with_string(&needle)); // Substring search -assert!(haystack.contains(np)); -assert_eq!(haystack.find(np), Some(1)); // "01" starts at index 1 -assert_eq!(haystack.rfind(np), Some(5)); // last "01" at index 5 +assert!(haystack.contains_string(&needle)); +assert_eq!(haystack.find_string(&needle), Some(1)); // "01" starts at index 1 +assert_eq!(haystack.rfind_string(&needle), Some(5)); // last "01" at index 5 // Strip let s = BitString::try_from("00010100").unwrap(); -let stripped = s.strip_prefix(&BitString::try_from("00").unwrap().as_bit_str()); +let stripped = s.strip_prefix_str(BitString::try_from("00").unwrap().as_bit_str()); assert_eq!(stripped.unwrap().to_string(), "010100"); ``` @@ -157,14 +155,14 @@ assert_eq!(stripped.unwrap().to_string(), "010100"); let x = BitString::try_from("1010").unwrap(); let y = BitString::try_from("0110").unwrap(); -// Consuming (`_into`) — reuses allocation -let z = x.and_into(&y).unwrap(); // "0010" -let z = y.or_into(&x).unwrap(); // "1110" -let z = y.xor_into(&x).unwrap(); // "1100" +// Non-mutating operations — return a new BitString +let z = x.and(&y).unwrap(); // "0010" +let z = y.or(&x).unwrap(); // "1110" +let z = y.xor(&x).unwrap(); // "1100" // In-place (`_assign`) let mut w = x.clone(); -w.and_assign(&y); // w = "0010" +w.and_assign(&y).unwrap(); // w = "0010" // Shift let s = BitString::try_from("1001").unwrap(); @@ -205,10 +203,13 @@ The crate selects the fastest SIMD backend enabled for the compilation target: | Backend | Target | Width | |---------|--------|-------| | AVX2 | x86 / x86_64 | 256-bit (4×u64) | -| SSSE3 | x86 / x86_64 | 128-bit (2×u64) | +| SSE2 / SSE4.1 / SSSE3 | x86 / x86_64 | 128-bit (2×u64) | | NEON | aarch64 | 128-bit (2×u64) | | Scalar | all targets | fallback | +The selected backend is operation-dependent and is determined by the +compile-time target features. + There is no runtime CPU detection. For maximum local performance, compile for the host CPU by copying the example config: ```bash diff --git a/src/consts_for_bits.rs b/src/consts_for_bits.rs index 48af95b..eedf4e3 100644 --- a/src/consts_for_bits.rs +++ b/src/consts_for_bits.rs @@ -1,8 +1,9 @@ /// Number of bits in a `u64` — the word size for `BitString` backing storage. pub(crate) const WORD_BITS: usize = u64::BITS as usize; -/// Below this many full words, scalar loops beat SIMD dispatch overhead. -/// Must match each backend's `LANES`. +/// Below this many full words, scalar loops beat the selected SIMD dispatch +/// overhead. The value is selected per target-feature configuration; it is +/// not a universal requirement for every accelerator in the crate. #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx2" diff --git a/src/traits/word_ord.rs b/src/traits/word_ord.rs index c18bbad..2d92b41 100644 --- a/src/traits/word_ord.rs +++ b/src/traits/word_ord.rs @@ -2,7 +2,7 @@ use core::cmp::Ordering; /// Lexicographic comparison of two `u64` values (LSB-first bit order). /// -/// This is the word-level primitive used by [`BitsOrd`](super::BitsOrd) +/// This is the word-level primitive used by [`WordsOrd`](super::WordsOrd) /// to resolve ordering once the first differing word has been found. pub(crate) trait WordOrd { fn bitwise_cmp(self, other: Self) -> Ordering; diff --git a/src/traits/words_eq.rs b/src/traits/words_eq.rs index f654922..8903e71 100644 --- a/src/traits/words_eq.rs +++ b/src/traits/words_eq.rs @@ -1,22 +1,26 @@ /// Word-level equality comparison on `[u64]` backing storage. /// -/// Accepts a bit-level `offset` and handles word slicing and intra-word -/// shift internally: -/// - `offset % WORD_BITS == 0` — word-aligned, uses [`funcs_for_eq_words_aligned_core`]. -/// - `offset % WORD_BITS != 0` — unaligned shifted-window, uses [`funcs_for_eq_words_unaligned_core`]. +/// The caller pre-trims both slices to the relevant backing words and passes +/// the haystack's intra-word shift separately: +/// - `haystack_shift == 0` — word-aligned, uses [`funcs_for_eq_words_aligned_core`]. +/// - `haystack_shift != 0` — unaligned shifted-window, uses [`funcs_for_eq_words_unaligned_core`]. /// -/// Short inputs fall back to scalar in both backends. +/// This trait does not perform bit-level slicing or derive backing-word +/// offsets; those are handled by the higher-level view implementation. +/// +/// Short inputs fall back to scalar before SIMD dispatch. pub(crate) trait WordsEq { - /// Returns `true` if `other` matches `self` starting at `offset` bits. + /// Returns `true` if `other` matches the first `full_words` words of `self`. /// /// `count` is the number of full `u64` words to compare (computed from - /// the needle bit length). The intra-word shift and word slicing are - /// derived from `offset` internally. + /// the needle bit length). The caller supplies the pre-trimmed slices and + /// the haystack's intra-word shift. /// `self` is pre-trimmed haystack `words[base..]`. /// `needle` is always word-aligned (pre-trimmed by the caller). /// `full_words` is `needle_bit_len / WORD_BITS` — how many /// complete u64 words to compare. - /// `haystack_shift` is `original_offset % WORD_BITS`. + /// `haystack_shift` is the haystack slice's physical start modulo + /// `WORD_BITS`. /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is /// guaranteed and the aligned backend is used unconditionally. /// When it is `false`, no alignment guarantee is made. diff --git a/src/traits/words_find/funcs_for_contains_core/avx2.rs b/src/traits/words_find/funcs_for_contains_core/avx2.rs index a4b7d40..c4fa0b6 100644 --- a/src/traits/words_find/funcs_for_contains_core/avx2.rs +++ b/src/traits/words_find/funcs_for_contains_core/avx2.rs @@ -13,7 +13,7 @@ use core::arch::x86_64::{ const LANES: usize = 4; -/// AVX2 backend: same as SSE2 but with 4-lane (256-bit) vectors. +/// AVX2 backend: same as SSE4.1 but with 4-lane (256-bit) vectors. #[target_feature(enable = "avx2")] pub(super) unsafe fn find_any( haystack: &[u64], diff --git a/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs b/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs index edd03b1..50798d3 100644 --- a/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs @@ -1,6 +1,6 @@ //! Verify that `find_any_candidate` produces correct candidate positions //! against brute-force, implicitly validating whichever SIMD backend is -//! active (SSE2, AVX2, or NEON) against the scalar oracle. +//! active (SSE4.1, AVX2, or NEON) against the scalar oracle. use proptest::collection::vec; use proptest::prelude::*; diff --git a/src/traits/words_find/funcs_for_find_core/sse41.rs b/src/traits/words_find/funcs_for_find_core/sse41.rs index cf9db33..8c7fe03 100644 --- a/src/traits/words_find/funcs_for_find_core/sse41.rs +++ b/src/traits/words_find/funcs_for_find_core/sse41.rs @@ -1,5 +1,5 @@ // --------------------------------------------------------------------------- -// SSE2 — same loop as scalar but checks 2 consecutive shifts at once +// SSE4.1 — same loop as scalar but checks 2 consecutive shifts at once // --------------------------------------------------------------------------- use super::*; diff --git a/src/traits/words_find/funcs_for_rfind_core/sse41.rs b/src/traits/words_find/funcs_for_rfind_core/sse41.rs index e334bc7..ff09522 100644 --- a/src/traits/words_find/funcs_for_rfind_core/sse41.rs +++ b/src/traits/words_find/funcs_for_rfind_core/sse41.rs @@ -1,5 +1,5 @@ // --------------------------------------------------------------------------- -// SSE2 — 2 consecutive shifts at once, reverse +// SSE4.1 — 2 consecutive shifts at once, reverse // --------------------------------------------------------------------------- use super::*; diff --git a/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs b/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs index 7b84a25..c4b7079 100644 --- a/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs @@ -1,6 +1,6 @@ //! Verify that `find_last_word` produces correct rightmost matches //! against brute-force, implicitly validating whichever SIMD backend is -//! active (SSE2, AVX2, or NEON) against the scalar oracle. +//! active (SSE4.1, AVX2, or NEON) against the scalar oracle. use proptest::collection::vec; use proptest::prelude::*; diff --git a/src/traits/words_ord.rs b/src/traits/words_ord.rs index 97e7bf9..bf1b9ca 100644 --- a/src/traits/words_ord.rs +++ b/src/traits/words_ord.rs @@ -6,21 +6,21 @@ use core::cmp::Ordering; /// Returns `Some(Ordering)` at the first differing word, or `None` when all /// `count` words are identical. /// -/// `other` must be word-aligned. `self` may have an intra-word `offset` -/// (0 = word-aligned), in which case each logical word is reconstructed as -/// a shifted window `(self[i] >> shift) | (self[i+1] << (64-shift))`. +/// `other` is word-aligned. `self` may have a non-zero physical start shift, +/// in which case each logical word is reconstructed as a shifted window +/// `(self[i] >> shift) | (self[i+1] << (64-shift))`. /// /// Both paths dispatch to SIMD backends when available: /// - AVX2 (x86/x86_64, 4×u64 per iteration) -/// - SSE2 (x86/x86_64, 2×u64 per iteration) +/// - SSE4.1 (x86/x86_64, 2×u64 per iteration) /// - NEON (aarch64, 2×u64 per iteration) /// -/// Short inputs fall back to scalar in all backends. +/// Short inputs fall back to scalar before SIMD dispatch. pub(crate) trait WordsOrd { - /// `self` is pre-trimmed haystack `words[base..]`. + /// `self` is the pre-trimmed haystack slice `words[base..]`. /// `needle` is always word-aligned (pre-trimmed by the caller). /// `full_words` is the number of complete u64 words to compare. - /// `haystack_shift` is the intra-word offset within the first word. + /// `haystack_shift` is the physical start modulo `WORD_BITS`. /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is /// guaranteed and the aligned backend is used unconditionally. /// When it is `false`, no alignment guarantee is made. diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs index b475daa..665f996 100644 --- a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs @@ -57,7 +57,7 @@ mod scalar; mod avx2; // --------------------------------------------------------------------------- -// SSE2 — 2 × u64 per iteration +// SSE4.1 — 2 × u64 per iteration // --------------------------------------------------------------------------- #[allow(unused)] diff --git a/src/traits/words_scan/funcs_for_count_ones.rs b/src/traits/words_scan/funcs_for_count_ones.rs index 13c244b..c9ee69e 100644 --- a/src/traits/words_scan/funcs_for_count_ones.rs +++ b/src/traits/words_scan/funcs_for_count_ones.rs @@ -44,8 +44,8 @@ fn count_full_words(words: &[u64]) -> usize { /// - `src` must be valid for reads of `len` initialized `u64` values. #[inline] unsafe fn dispatch(src: *const u64, len: usize) -> usize { - // Small inputs: skip SIMD setup overhead, go straight to scalar popcnt. - // Threshold equals the backend's LANES count. + // Small inputs: skip SIMD setup overhead, go straight to scalar popcount. + // The threshold is selected by the active target-feature configuration. #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), From 2a954e5d08346dd00d6d543c25eadef6378f412c Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 1 Sep 2026 13:35:45 +0000 Subject: [PATCH 53/79] chore: identify Python tooling project --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 69e1a2e..9aa5d11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [project] -name = "bit-string" +name = "bit-string-dev-tools" version = "0.1.0" requires-python = ">=3.11" dependencies = [] + +[tool.uv] +package = false From c8b4373a93b38a9e7d628216cf5fe1b88116f2d8 Mon Sep 17 00:00:00 2001 From: juncheng Date: Tue, 1 Sep 2026 14:13:30 +0000 Subject: [PATCH 54/79] chore: close bit-string cleanup --- README.md | 8 ++++---- .../funcs_for_copy_words_shifted_core.rs | 2 +- src/traits/words_eq.rs | 14 ++++---------- src/traits/words_ord.rs | 8 ++++---- uv.lock | 2 +- 5 files changed, 14 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 2801608..24b8ae7 100644 --- a/README.md +++ b/README.md @@ -135,13 +135,13 @@ let drained = replaced.drain_interval(UsizeCO::try_new(0, 2).unwrap()); // "1" let haystack = BitString::try_from("00101100").unwrap(); let needle = BitString::try_from("01").unwrap(); // Fixed-end checks -assert!(haystack.ends_with_string(&needle)); +assert!(!haystack.ends_with_string(&needle)); assert!(!haystack.starts_with_string(&needle)); // Substring search assert!(haystack.contains_string(&needle)); assert_eq!(haystack.find_string(&needle), Some(1)); // "01" starts at index 1 -assert_eq!(haystack.rfind_string(&needle), Some(5)); // last "01" at index 5 +assert_eq!(haystack.rfind_string(&needle), Some(3)); // last "01" starts at index 3 // Strip let s = BitString::try_from("00010100").unwrap(); @@ -166,8 +166,8 @@ w.and_assign(&y).unwrap(); // w = "0010" // Shift let s = BitString::try_from("1001").unwrap(); -assert_eq!(s.shl(2).to_string(), "0100"); // left shift, zero-fill -assert_eq!(s.shr(1).to_string(), "0100"); // right shift, zero-fill +assert_eq!(s.shl(2).to_string(), "0010"); // left shift, zero-fill +assert_eq!(s.shr(1).to_string(), "0010"); // right shift, zero-fill // Not assert_eq!((!s).to_string(), "0110"); diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs index d019c6d..22a2cd1 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs @@ -63,7 +63,7 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi } // --------------------------------------------------------------------------- -// AVX2 — 4 words per iteration +// Backends // --------------------------------------------------------------------------- #[allow(unused)] diff --git a/src/traits/words_eq.rs b/src/traits/words_eq.rs index 8903e71..c0740c5 100644 --- a/src/traits/words_eq.rs +++ b/src/traits/words_eq.rs @@ -10,17 +10,11 @@ /// /// Short inputs fall back to scalar before SIMD dispatch. pub(crate) trait WordsEq { - /// Returns `true` if `other` matches the first `full_words` words of `self`. + /// Compares `full_words` logical haystack words with `needle`. /// - /// `count` is the number of full `u64` words to compare (computed from - /// the needle bit length). The caller supplies the pre-trimmed slices and - /// the haystack's intra-word shift. - /// `self` is pre-trimmed haystack `words[base..]`. - /// `needle` is always word-aligned (pre-trimmed by the caller). - /// `full_words` is `needle_bit_len / WORD_BITS` — how many - /// complete u64 words to compare. - /// `haystack_shift` is the haystack slice's physical start modulo - /// `WORD_BITS`. + /// `self` is the pre-trimmed haystack backing slice. + /// `needle` is word-aligned and pre-trimmed by the caller. + /// `haystack_shift` is the physical start offset within `self[0]`. /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is /// guaranteed and the aligned backend is used unconditionally. /// When it is `false`, no alignment guarantee is made. diff --git a/src/traits/words_ord.rs b/src/traits/words_ord.rs index bf1b9ca..8863a09 100644 --- a/src/traits/words_ord.rs +++ b/src/traits/words_ord.rs @@ -2,11 +2,11 @@ use core::cmp::Ordering; /// Word-level lexicographic comparison on `[u64]` backing storage. /// -/// Compares words from index 0 upward (LSB-first bit order within each word). -/// Returns `Some(Ordering)` at the first differing word, or `None` when all -/// `count` words are identical. +/// Compares `full_words` logical haystack words with `needle` from index 0 +/// upward (LSB-first bit order within each word). Returns `Some(Ordering)` at +/// the first differing word, or `None` when all logical words are identical. /// -/// `other` is word-aligned. `self` may have a non-zero physical start shift, +/// `needle` is word-aligned. `self` may have a non-zero physical start shift, /// in which case each logical word is reconstructed as a shifted window /// `(self[i] >> shift) | (self[i+1] << (64-shift))`. /// diff --git a/uv.lock b/uv.lock index 47ff850..2d9e4f8 100644 --- a/uv.lock +++ b/uv.lock @@ -3,6 +3,6 @@ revision = 3 requires-python = ">=3.11" [[package]] -name = "bit-string" +name = "bit-string-dev-tools" version = "0.1.0" source = { virtual = "." } From 03c09ee685b1f38c955deac64f0890271357a224 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 2 Sep 2026 14:08:34 +0000 Subject: [PATCH 55/79] refactor packed string length derivation --- src/packed_string.rs | 1 - src/packed_string/impls_for_access.rs | 11 ++++------- .../impls_for_access/tests_for_access.rs | 4 ++-- src/packed_string/impls_for_construction.rs | 6 ++++-- src/packed_string/impls_for_editing.rs | 12 ++++-------- src/packed_string/impls_for_eq.rs | 2 +- src/packed_string/impls_for_iter.rs | 2 +- src/packed_string/tests_for_support.rs | 4 ++-- 8 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/packed_string.rs b/src/packed_string.rs index 9223639..8c328f4 100644 --- a/src/packed_string.rs +++ b/src/packed_string.rs @@ -15,7 +15,6 @@ where C: PackedChar, { bits: BitString, - char_len: usize, marker: PhantomData C>, } diff --git a/src/packed_string/impls_for_access.rs b/src/packed_string/impls_for_access.rs index d855edc..3ee54d3 100644 --- a/src/packed_string/impls_for_access.rs +++ b/src/packed_string/impls_for_access.rs @@ -7,12 +7,12 @@ where /// Number of packed characters, not number of bits. #[inline] pub fn char_len(&self) -> usize { - self.char_len + self.bits.bit_len() / usize::from(BITS) } #[inline] pub fn is_empty(&self) -> bool { - self.char_len == 0 + self.bits.is_empty() } #[inline] @@ -27,7 +27,7 @@ where #[inline] pub fn get(&self, index: usize) -> Option { - if index >= self.char_len { + if index >= self.char_len() { return None; } Some( @@ -43,14 +43,11 @@ where #[inline] pub fn last(&self) -> Option { - self.char_len.checked_sub(1).and_then(|i| self.get(i)) + self.char_len().checked_sub(1).and_then(|i| self.get(i)) } #[inline] fn code_at(&self, index: usize) -> u8 { - if BITS == 0 { - return 0; - } let start = index * usize::from(BITS); (self.bits.get_chunk(start) & u64::from(code_mask::())) as u8 } diff --git a/src/packed_string/impls_for_access/tests_for_access.rs b/src/packed_string/impls_for_access/tests_for_access.rs index 232261b..be4b598 100644 --- a/src/packed_string/impls_for_access/tests_for_access.rs +++ b/src/packed_string/impls_for_access/tests_for_access.rs @@ -9,9 +9,9 @@ fn get_first_and_last_decode_values() { } #[test] -fn one_value_type_needs_no_payload_bits() { +fn one_value_type_still_has_one_bit_per_character() { let value = OnlyString::from_chars([Only::Value; 100]); assert_eq!(value.char_len(), 100); - assert_eq!(value.bits().bit_len(), 0); + assert_eq!(value.bits().bit_len(), 100); assert_eq!(value.get(99), Some(Only::Value)); } diff --git a/src/packed_string/impls_for_construction.rs b/src/packed_string/impls_for_construction.rs index c81ba1f..386f4ae 100644 --- a/src/packed_string/impls_for_construction.rs +++ b/src/packed_string/impls_for_construction.rs @@ -4,14 +4,16 @@ impl PackedString where C: PackedChar, { - const VALID_WIDTH: () = assert!(BITS <= 8, "packed character width must not exceed 8"); + const VALID_WIDTH: () = assert!( + BITS > 0 && BITS <= 8, + "packed character width must be between 1 and 8" + ); #[inline] pub fn new() -> Self { let () = Self::VALID_WIDTH; Self { bits: BitString::new(), - char_len: 0, marker: PhantomData, } } diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 194de85..7972448 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -7,20 +7,18 @@ where /// Appends the character's code directly to the payload. pub fn push(&mut self, character: C) { let code = character.code(); - let new_len = self - .char_len + self.char_len() .checked_add(1) .expect("packed string length overflow"); for shift in 0..BITS { self.bits.push((code >> shift) & 1 != 0); } - self.char_len = new_len; } pub fn pop(&mut self) -> Option { let character = self.last()?; - self.char_len -= 1; - self.bits.truncate(self.char_len * usize::from(BITS)); + self.bits + .truncate((self.char_len() - 1) * usize::from(BITS)); Some(character) } @@ -31,15 +29,13 @@ where } pub fn truncate(&mut self, new_len: usize) { - if new_len >= self.char_len { + if new_len >= self.char_len() { return; } - self.char_len = new_len; self.bits.truncate(new_len * usize::from(BITS)); } pub fn clear(&mut self) { - self.char_len = 0; self.bits.clear(); } } diff --git a/src/packed_string/impls_for_eq.rs b/src/packed_string/impls_for_eq.rs index 6340e13..f13d84f 100644 --- a/src/packed_string/impls_for_eq.rs +++ b/src/packed_string/impls_for_eq.rs @@ -5,7 +5,7 @@ where C: PackedChar, { fn eq(&self, other: &Self) -> bool { - self.char_len == other.char_len && self.bits == other.bits + self.bits == other.bits } } diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index 317ff7d..0cd7900 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -11,7 +11,7 @@ where Iter { string: self, front: 0, - back: self.char_len, + back: self.char_len(), } } diff --git a/src/packed_string/tests_for_support.rs b/src/packed_string/tests_for_support.rs index 251aba1..b5dfe4a 100644 --- a/src/packed_string/tests_for_support.rs +++ b/src/packed_string/tests_for_support.rs @@ -25,10 +25,10 @@ impl fmt::Display for Letter { #[derive(Clone, Copy, Debug, PartialEq, Eq, PackedChar)] #[repr(u8)] -#[packed(bits = 0)] +#[packed(bits = 1)] pub(super) enum Only { Value = 0, } pub(super) type LetterString = super::PackedString; -pub(super) type OnlyString = super::PackedString; +pub(super) type OnlyString = super::PackedString; From c84ce03db1c5661df6fe98a9b2ac02f1ab1ae075 Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 2 Sep 2026 14:11:39 +0000 Subject: [PATCH 56/79] validate packed string bit payloads --- .../impls_for_from_bits.rs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/packed_string/impls_for_construction/impls_for_from_bits.rs b/src/packed_string/impls_for_construction/impls_for_from_bits.rs index 9c3aae1..4352a9a 100644 --- a/src/packed_string/impls_for_construction/impls_for_from_bits.rs +++ b/src/packed_string/impls_for_construction/impls_for_from_bits.rs @@ -5,15 +5,23 @@ where C: PackedChar, { /// Validates and adopts an already packed bit payload. - pub fn from_bits(_bits: BitString, _char_len: usize) -> Option { - unimplemented!("PackedString::from_bits") + pub fn from_bits(bits: BitString) -> Option { + let () = Self::VALID_WIDTH; + let bits_per_char = usize::from(BITS); + if bits.bit_len() % bits_per_char != 0 { + return None; + } + let code_mask = u64::from(code_mask::()); + for start in (0..bits.bit_len()).step_by(bits_per_char) { + C::from_code((bits.get_chunk(start) & code_mask) as u8)?; + } + Some(Self { + bits, + marker: PhantomData, + }) } pub fn into_bits(self) -> BitString { unimplemented!("PackedString::into_bits") } - - pub fn into_parts(self) -> (BitString, usize) { - unimplemented!("PackedString::into_parts") - } } From ee9b6115b49207a2e56d81b805a7e065073e03de Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 2 Sep 2026 14:17:43 +0000 Subject: [PATCH 57/79] refactor packed string views around bitstr --- src/packed_str.rs | 6 ++-- src/packed_str/impls_for_access.rs | 29 ++++++++++--------- src/packed_str/impls_for_conversion.rs | 7 +++-- src/packed_str/impls_for_iter.rs | 26 +++++++++++++---- src/packed_str/impls_for_slice.rs | 40 ++++++++++++++++++++++---- src/packed_string.rs | 2 +- src/packed_string/funcs_for_code.rs | 2 +- 7 files changed, 79 insertions(+), 33 deletions(-) diff --git a/src/packed_str.rs b/src/packed_str.rs index 5efc08a..193b407 100644 --- a/src/packed_str.rs +++ b/src/packed_str.rs @@ -2,7 +2,7 @@ use core::marker::PhantomData; -use crate::{PackedString, traits::PackedChar}; +use crate::{BitStr, PackedString, traits::PackedChar}; /// A character-aligned borrowed view into a [`PackedString`]. #[derive(Clone, Copy)] @@ -11,9 +11,7 @@ pub struct PackedStr<'ps, C, const BITS: u8> where C: PackedChar, { - source: &'ps PackedString, - char_start: usize, - char_len: usize, + bits: BitStr<'ps>, marker: PhantomData C>, } diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs index 10a0d60..02f1e59 100644 --- a/src/packed_str/impls_for_access.rs +++ b/src/packed_str/impls_for_access.rs @@ -4,31 +4,32 @@ impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> where C: PackedChar, { - pub fn source(&self) -> &'ps PackedString { - unimplemented!("PackedStr::source") - } - - pub fn char_start(&self) -> usize { - unimplemented!("PackedStr::char_start") - } - pub fn char_len(&self) -> usize { - unimplemented!("PackedStr::char_len") + self.bits.bit_len() / usize::from(BITS) } pub fn is_empty(&self) -> bool { - unimplemented!("PackedStr::is_empty") + self.bits.bit_len() == 0 } - pub fn get(&self, _index: usize) -> Option { - unimplemented!("PackedStr::get") + pub fn get(&self, index: usize) -> Option { + if index >= self.char_len() { + return None; + } + C::from_code( + (self.bits.get_chunk(index * usize::from(BITS)) + & u64::from(crate::packed_string::funcs_for_code::code_mask::())) + as u8, + ) } pub fn first(&self) -> Option { - unimplemented!("PackedStr::first") + self.get(0) } pub fn last(&self) -> Option { - unimplemented!("PackedStr::last") + self.char_len() + .checked_sub(1) + .and_then(|index| self.get(index)) } } diff --git a/src/packed_str/impls_for_conversion.rs b/src/packed_str/impls_for_conversion.rs index b6651ba..d024c31 100644 --- a/src/packed_str/impls_for_conversion.rs +++ b/src/packed_str/impls_for_conversion.rs @@ -5,7 +5,7 @@ where C: PackedChar, { pub fn to_packed_string(&self) -> PackedString { - unimplemented!("PackedStr::to_packed_string") + PackedString::from_bits(self.bits.to_bit_string()).expect("PackedStr invariant violated") } } @@ -14,6 +14,9 @@ where C: PackedChar, { pub fn as_packed_str(&self) -> PackedStr<'_, C, BITS> { - unimplemented!("PackedString::as_packed_str") + PackedStr { + bits: self.bits().as_bit_str(), + marker: core::marker::PhantomData, + } } } diff --git a/src/packed_str/impls_for_iter.rs b/src/packed_str/impls_for_iter.rs index 4183586..8eb20d6 100644 --- a/src/packed_str/impls_for_iter.rs +++ b/src/packed_str/impls_for_iter.rs @@ -7,7 +7,11 @@ where C: PackedChar, { pub fn iter(&self) -> Iter<'ps, C, BITS> { - unimplemented!("PackedStr::iter") + Iter { + view: *self, + front: 0, + back: self.char_len(), + } } } @@ -28,11 +32,18 @@ where type Item = C; fn next(&mut self) -> Option { - unimplemented!("PackedStr::Iter::next") + if self.front == self.back { + None + } else { + let c = self.view.get(self.front); + self.front += 1; + c + } } fn size_hint(&self) -> (usize, Option) { - unimplemented!("PackedStr::Iter::size_hint") + let len = self.back - self.front; + (len, Some(len)) } } @@ -41,7 +52,12 @@ where C: PackedChar, { fn next_back(&mut self) -> Option { - unimplemented!("PackedStr::Iter::next_back") + if self.front == self.back { + None + } else { + self.back -= 1; + self.view.get(self.back) + } } } @@ -56,6 +72,6 @@ where type IntoIter = Iter<'ps, C, BITS>; fn into_iter(self) -> Self::IntoIter { - unimplemented!("PackedStr::into_iter") + self.iter() } } diff --git a/src/packed_str/impls_for_slice.rs b/src/packed_str/impls_for_slice.rs index 2c623d8..65a9c67 100644 --- a/src/packed_str/impls_for_slice.rs +++ b/src/packed_str/impls_for_slice.rs @@ -6,15 +6,43 @@ impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> where C: PackedChar, { - pub fn slice(&self, _interval: UsizeCO) -> Self { - unimplemented!("PackedStr::slice") + pub fn slice(&self, interval: UsizeCO) -> Self { + let bits = usize::from(BITS); + let start = interval.start().min(self.char_len()).saturating_mul(bits); + let end = interval + .end_excl() + .min(self.char_len()) + .saturating_mul(bits); + if end <= start { + return Self { + bits: self.bits.slice_until(0), + marker: core::marker::PhantomData, + }; + } + let interval = UsizeCO::checked_from_start_len(start, end - start).unwrap(); + Self { + bits: self.bits.slice(interval), + marker: core::marker::PhantomData, + } } - pub fn slice_from(&self, _start: usize) -> Self { - unimplemented!("PackedStr::slice_from") + pub fn slice_from(&self, start: usize) -> Self { + if start >= self.char_len() { + return Self { + bits: self.bits.slice_from(self.bits.bit_len()), + marker: core::marker::PhantomData, + }; + } + self.slice(UsizeCO::checked_from_start_len(start, self.char_len() - start).unwrap()) } - pub fn slice_until(&self, _end: usize) -> Self { - unimplemented!("PackedStr::slice_until") + pub fn slice_until(&self, end: usize) -> Self { + if end == 0 { + return Self { + bits: self.bits.slice_until(0), + marker: core::marker::PhantomData, + }; + } + self.slice(UsizeCO::checked_from_start_len(0, end).unwrap()) } } diff --git a/src/packed_string.rs b/src/packed_string.rs index 8c328f4..08185a6 100644 --- a/src/packed_string.rs +++ b/src/packed_string.rs @@ -18,7 +18,7 @@ where marker: PhantomData C>, } -mod funcs_for_code; +pub(crate) mod funcs_for_code; mod impls_for_access; mod impls_for_construction; mod impls_for_editing; diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 2c85441..93b5647 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -1,7 +1,7 @@ use super::*; #[inline] -pub(super) const fn code_mask() -> u8 { +pub(crate) const fn code_mask() -> u8 { match BITS { 0 => 0, 1..=7 => (1u8 << BITS) - 1, From dc05db1a8d57169599bc5c99130dbd4187467cce Mon Sep 17 00:00:00 2001 From: juncheng Date: Wed, 2 Sep 2026 14:24:49 +0000 Subject: [PATCH 58/79] add packed attribute macro --- derive/src/funcs_for_expand.rs | 7 +++-- .../src/funcs_for_expand/tests_for_expand.rs | 14 ++++++++- derive/src/lib.rs | 31 ++++++++++++++++++- src/lib.rs | 2 +- tests/packed_char_derive.rs | 16 +++++++++- 5 files changed, 64 insertions(+), 6 deletions(-) diff --git a/derive/src/funcs_for_expand.rs b/derive/src/funcs_for_expand.rs index 7674b05..8eb3eb9 100644 --- a/derive/src/funcs_for_expand.rs +++ b/derive/src/funcs_for_expand.rs @@ -8,8 +8,11 @@ use syn::{Data, DeriveInput, Error, Expr, Fields, Lit, LitInt, Result}; pub(super) fn expand(input: DeriveInput) -> Result { require_repr_u8(&input)?; let (bits, bits_span) = parse_bits(&input)?; - if bits > 8 { - return Err(Error::new(bits_span, "packed bit width must not exceed 8")); + if !(1..=8).contains(&bits) { + return Err(Error::new( + bits_span, + "packed bit width must be between 1 and 8", + )); } let Data::Enum(data) = &input.data else { diff --git a/derive/src/funcs_for_expand/tests_for_expand.rs b/derive/src/funcs_for_expand/tests_for_expand.rs index 47fb61a..b8e08be 100644 --- a/derive/src/funcs_for_expand/tests_for_expand.rs +++ b/derive/src/funcs_for_expand/tests_for_expand.rs @@ -51,7 +51,19 @@ fn rejects_widths_above_u8_capacity() { #[packed(bits = 9)] enum Letter { A = 0 } }) - .contains("must not exceed 8") + .contains("must be between 1 and 8") + ); +} + +#[test] +fn rejects_zero_width() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 0)] + enum Letter { A = 0 } + }) + .contains("must be between 1 and 8") ); } diff --git a/derive/src/lib.rs b/derive/src/lib.rs index ee3e10b..6a4dde6 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,5 +1,6 @@ use proc_macro::TokenStream; -use syn::{DeriveInput, Error, parse_macro_input}; +use quote::quote; +use syn::{Attribute, DeriveInput, Error, parse_macro_input, parse_quote}; mod funcs_for_expand; @@ -17,3 +18,31 @@ pub fn derive_packed_char(input: TokenStream) -> TokenStream { .unwrap_or_else(Error::into_compile_error) .into() } + +/// Marks a fieldless enum as a packed alphabet and supplies its implementation. +/// +/// This is the preferred spelling for new code: +/// `#[packed(bits = 2)] enum Symbol { ... }`. +#[proc_macro_attribute] +pub fn packed(args: TokenStream, input: TokenStream) -> TokenStream { + let mut item = parse_macro_input!(input as DeriveInput); + let bits: proc_macro2::TokenStream = args.into(); + + let packed: Attribute = parse_quote!(#[packed(#bits)]); + item.attrs.push(packed); + if !item + .attrs + .iter() + .any(|attribute| attribute.path().is_ident("repr")) + { + item.attrs.push(parse_quote!(#[repr(u8)])); + } + + let implementation = match expand(item.clone()) { + Ok(tokens) => tokens, + Err(error) => error.into_compile_error(), + }; + item.attrs + .retain(|attribute| !attribute.path().is_ident("packed")); + quote!(#item #implementation).into() +} diff --git a/src/lib.rs b/src/lib.rs index 58223ad..eb50405 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,7 @@ mod packed_string; pub use bit_str::BitStr; pub use bit_string::BitString; -pub use bit_string_derive::PackedChar; +pub use bit_string_derive::{PackedChar, packed}; pub use packed_str::PackedStr; pub use packed_string::PackedString; diff --git a/tests/packed_char_derive.rs b/tests/packed_char_derive.rs index e8c5dbf..d2daf21 100644 --- a/tests/packed_char_derive.rs +++ b/tests/packed_char_derive.rs @@ -1,4 +1,11 @@ -use bit_string::{PackedString, traits::PackedChar}; +use bit_string::{PackedString, packed, traits::PackedChar}; + +#[packed(bits = 1)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PackedSymbol { + Zero = 0, + One = 1, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, PackedChar)] #[repr(u8)] @@ -16,3 +23,10 @@ fn derived_encoding_round_trips_through_packed_string() { assert_eq!(string.get(1), Some(Symbol::Zero)); assert_eq!(string.get(2), Some(Symbol::One)); } + +#[test] +fn attribute_macro_generates_packed_char_impl() { + let string = + PackedString::::from_chars([PackedSymbol::One, PackedSymbol::Zero]); + assert_eq!(string.get(0), Some(PackedSymbol::One)); +} From 6cccb1d57006c22154c6a8d63cb7950f59059d79 Mon Sep 17 00:00:00 2001 From: juncheng Date: Fri, 4 Sep 2026 13:45:47 +0000 Subject: [PATCH 59/79] implement packed matching and ordering --- src/packed_str/impls_for_eq.rs | 4 +- src/packed_str/impls_for_matching.rs | 48 ++++++++++++------- src/packed_str/impls_for_ord.rs | 8 ++-- .../impls_for_from_bits.rs | 2 +- src/packed_string/impls_for_hash.rs | 4 +- src/packed_string/impls_for_iter.rs | 2 +- src/packed_string/impls_for_matching.rs | 46 +++++++++++------- src/packed_string/impls_for_ord.rs | 12 ++--- 8 files changed, 78 insertions(+), 48 deletions(-) diff --git a/src/packed_str/impls_for_eq.rs b/src/packed_str/impls_for_eq.rs index d90b37a..12ded5f 100644 --- a/src/packed_str/impls_for_eq.rs +++ b/src/packed_str/impls_for_eq.rs @@ -4,8 +4,8 @@ impl PartialEq for PackedStr<'_, C, BITS> where C: PackedChar, { - fn eq(&self, _other: &Self) -> bool { - unimplemented!("PackedStr::eq") + fn eq(&self, other: &Self) -> bool { + self.bits == other.bits } } diff --git a/src/packed_str/impls_for_matching.rs b/src/packed_str/impls_for_matching.rs index eef34ef..6a165eb 100644 --- a/src/packed_str/impls_for_matching.rs +++ b/src/packed_str/impls_for_matching.rs @@ -4,35 +4,51 @@ impl PackedStr<'_, C, BITS> where C: PackedChar, { - pub fn matches_at(&self, _index: usize, _pattern: Self) -> bool { - unimplemented!("PackedStr::matches_at") + pub fn matches_at(&self, index: usize, pattern: Self) -> bool { + let Some(bit_index) = index.checked_mul(usize::from(BITS)) else { + return false; + }; + if index > self.char_len() { + return false; + } + self.bits.matches_at_str(bit_index, pattern.bits) } - pub fn starts_with(&self, _prefix: Self) -> bool { - unimplemented!("PackedStr::starts_with") + pub fn starts_with(&self, prefix: Self) -> bool { + self.bits.starts_with_str(prefix.bits) } - pub fn ends_with(&self, _suffix: Self) -> bool { - unimplemented!("PackedStr::ends_with") + pub fn ends_with(&self, suffix: Self) -> bool { + self.bits.ends_with_str(suffix.bits) } - pub fn contains(&self, _needle: Self) -> bool { - unimplemented!("PackedStr::contains") + pub fn contains(&self, needle: Self) -> bool { + (0..=self.char_len().saturating_sub(needle.char_len())) + .any(|index| self.matches_at(index, needle)) } - pub fn find(&self, _needle: Self) -> Option { - unimplemented!("PackedStr::find") + pub fn find(&self, needle: Self) -> Option { + (0..=self.char_len().saturating_sub(needle.char_len())) + .find(|&index| self.matches_at(index, needle)) } - pub fn rfind(&self, _needle: Self) -> Option { - unimplemented!("PackedStr::rfind") + pub fn rfind(&self, needle: Self) -> Option { + (0..=self.char_len().saturating_sub(needle.char_len())) + .rev() + .find(|&index| self.matches_at(index, needle)) } - pub fn strip_prefix(&self, _prefix: Self) -> Option { - unimplemented!("PackedStr::strip_prefix") + pub fn strip_prefix(&self, prefix: Self) -> Option { + self.bits.strip_prefix_str(prefix.bits).map(|bits| Self { + bits, + marker: core::marker::PhantomData, + }) } - pub fn strip_suffix(&self, _suffix: Self) -> Option { - unimplemented!("PackedStr::strip_suffix") + pub fn strip_suffix(&self, suffix: Self) -> Option { + self.bits.strip_suffix_str(suffix.bits).map(|bits| Self { + bits, + marker: core::marker::PhantomData, + }) } } diff --git a/src/packed_str/impls_for_ord.rs b/src/packed_str/impls_for_ord.rs index a1d491f..ca7eaa4 100644 --- a/src/packed_str/impls_for_ord.rs +++ b/src/packed_str/impls_for_ord.rs @@ -6,8 +6,8 @@ impl PartialOrd for PackedStr<'_, C, BITS> where C: PackedChar, { - fn partial_cmp(&self, _other: &Self) -> Option { - unimplemented!("PackedStr::partial_cmp") + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.bits.cmp_str(&other.bits)) } } @@ -15,7 +15,7 @@ impl Ord for PackedStr<'_, C, BITS> where C: PackedChar, { - fn cmp(&self, _other: &Self) -> Ordering { - unimplemented!("PackedStr::cmp") + fn cmp(&self, other: &Self) -> Ordering { + self.bits.cmp_str(&other.bits) } } diff --git a/src/packed_string/impls_for_construction/impls_for_from_bits.rs b/src/packed_string/impls_for_construction/impls_for_from_bits.rs index 4352a9a..3507be2 100644 --- a/src/packed_string/impls_for_construction/impls_for_from_bits.rs +++ b/src/packed_string/impls_for_construction/impls_for_from_bits.rs @@ -22,6 +22,6 @@ where } pub fn into_bits(self) -> BitString { - unimplemented!("PackedString::into_bits") + self.bits } } diff --git a/src/packed_string/impls_for_hash.rs b/src/packed_string/impls_for_hash.rs index 6027697..315776a 100644 --- a/src/packed_string/impls_for_hash.rs +++ b/src/packed_string/impls_for_hash.rs @@ -6,7 +6,7 @@ impl Hash for PackedString where C: PackedChar, { - fn hash(&self, _state: &mut H) { - unimplemented!("PackedString::hash") + fn hash(&self, state: &mut H) { + self.bits.hash(state) } } diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index 0cd7900..48273e3 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -17,7 +17,7 @@ where /// Collects the decoded characters into a vector. pub fn to_vec(&self) -> alloc::vec::Vec { - unimplemented!("PackedString::to_vec") + self.iter().collect() } } diff --git a/src/packed_string/impls_for_matching.rs b/src/packed_string/impls_for_matching.rs index afe461c..379e63c 100644 --- a/src/packed_string/impls_for_matching.rs +++ b/src/packed_string/impls_for_matching.rs @@ -4,35 +4,49 @@ impl PackedString where C: PackedChar, { - pub fn matches_at(&self, _index: usize, _pattern: &Self) -> bool { - unimplemented!("PackedString::matches_at") + pub fn matches_at(&self, index: usize, pattern: &Self) -> bool { + let Some(bit_index) = index.checked_mul(usize::from(BITS)) else { + return false; + }; + if index > self.char_len() { + return false; + } + self.bits.matches_at_string(bit_index, &pattern.bits) } - pub fn starts_with(&self, _prefix: &Self) -> bool { - unimplemented!("PackedString::starts_with") + pub fn starts_with(&self, prefix: &Self) -> bool { + self.bits.starts_with_string(&prefix.bits) } - pub fn ends_with(&self, _suffix: &Self) -> bool { - unimplemented!("PackedString::ends_with") + pub fn ends_with(&self, suffix: &Self) -> bool { + self.bits.ends_with_string(&suffix.bits) } - pub fn contains(&self, _needle: &Self) -> bool { - unimplemented!("PackedString::contains") + pub fn contains(&self, needle: &Self) -> bool { + (0..=self.char_len().saturating_sub(needle.char_len())) + .any(|index| self.matches_at(index, needle)) } - pub fn find(&self, _needle: &Self) -> Option { - unimplemented!("PackedString::find") + pub fn find(&self, needle: &Self) -> Option { + (0..=self.char_len().saturating_sub(needle.char_len())) + .find(|&index| self.matches_at(index, needle)) } - pub fn rfind(&self, _needle: &Self) -> Option { - unimplemented!("PackedString::rfind") + pub fn rfind(&self, needle: &Self) -> Option { + (0..=self.char_len().saturating_sub(needle.char_len())) + .rev() + .find(|&index| self.matches_at(index, needle)) } - pub fn strip_prefix(&self, _prefix: &Self) -> Option { - unimplemented!("PackedString::strip_prefix") + pub fn strip_prefix(&self, prefix: &Self) -> Option { + self.bits + .strip_prefix_string(&prefix.bits) + .and_then(Self::from_bits) } - pub fn strip_suffix(&self, _suffix: &Self) -> Option { - unimplemented!("PackedString::strip_suffix") + pub fn strip_suffix(&self, suffix: &Self) -> Option { + self.bits + .strip_suffix_string(&suffix.bits) + .and_then(Self::from_bits) } } diff --git a/src/packed_string/impls_for_ord.rs b/src/packed_string/impls_for_ord.rs index 18880e6..fe9bc81 100644 --- a/src/packed_string/impls_for_ord.rs +++ b/src/packed_string/impls_for_ord.rs @@ -7,8 +7,8 @@ where C: PackedChar, { /// Compares characters lexicographically by their packed code values. - pub fn cmp_string(&self, _other: &Self) -> Ordering { - unimplemented!("PackedString::cmp_string") + pub fn cmp_string(&self, other: &Self) -> Ordering { + self.bits.as_bit_str().cmp_str(&other.bits.as_bit_str()) } } @@ -16,8 +16,8 @@ impl PartialOrd for PackedString where C: PackedChar, { - fn partial_cmp(&self, _other: &Self) -> Option { - unimplemented!("PackedString::partial_cmp") + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp_string(other)) } } @@ -25,7 +25,7 @@ impl Ord for PackedString where C: PackedChar, { - fn cmp(&self, _other: &Self) -> Ordering { - unimplemented!("PackedString::cmp") + fn cmp(&self, other: &Self) -> Ordering { + self.cmp_string(other) } } From f0f67cea0a6b499c761387199084d6b8708f807e Mon Sep 17 00:00:00 2001 From: juncheng Date: Fri, 4 Sep 2026 13:54:24 +0000 Subject: [PATCH 60/79] implement packed construction and insertion --- .../impls_for_from_slice.rs | 8 +++--- .../impls_for_repeat.rs | 6 ++-- src/packed_string/impls_for_editing.rs | 5 ++-- .../impls_for_insert_remove.rs | 28 ++++++++++++++++--- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/packed_string/impls_for_construction/impls_for_from_slice.rs b/src/packed_string/impls_for_construction/impls_for_from_slice.rs index 91e3140..ceaa0ae 100644 --- a/src/packed_string/impls_for_construction/impls_for_from_slice.rs +++ b/src/packed_string/impls_for_construction/impls_for_from_slice.rs @@ -4,8 +4,8 @@ impl From<&[C]> for PackedString where C: PackedChar, { - fn from(_characters: &[C]) -> Self { - unimplemented!("PackedString::from(&[C])") + fn from(characters: &[C]) -> Self { + Self::from_chars(characters.iter().copied()) } } @@ -13,7 +13,7 @@ impl From<[C; N]> for PackedString where C: PackedChar, { - fn from(_characters: [C; N]) -> Self { - unimplemented!("PackedString::from([C; N])") + fn from(characters: [C; N]) -> Self { + Self::from_chars(characters) } } diff --git a/src/packed_string/impls_for_construction/impls_for_repeat.rs b/src/packed_string/impls_for_construction/impls_for_repeat.rs index fcbc7a2..4ba0b06 100644 --- a/src/packed_string/impls_for_construction/impls_for_repeat.rs +++ b/src/packed_string/impls_for_construction/impls_for_repeat.rs @@ -4,7 +4,9 @@ impl PackedString where C: PackedChar, { - pub fn repeat(_character: C, _char_len: usize) -> Self { - unimplemented!("PackedString::repeat") + pub fn repeat(character: C, char_len: usize) -> Self { + let mut result = Self::new(); + result.extend(core::iter::repeat_n(character, char_len)); + result } } diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 7972448..000f4f1 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -7,8 +7,9 @@ where /// Appends the character's code directly to the payload. pub fn push(&mut self, character: C) { let code = character.code(); - self.char_len() - .checked_add(1) + self.bits + .bit_len() + .checked_add(usize::from(BITS)) .expect("packed string length overflow"); for shift in 0..BITS { self.bits.push((code >> shift) & 1 != 0); diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs index 36156ab..7502f52 100644 --- a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs @@ -4,11 +4,31 @@ impl PackedString where C: PackedChar, { - pub fn insert(&mut self, _index: usize, _character: C) { - unimplemented!("PackedString::insert") + pub fn insert(&mut self, index: usize, character: C) { + if index > self.char_len() { + return; + } + self.bits + .bit_len() + .checked_add(usize::from(BITS)) + .expect("packed string length overflow"); + let start = index * usize::from(BITS); + for shift in (0..BITS).rev() { + self.bits + .insert(start, (character.code() >> shift) & 1 != 0); + } } - pub fn remove(&mut self, _index: usize) -> C { - unimplemented!("PackedString::remove") + pub fn remove(&mut self, index: usize) -> C { + let character = self.get(index).expect("packed string index out of bounds"); + let start = index * usize::from(BITS); + let mut code = 0; + for shift in 0..BITS { + if self.bits.remove(start) { + code |= 1 << shift; + } + } + debug_assert_eq!(code, character.code()); + character } } From 8cdcc95f0a05cad85246ea9e574b69ded6ab7c1e Mon Sep 17 00:00:00 2001 From: juncheng Date: Fri, 4 Sep 2026 14:00:53 +0000 Subject: [PATCH 61/79] implement packed concatenation and slicing --- .../impls_for_editing/impls_for_concat.rs | 14 +++++---- .../impls_for_editing/impls_for_slice.rs | 31 +++++++++++++++---- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/packed_string/impls_for_editing/impls_for_concat.rs b/src/packed_string/impls_for_editing/impls_for_concat.rs index 60f74b2..22220a4 100644 --- a/src/packed_string/impls_for_editing/impls_for_concat.rs +++ b/src/packed_string/impls_for_editing/impls_for_concat.rs @@ -4,15 +4,17 @@ impl PackedString where C: PackedChar, { - pub fn push_packed_string(&mut self, _other: &Self) { - unimplemented!("PackedString::push_packed_string") + pub fn push_packed_string(&mut self, other: &Self) { + self.bits.push_bit_string(&other.bits); } - pub fn insert_packed_string(&mut self, _index: usize, _other: &Self) { - unimplemented!("PackedString::insert_packed_string") + pub fn insert_packed_string(&mut self, index: usize, other: &Self) { + let start = index.min(self.char_len()) * usize::from(BITS); + self.bits.insert_bit_string(start, &other.bits); } - pub fn split_off(&mut self, _at: usize) -> Self { - unimplemented!("PackedString::split_off") + pub fn split_off(&mut self, at: usize) -> Self { + let at = at.min(self.char_len()) * usize::from(BITS); + Self::from_bits(self.bits.split_off(at)).expect("PackedString invariant violated") } } diff --git a/src/packed_string/impls_for_editing/impls_for_slice.rs b/src/packed_string/impls_for_editing/impls_for_slice.rs index d73eab6..2fcae50 100644 --- a/src/packed_string/impls_for_editing/impls_for_slice.rs +++ b/src/packed_string/impls_for_editing/impls_for_slice.rs @@ -6,15 +6,34 @@ impl PackedString where C: PackedChar, { - pub fn slice(&self, _interval: UsizeCO) -> Self { - unimplemented!("PackedString::slice") + pub fn slice(&self, interval: UsizeCO) -> Self { + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + if start == end { + return Self::new(); + } + let bits = usize::from(BITS); + Self::from_bits( + self.bits.slice( + UsizeCO::checked_from_start_len(start * bits, (end - start) * bits).unwrap(), + ), + ) + .expect("PackedString invariant violated") } - pub fn slice_from(&self, _start: usize) -> Self { - unimplemented!("PackedString::slice_from") + pub fn slice_from(&self, start: usize) -> Self { + if start >= self.char_len() { + return Self::new(); + } + self.slice( + UsizeCO::checked_from_start_len(start, self.char_len().saturating_sub(start)).unwrap(), + ) } - pub fn slice_until(&self, _end: usize) -> Self { - unimplemented!("PackedString::slice_until") + pub fn slice_until(&self, end: usize) -> Self { + if end == 0 { + return Self::new(); + } + self.slice(UsizeCO::checked_from_start_len(0, end).unwrap()) } } From e7eeddb90eb623a683926746b417d6a96e97dbfa Mon Sep 17 00:00:00 2001 From: juncheng Date: Fri, 4 Sep 2026 14:06:01 +0000 Subject: [PATCH 62/79] implement packed replacement and filtering --- .../impls_for_editing/impls_for_drain.rs | 17 ++++++--- .../impls_for_editing/impls_for_replace.rs | 36 ++++++++++++++----- .../impls_for_editing/impls_for_retain.rs | 5 +-- .../impls_for_editing/impls_for_reverse.rs | 4 +-- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/src/packed_string/impls_for_editing/impls_for_drain.rs b/src/packed_string/impls_for_editing/impls_for_drain.rs index c0cfedf..ba0e920 100644 --- a/src/packed_string/impls_for_editing/impls_for_drain.rs +++ b/src/packed_string/impls_for_editing/impls_for_drain.rs @@ -6,11 +6,20 @@ impl PackedString where C: PackedChar, { - pub fn drain_interval(&self, _interval: UsizeCO) -> Self { - unimplemented!("PackedString::drain_interval") + pub fn drain_interval(&self, interval: UsizeCO) -> Self { + let bits = usize::from(BITS); + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + if start == end { + return self.clone(); + } + let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; + Self::from_bits(self.bits.drain_interval(interval)) + .expect("PackedString invariant violated") } - pub fn drain_interval_assign(&mut self, _interval: UsizeCO) { - unimplemented!("PackedString::drain_interval_assign") + pub fn drain_interval_assign(&mut self, interval: UsizeCO) { + let result = self.drain_interval(interval); + *self = result; } } diff --git a/src/packed_string/impls_for_editing/impls_for_replace.rs b/src/packed_string/impls_for_editing/impls_for_replace.rs index ca90eec..78f25b2 100644 --- a/src/packed_string/impls_for_editing/impls_for_replace.rs +++ b/src/packed_string/impls_for_editing/impls_for_replace.rs @@ -6,19 +6,39 @@ impl PackedString where C: PackedChar, { - pub fn replace_interval(&self, _interval: UsizeCO, _replacement: &Self) -> Self { - unimplemented!("PackedString::replace_interval") + pub fn replace_interval(&self, interval: UsizeCO, replacement: &Self) -> Self { + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + self.replace_range(start, end, replacement) } - pub fn replace_interval_assign(&mut self, _interval: UsizeCO, _replacement: &Self) { - unimplemented!("PackedString::replace_interval_assign") + pub fn replace_interval_assign(&mut self, interval: UsizeCO, replacement: &Self) { + let result = self.replace_interval(interval, replacement); + *self = result; } - pub fn replace(&self, _start: usize, _replacement: &Self) -> Self { - unimplemented!("PackedString::replace") + pub fn replace(&self, start: usize, replacement: &Self) -> Self { + let start = start.min(self.char_len()); + let end = start + .saturating_add(replacement.char_len()) + .min(self.char_len()); + self.replace_range(start, end, replacement) } - pub fn replace_assign(&mut self, _start: usize, _replacement: &Self) { - unimplemented!("PackedString::replace_assign") + pub fn replace_assign(&mut self, start: usize, replacement: &Self) { + let result = self.replace(start, replacement); + *self = result; + } + + fn replace_range(&self, start: usize, end: usize, replacement: &Self) -> Self { + let bits = usize::from(BITS); + let result = self.bits.replace(start * bits, &replacement.bits); + Self::from_bits(if end > start { + let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; + self.bits.replace_interval(interval, &replacement.bits) + } else { + result + }) + .expect("PackedString invariant violated") } } diff --git a/src/packed_string/impls_for_editing/impls_for_retain.rs b/src/packed_string/impls_for_editing/impls_for_retain.rs index 2379321..fee913f 100644 --- a/src/packed_string/impls_for_editing/impls_for_retain.rs +++ b/src/packed_string/impls_for_editing/impls_for_retain.rs @@ -4,10 +4,11 @@ impl PackedString where C: PackedChar, { - pub fn retain(&mut self, _predicate: F) + pub fn retain(&mut self, mut predicate: F) where F: FnMut(C) -> bool, { - unimplemented!("PackedString::retain") + let retained = self.iter().filter(|&character| predicate(character)); + *self = Self::from_chars(retained); } } diff --git a/src/packed_string/impls_for_editing/impls_for_reverse.rs b/src/packed_string/impls_for_editing/impls_for_reverse.rs index b85a9b7..2a6743d 100644 --- a/src/packed_string/impls_for_editing/impls_for_reverse.rs +++ b/src/packed_string/impls_for_editing/impls_for_reverse.rs @@ -5,10 +5,10 @@ where C: PackedChar, { pub fn reverse(&self) -> Self { - unimplemented!("PackedString::reverse") + Self::from_chars(self.iter().rev()) } pub fn reverse_assign(&mut self) { - unimplemented!("PackedString::reverse_assign") + *self = self.reverse(); } } From 38e78f0b25e8bfc38693ddffebc40480c0d11a4b Mon Sep 17 00:00:00 2001 From: juncheng Date: Fri, 4 Sep 2026 14:18:43 +0000 Subject: [PATCH 63/79] consolidate packed macro API --- derive/Cargo.toml | 2 +- derive/src/funcs_for_expand.rs | 12 ++++----- derive/src/lib.rs | 15 +---------- src/lib.rs | 2 +- src/packed_string/tests_for_support.rs | 8 +++--- src/traits.rs | 1 - src/traits/packed_char.rs | 10 +++---- .../{packed_char_derive.rs => packed_char.rs} | 26 ++++++++++++++++--- 8 files changed, 40 insertions(+), 36 deletions(-) rename tests/{packed_char_derive.rs => packed_char.rs} (58%) diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 81ac59e..08ff9b5 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -2,7 +2,7 @@ name = "bit-string-derive" version = "0.5.0" edition = "2024" -description = "Derive macros for bit-string" +description = "Procedural macros for bit-string" repository = "https://github.com/jcfangc/bit-string" license = "MIT OR Apache-2.0" diff --git a/derive/src/funcs_for_expand.rs b/derive/src/funcs_for_expand.rs index 8eb3eb9..6afed9b 100644 --- a/derive/src/funcs_for_expand.rs +++ b/derive/src/funcs_for_expand.rs @@ -18,7 +18,7 @@ pub(super) fn expand(input: DeriveInput) -> Result { let Data::Enum(data) = &input.data else { return Err(Error::new_spanned( &input.ident, - "PackedChar can only be derived for enums", + "packed can only be applied to enums", )); }; @@ -30,20 +30,20 @@ pub(super) fn expand(input: DeriveInput) -> Result { if !matches!(variant.fields, Fields::Unit) { return Err(Error::new_spanned( &variant.fields, - "PackedChar variants must not contain fields", + "packed variants must not contain fields", )); } let Some((_, Expr::Lit(expression))) = &variant.discriminant else { return Err(Error::new_spanned( &variant.ident, - "PackedChar variants require an explicit integer discriminant", + "packed variants require an explicit integer discriminant", )); }; let Lit::Int(discriminant) = &expression.lit else { return Err(Error::new_spanned( &expression.lit, - "PackedChar discriminants must be integer literals", + "packed discriminants must be integer literals", )); }; let code = discriminant.base10_parse::()?; @@ -106,7 +106,7 @@ fn require_repr_u8(input: &DeriveInput) -> Result<()> { } else { Err(Error::new_spanned( &input.ident, - "PackedChar requires #[repr(u8)]", + "packed requires #[repr(u8)]", )) } } @@ -129,7 +129,7 @@ fn parse_bits(input: &DeriveInput) -> Result<(u8, Span)> { Ok(()) })?; } - bits.ok_or_else(|| Error::new_spanned(&input.ident, "PackedChar requires #[packed(bits = N)]")) + bits.ok_or_else(|| Error::new_spanned(&input.ident, "packed requires #[packed(bits = N)]")) } fn bit_string_path() -> TokenStream2 { diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 6a4dde6..242b1bc 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,24 +1,11 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{Attribute, DeriveInput, Error, parse_macro_input, parse_quote}; +use syn::{Attribute, DeriveInput, parse_macro_input, parse_quote}; mod funcs_for_expand; use funcs_for_expand::expand; -/// Derives a fixed-width packed encoding from explicit `#[repr(u8)]` enum -/// discriminants. -/// -/// The enum must use `#[packed(bits = N)]`, contain only fieldless variants, -/// and give every variant an integer-literal discriminant that fits in `N` -/// bits. -#[proc_macro_derive(PackedChar, attributes(packed))] -pub fn derive_packed_char(input: TokenStream) -> TokenStream { - expand(parse_macro_input!(input as DeriveInput)) - .unwrap_or_else(Error::into_compile_error) - .into() -} - /// Marks a fieldless enum as a packed alphabet and supplies its implementation. /// /// This is the preferred spelling for new code: diff --git a/src/lib.rs b/src/lib.rs index eb50405..277c560 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,7 @@ mod packed_string; pub use bit_str::BitStr; pub use bit_string::BitString; -pub use bit_string_derive::{PackedChar, packed}; +pub use bit_string_derive::packed; pub use packed_str::PackedStr; pub use packed_string::PackedString; diff --git a/src/packed_string/tests_for_support.rs b/src/packed_string/tests_for_support.rs index b5dfe4a..855145f 100644 --- a/src/packed_string/tests_for_support.rs +++ b/src/packed_string/tests_for_support.rs @@ -1,9 +1,8 @@ use core::fmt; -use super::PackedChar; +use crate::packed; -#[derive(Clone, Copy, Debug, PartialEq, Eq, PackedChar)] -#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[packed(bits = 2)] pub(super) enum Letter { A = 0b00, @@ -23,8 +22,7 @@ impl fmt::Display for Letter { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, PackedChar)] -#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[packed(bits = 1)] pub(super) enum Only { Value = 0, diff --git a/src/traits.rs b/src/traits.rs index 0b48cb0..c682bf3 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -7,7 +7,6 @@ pub(crate) mod words_find; pub(crate) mod words_ord; pub(crate) mod words_scan; -pub use bit_string_derive::PackedChar; pub use packed_char::PackedChar; pub(crate) use word_ord::*; pub(crate) use words_arith::*; diff --git a/src/traits/packed_char.rs b/src/traits/packed_char.rs index 547574a..c684277 100644 --- a/src/traits/packed_char.rs +++ b/src/traits/packed_char.rs @@ -1,7 +1,7 @@ /// A character with a fixed-width bit representation. /// -/// This trait is intended for fieldless `#[repr(u8)]` enums. Prefer deriving -/// it with `#[derive(PackedChar)]` and `#[packed(bits = N)]`; manual +/// This trait is intended for fieldless `#[repr(u8)]` enums. Use the +/// `#[packed(bits = N)]` attribute for that common case; manual /// implementations must satisfy all of the following invariants: /// /// - `BITS` does not exceed eight; @@ -9,9 +9,9 @@ /// - `from_code(code(value)) == Some(value)`; /// - each valid code identifies at most one value. /// -/// [`PackedString`](crate::PackedString) relies on these invariants. The derive -/// macro validates them at compile time and generates both directions from the -/// same discriminants. No `transmute` is used. +/// [`PackedString`](crate::PackedString) relies on these invariants. The packed +/// attribute validates them at compile time and generates both directions from +/// the same discriminants. No `transmute` is used. pub trait PackedChar: Copy + Eq { /// Returns the character's packed value. fn code(self) -> u8; diff --git a/tests/packed_char_derive.rs b/tests/packed_char.rs similarity index 58% rename from tests/packed_char_derive.rs rename to tests/packed_char.rs index d2daf21..581c364 100644 --- a/tests/packed_char_derive.rs +++ b/tests/packed_char.rs @@ -7,8 +7,7 @@ enum PackedSymbol { One = 1, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, PackedChar)] -#[repr(u8)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[packed(bits = 2)] enum Symbol { Zero = 0, @@ -17,13 +16,34 @@ enum Symbol { } #[test] -fn derived_encoding_round_trips_through_packed_string() { +fn packed_attribute_encoding_round_trips_through_packed_string() { let string = PackedString::::from_chars([Symbol::Two, Symbol::Zero, Symbol::One]); assert_eq!(string.get(0), Some(Symbol::Two)); assert_eq!(string.get(1), Some(Symbol::Zero)); assert_eq!(string.get(2), Some(Symbol::One)); } +#[derive(Clone, Copy, PartialEq, Eq)] +struct ManualSymbol; + +impl PackedChar<1> for ManualSymbol { + fn code(self) -> u8 { + 0 + } + + fn from_code(code: u8) -> Option { + (code == 0).then_some(Self) + } +} + +#[test] +fn manual_packed_char_implementations_remain_supported() { + assert_eq!( + PackedString::::from_chars([ManualSymbol]).char_len(), + 1 + ); +} + #[test] fn attribute_macro_generates_packed_char_impl() { let string = From 6c92eb042c9d51fee8414544806601f10d5e5ad5 Mon Sep 17 00:00:00 2001 From: juncheng Date: Fri, 4 Sep 2026 14:22:34 +0000 Subject: [PATCH 64/79] fix bitstr rustdoc link --- src/bit_string.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bit_string.rs b/src/bit_string.rs index 474e7e6..621c25a 100644 --- a/src/bit_string.rs +++ b/src/bit_string.rs @@ -23,7 +23,7 @@ impl BitString { &self.words } - /// Returns a zero-copy [`BitStr`] view of the entire bit string. + /// Returns a zero-copy [`crate::BitStr`] view of the entire bit string. #[inline] pub fn as_bit_str(&self) -> crate::BitStr<'_> { crate::BitStr { From 482ff8f3391c5a1e6199a47030c2d9cc21b18d6a Mon Sep 17 00:00:00 2001 From: juncheng Date: Fri, 4 Sep 2026 14:26:48 +0000 Subject: [PATCH 65/79] test packed character views and edits --- tests/packed_char.rs | 51 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/packed_char.rs b/tests/packed_char.rs index 581c364..3ad5164 100644 --- a/tests/packed_char.rs +++ b/tests/packed_char.rs @@ -1,4 +1,5 @@ -use bit_string::{PackedString, packed, traits::PackedChar}; +use bit_string::{BitString, PackedString, packed, traits::PackedChar}; +use int_interval::UsizeCO; #[packed(bits = 1)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -50,3 +51,51 @@ fn attribute_macro_generates_packed_char_impl() { PackedString::::from_chars([PackedSymbol::One, PackedSymbol::Zero]); assert_eq!(string.get(0), Some(PackedSymbol::One)); } + +#[test] +fn packed_views_and_editing_remain_character_aligned() { + let mut string = PackedString::::from_chars([ + Symbol::Zero, + Symbol::One, + Symbol::Two, + Symbol::One, + ]); + + let view = string.as_packed_str(); + assert_eq!(view.char_len(), 4); + assert_eq!( + view.slice(UsizeCO::checked_from_start_len(1, 2).unwrap()) + .get(0), + Some(Symbol::One) + ); + assert_eq!(view.find(view.slice_until(1)), Some(0)); + assert!(view.contains(view.slice(UsizeCO::checked_from_start_len(1, 1).unwrap()))); + + let haystack_string = + PackedString::::from_chars([Symbol::Zero, Symbol::Zero, Symbol::One]); + let needle_string = PackedString::::from_chars([Symbol::Two]); + let haystack = haystack_string.as_packed_str(); + let needle = needle_string.as_packed_str(); + assert!(!haystack.contains(needle)); + assert_eq!(haystack.find(needle), None); + assert_eq!(haystack.rfind(needle), None); + assert!(!haystack.matches_at(1, needle)); + + string.insert(2, Symbol::Zero); + assert_eq!(string.remove(2), Symbol::Zero); + assert_eq!( + string.reverse().to_vec(), + vec![Symbol::One, Symbol::Two, Symbol::One, Symbol::Zero] + ); + string.retain(|symbol| symbol != Symbol::Two); + assert_eq!( + string.to_vec(), + vec![Symbol::Zero, Symbol::One, Symbol::One] + ); +} + +#[test] +fn from_bits_rejects_misaligned_and_unknown_codes() { + assert!(PackedString::::from_bits(BitString::from_iter([true])).is_none()); + assert!(PackedString::::from_bits(BitString::from_iter([true, true])).is_none()); +} From 1d65fdc6e71805eb68540a76d5bc9e97e6d9df3c Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 00:46:09 +0000 Subject: [PATCH 66/79] fix packed string ordering semantics --- src/packed_str/impls_for_ord.rs | 24 ++++++++++++++++++++++-- src/packed_string/impls_for_ord.rs | 2 +- tests/packed_char.rs | 8 ++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/packed_str/impls_for_ord.rs b/src/packed_str/impls_for_ord.rs index ca7eaa4..b3fe22c 100644 --- a/src/packed_str/impls_for_ord.rs +++ b/src/packed_str/impls_for_ord.rs @@ -2,12 +2,32 @@ use core::cmp::Ordering; use super::*; +impl PackedStr<'_, C, BITS> +where + C: PackedChar, +{ + pub(crate) fn cmp_codes(&self, other: &Self) -> Ordering { + let mask = u64::from(crate::packed_string::funcs_for_code::code_mask::()); + let shared_len = self.char_len().min(other.char_len()); + for index in 0..shared_len { + let start = index * usize::from(BITS); + let lhs = self.bits.get_chunk(start) & mask; + let rhs = other.bits.get_chunk(start) & mask; + match lhs.cmp(&rhs) { + Ordering::Equal => {} + ordering => return ordering, + } + } + self.char_len().cmp(&other.char_len()) + } +} + impl PartialOrd for PackedStr<'_, C, BITS> where C: PackedChar, { fn partial_cmp(&self, other: &Self) -> Option { - Some(self.bits.cmp_str(&other.bits)) + Some(self.cmp_codes(other)) } } @@ -16,6 +36,6 @@ where C: PackedChar, { fn cmp(&self, other: &Self) -> Ordering { - self.bits.cmp_str(&other.bits) + self.cmp_codes(other) } } diff --git a/src/packed_string/impls_for_ord.rs b/src/packed_string/impls_for_ord.rs index fe9bc81..8e4b596 100644 --- a/src/packed_string/impls_for_ord.rs +++ b/src/packed_string/impls_for_ord.rs @@ -8,7 +8,7 @@ where { /// Compares characters lexicographically by their packed code values. pub fn cmp_string(&self, other: &Self) -> Ordering { - self.bits.as_bit_str().cmp_str(&other.bits.as_bit_str()) + self.as_packed_str().cmp_codes(&other.as_packed_str()) } } diff --git a/tests/packed_char.rs b/tests/packed_char.rs index 3ad5164..5451180 100644 --- a/tests/packed_char.rs +++ b/tests/packed_char.rs @@ -99,3 +99,11 @@ fn from_bits_rejects_misaligned_and_unknown_codes() { assert!(PackedString::::from_bits(BitString::from_iter([true])).is_none()); assert!(PackedString::::from_bits(BitString::from_iter([true, true])).is_none()); } + +#[test] +fn ordering_uses_packed_code_values() { + let one = PackedString::::from_chars([Symbol::One]); + let two = PackedString::::from_chars([Symbol::Two]); + assert!(one < two); + assert!(one.as_packed_str() < two.as_packed_str()); +} From 9e31708095df398dc280d7ccab4237dfebaf7d7c Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 00:51:50 +0000 Subject: [PATCH 67/79] propagate packed string invariants --- src/packed_str/impls_for_access.rs | 11 ++++--- src/packed_str/impls_for_conversion.rs | 19 ++++++++--- src/packed_str/impls_for_matching.rs | 14 ++++---- src/packed_str/impls_for_slice.rs | 20 +++--------- src/packed_string/funcs_for_code.rs | 3 +- .../impls_for_from_bits.rs | 13 ++++++-- .../impls_for_editing/impls_for_concat.rs | 2 +- .../impls_for_editing/impls_for_drain.rs | 13 +++++--- .../impls_for_insert_remove.rs | 4 +-- .../impls_for_editing/impls_for_replace.rs | 32 +++++++++++++------ .../impls_for_editing/impls_for_slice.rs | 3 +- src/packed_string/impls_for_matching.rs | 4 +-- src/traits/packed_char.rs | 2 +- 13 files changed, 81 insertions(+), 59 deletions(-) diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs index 02f1e59..0eabfaa 100644 --- a/src/packed_str/impls_for_access.rs +++ b/src/packed_str/impls_for_access.rs @@ -16,10 +16,13 @@ where if index >= self.char_len() { return None; } - C::from_code( - (self.bits.get_chunk(index * usize::from(BITS)) - & u64::from(crate::packed_string::funcs_for_code::code_mask::())) - as u8, + Some( + C::from_code( + (self.bits.get_chunk(index * usize::from(BITS)) + & u64::from(crate::packed_string::funcs_for_code::code_mask::())) + as u8, + ) + .expect("PackedChar rejected a code in a PackedStr invariant"), ) } diff --git a/src/packed_str/impls_for_conversion.rs b/src/packed_str/impls_for_conversion.rs index d024c31..d9057bd 100644 --- a/src/packed_str/impls_for_conversion.rs +++ b/src/packed_str/impls_for_conversion.rs @@ -5,7 +5,19 @@ where C: PackedChar, { pub fn to_packed_string(&self) -> PackedString { - PackedString::from_bits(self.bits.to_bit_string()).expect("PackedStr invariant violated") + PackedString::from_valid_bits(self.bits.to_bit_string()) + } + + #[inline] + pub(crate) fn from_valid_bit_str(bits: BitStr<'ps>) -> Self { + let width = usize::from(BITS); + let _ = crate::packed_string::funcs_for_code::code_mask::(); + debug_assert_eq!(bits.start() % width, 0); + debug_assert_eq!(bits.bit_len() % width, 0); + Self { + bits, + marker: core::marker::PhantomData, + } } } @@ -14,9 +26,6 @@ where C: PackedChar, { pub fn as_packed_str(&self) -> PackedStr<'_, C, BITS> { - PackedStr { - bits: self.bits().as_bit_str(), - marker: core::marker::PhantomData, - } + PackedStr::from_valid_bit_str(self.bits().as_bit_str()) } } diff --git a/src/packed_str/impls_for_matching.rs b/src/packed_str/impls_for_matching.rs index 6a165eb..496c735 100644 --- a/src/packed_str/impls_for_matching.rs +++ b/src/packed_str/impls_for_matching.rs @@ -39,16 +39,14 @@ where } pub fn strip_prefix(&self, prefix: Self) -> Option { - self.bits.strip_prefix_str(prefix.bits).map(|bits| Self { - bits, - marker: core::marker::PhantomData, - }) + self.bits + .strip_prefix_str(prefix.bits) + .map(Self::from_valid_bit_str) } pub fn strip_suffix(&self, suffix: Self) -> Option { - self.bits.strip_suffix_str(suffix.bits).map(|bits| Self { - bits, - marker: core::marker::PhantomData, - }) + self.bits + .strip_suffix_str(suffix.bits) + .map(Self::from_valid_bit_str) } } diff --git a/src/packed_str/impls_for_slice.rs b/src/packed_str/impls_for_slice.rs index 65a9c67..3d380a4 100644 --- a/src/packed_str/impls_for_slice.rs +++ b/src/packed_str/impls_for_slice.rs @@ -14,34 +14,22 @@ where .min(self.char_len()) .saturating_mul(bits); if end <= start { - return Self { - bits: self.bits.slice_until(0), - marker: core::marker::PhantomData, - }; + return Self::from_valid_bit_str(self.bits.slice_until(0)); } let interval = UsizeCO::checked_from_start_len(start, end - start).unwrap(); - Self { - bits: self.bits.slice(interval), - marker: core::marker::PhantomData, - } + Self::from_valid_bit_str(self.bits.slice(interval)) } pub fn slice_from(&self, start: usize) -> Self { if start >= self.char_len() { - return Self { - bits: self.bits.slice_from(self.bits.bit_len()), - marker: core::marker::PhantomData, - }; + return Self::from_valid_bit_str(self.bits.slice_from(self.bits.bit_len())); } self.slice(UsizeCO::checked_from_start_len(start, self.char_len() - start).unwrap()) } pub fn slice_until(&self, end: usize) -> Self { if end == 0 { - return Self { - bits: self.bits.slice_until(0), - marker: core::marker::PhantomData, - }; + return Self::from_valid_bit_str(self.bits.slice_until(0)); } self.slice(UsizeCO::checked_from_start_len(0, end).unwrap()) } diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 93b5647..f829fb1 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -3,10 +3,9 @@ use super::*; #[inline] pub(crate) const fn code_mask() -> u8 { match BITS { - 0 => 0, 1..=7 => (1u8 << BITS) - 1, 8 => u8::MAX, - _ => panic!("packed character width must not exceed 8"), + _ => panic!("packed character width must be between 1 and 8"), } } diff --git a/src/packed_string/impls_for_construction/impls_for_from_bits.rs b/src/packed_string/impls_for_construction/impls_for_from_bits.rs index 3507be2..4a90ef2 100644 --- a/src/packed_string/impls_for_construction/impls_for_from_bits.rs +++ b/src/packed_string/impls_for_construction/impls_for_from_bits.rs @@ -15,10 +15,19 @@ where for start in (0..bits.bit_len()).step_by(bits_per_char) { C::from_code((bits.get_chunk(start) & code_mask) as u8)?; } - Some(Self { + Some(Self::from_valid_bits(bits)) + } + + /// Adopts a bit payload whose packed-character invariant is already known. + #[inline] + pub(crate) fn from_valid_bits(bits: BitString) -> Self { + let width = usize::from(BITS); + let _ = code_mask::(); + debug_assert_eq!(bits.bit_len() % width, 0); + Self { bits, marker: PhantomData, - }) + } } pub fn into_bits(self) -> BitString { diff --git a/src/packed_string/impls_for_editing/impls_for_concat.rs b/src/packed_string/impls_for_editing/impls_for_concat.rs index 22220a4..b65df9a 100644 --- a/src/packed_string/impls_for_editing/impls_for_concat.rs +++ b/src/packed_string/impls_for_editing/impls_for_concat.rs @@ -15,6 +15,6 @@ where pub fn split_off(&mut self, at: usize) -> Self { let at = at.min(self.char_len()) * usize::from(BITS); - Self::from_bits(self.bits.split_off(at)).expect("PackedString invariant violated") + Self::from_valid_bits(self.bits.split_off(at)) } } diff --git a/src/packed_string/impls_for_editing/impls_for_drain.rs b/src/packed_string/impls_for_editing/impls_for_drain.rs index ba0e920..cf582d6 100644 --- a/src/packed_string/impls_for_editing/impls_for_drain.rs +++ b/src/packed_string/impls_for_editing/impls_for_drain.rs @@ -14,12 +14,17 @@ where return self.clone(); } let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; - Self::from_bits(self.bits.drain_interval(interval)) - .expect("PackedString invariant violated") + Self::from_valid_bits(self.bits.drain_interval(interval)) } pub fn drain_interval_assign(&mut self, interval: UsizeCO) { - let result = self.drain_interval(interval); - *self = result; + let bits = usize::from(BITS); + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + if start == end { + return; + } + let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; + self.bits.drain_interval_assign(interval); } } diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs index 7502f52..03f4f02 100644 --- a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs @@ -5,9 +5,7 @@ where C: PackedChar, { pub fn insert(&mut self, index: usize, character: C) { - if index > self.char_len() { - return; - } + let index = index.min(self.char_len()); self.bits .bit_len() .checked_add(usize::from(BITS)) diff --git a/src/packed_string/impls_for_editing/impls_for_replace.rs b/src/packed_string/impls_for_editing/impls_for_replace.rs index 78f25b2..330ee7e 100644 --- a/src/packed_string/impls_for_editing/impls_for_replace.rs +++ b/src/packed_string/impls_for_editing/impls_for_replace.rs @@ -13,8 +13,9 @@ where } pub fn replace_interval_assign(&mut self, interval: UsizeCO, replacement: &Self) { - let result = self.replace_interval(interval, replacement); - *self = result; + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + self.replace_range_assign(start, end, replacement); } pub fn replace(&self, start: usize, replacement: &Self) -> Self { @@ -26,19 +27,32 @@ where } pub fn replace_assign(&mut self, start: usize, replacement: &Self) { - let result = self.replace(start, replacement); - *self = result; + let start = start.min(self.char_len()); + let end = start + .saturating_add(replacement.char_len()) + .min(self.char_len()); + self.replace_range_assign(start, end, replacement); } fn replace_range(&self, start: usize, end: usize, replacement: &Self) -> Self { let bits = usize::from(BITS); - let result = self.bits.replace(start * bits, &replacement.bits); - Self::from_bits(if end > start { + let bits = if end > start { let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; self.bits.replace_interval(interval, &replacement.bits) } else { - result - }) - .expect("PackedString invariant violated") + self.bits.replace(start * bits, &replacement.bits) + }; + Self::from_valid_bits(bits) + } + + fn replace_range_assign(&mut self, start: usize, end: usize, replacement: &Self) { + let bits = usize::from(BITS); + if end > start { + let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; + self.bits + .replace_interval_assign(interval, &replacement.bits); + } else { + self.bits.replace_assign(start * bits, &replacement.bits); + } } } diff --git a/src/packed_string/impls_for_editing/impls_for_slice.rs b/src/packed_string/impls_for_editing/impls_for_slice.rs index 2fcae50..a2ae8d9 100644 --- a/src/packed_string/impls_for_editing/impls_for_slice.rs +++ b/src/packed_string/impls_for_editing/impls_for_slice.rs @@ -13,12 +13,11 @@ where return Self::new(); } let bits = usize::from(BITS); - Self::from_bits( + Self::from_valid_bits( self.bits.slice( UsizeCO::checked_from_start_len(start * bits, (end - start) * bits).unwrap(), ), ) - .expect("PackedString invariant violated") } pub fn slice_from(&self, start: usize) -> Self { diff --git a/src/packed_string/impls_for_matching.rs b/src/packed_string/impls_for_matching.rs index 379e63c..a6fa37a 100644 --- a/src/packed_string/impls_for_matching.rs +++ b/src/packed_string/impls_for_matching.rs @@ -41,12 +41,12 @@ where pub fn strip_prefix(&self, prefix: &Self) -> Option { self.bits .strip_prefix_string(&prefix.bits) - .and_then(Self::from_bits) + .map(Self::from_valid_bits) } pub fn strip_suffix(&self, suffix: &Self) -> Option { self.bits .strip_suffix_string(&suffix.bits) - .and_then(Self::from_bits) + .map(Self::from_valid_bits) } } diff --git a/src/traits/packed_char.rs b/src/traits/packed_char.rs index c684277..e238f3c 100644 --- a/src/traits/packed_char.rs +++ b/src/traits/packed_char.rs @@ -4,7 +4,7 @@ /// `#[packed(bits = N)]` attribute for that common case; manual /// implementations must satisfy all of the following invariants: /// -/// - `BITS` does not exceed eight; +/// - `1 <= BITS <= 8`; /// - [`code`](Self::code) fits in `BITS` bits; /// - `from_code(code(value)) == Some(value)`; /// - each valid code identifies at most one value. From 25c43d155c5909fa44ad39dcf368097ab6b9df25 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 00:59:49 +0000 Subject: [PATCH 68/79] add packed string oracle tests --- tests/packed_char.rs | 125 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/tests/packed_char.rs b/tests/packed_char.rs index 5451180..da54867 100644 --- a/tests/packed_char.rs +++ b/tests/packed_char.rs @@ -1,5 +1,6 @@ use bit_string::{BitString, PackedString, packed, traits::PackedChar}; use int_interval::UsizeCO; +use proptest::prelude::*; #[packed(bits = 1)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -16,6 +17,130 @@ enum Symbol { Two = 2, } +fn symbol(code: u8) -> Symbol { + match code { + 0 => Symbol::Zero, + 1 => Symbol::One, + 2 => Symbol::Two, + _ => unreachable!(), + } +} + +fn packed(codes: &[u8]) -> PackedString { + PackedString::from_chars(codes.iter().copied().map(symbol)) +} + +fn matches_at(haystack: &[u8], start: usize, needle: &[u8]) -> bool { + haystack + .get(start..start.saturating_add(needle.len())) + .is_some_and(|window| window == needle) +} + +proptest! { + #[test] + fn packed_access_slice_and_order_match_vec_oracles( + left in prop::collection::vec(0u8..=2, 0..=32), + right in prop::collection::vec(0u8..=2, 0..=32), + start in 0usize..40, + len in 1usize..40, + ) { + let string = packed(&left); + prop_assert_eq!(string.char_len(), left.len()); + let view = string.as_packed_str(); + for index in 0..=left.len() + 1 { + prop_assert_eq!(string.get(index), left.get(index).copied().map(symbol)); + prop_assert_eq!(view.get(index), left.get(index).copied().map(symbol)); + } + + let oracle_start = start.min(left.len()); + let oracle_end = start.saturating_add(len).min(left.len()).max(oracle_start); + let slice = string.slice(UsizeCO::checked_from_start_len(start, len).unwrap()); + prop_assert_eq!(slice.to_vec(), left[oracle_start..oracle_end].iter().copied().map(symbol).collect::>()); + + let right_string = packed(&right); + prop_assert_eq!(string.cmp(&right_string), left.cmp(&right)); + prop_assert_eq!(string.as_packed_str().cmp(&right_string.as_packed_str()), left.cmp(&right)); + } + + #[test] + fn packed_matching_matches_vec_sliding_windows( + haystack in prop::collection::vec(0u8..=2, 0..=32), + needle in prop::collection::vec(0u8..=2, 0..=16), + ) { + let haystack_string = packed(&haystack); + let needle_string = packed(&needle); + let haystack_view = haystack_string.as_packed_str(); + let needle_view = needle_string.as_packed_str(); + let max_start = haystack.len().saturating_sub(needle.len()); + let expected_find = (0..=max_start).find(|&start| matches_at(&haystack, start, &needle)); + let expected_rfind = (0..=max_start).rev().find(|&start| matches_at(&haystack, start, &needle)); + + prop_assert_eq!(haystack_view.find(needle_view), expected_find); + prop_assert_eq!(haystack_view.rfind(needle_view), expected_rfind); + prop_assert_eq!(haystack_view.contains(needle_view), expected_find.is_some()); + } + + #[test] + fn packed_edits_match_vec_splice_semantics( + initial in prop::collection::vec(0u8..=2, 0..=24), + replacement in prop::collection::vec(0u8..=2, 0..=12), + insert_index in 0usize..32, + remove_index in 0usize..32, + replace_start in 0usize..32, + ) { + let mut string = packed(&initial); + let mut oracle = initial.clone(); + + let oracle_insert_index = insert_index.min(oracle.len()); + let inserted = symbol(2); + string.insert(insert_index, inserted); + oracle.insert(oracle_insert_index, 2); + + if !oracle.is_empty() { + let remove_index = remove_index.min(oracle.len() - 1); + prop_assert_eq!(string.remove(remove_index).code(), oracle.remove(remove_index)); + } + + let oracle_replace_start = replace_start.min(oracle.len()); + let replace_end = oracle_replace_start + .saturating_add(replacement.len()) + .min(oracle.len()); + string.replace_assign(replace_start, &packed(&replacement)); + oracle.splice(oracle_replace_start..replace_end, replacement.iter().copied()); + prop_assert_eq!( + string.to_vec().iter().copied().map(|value| value.code()).collect::>(), + oracle.clone(), + ); + + let drain_len = insert_index.max(1); + let oracle_drain_start = remove_index.min(oracle.len()); + let oracle_drain_end = remove_index + .saturating_add(drain_len) + .min(oracle.len()) + .max(oracle_drain_start); + string.drain_interval_assign( + UsizeCO::checked_from_start_len(remove_index, drain_len).unwrap(), + ); + oracle.drain(oracle_drain_start..oracle_drain_end); + prop_assert_eq!( + string.to_vec().iter().copied().map(|value| value.code()).collect::>(), + oracle.clone(), + ); + } +} + +#[test] +fn packed_out_of_bounds_intervals_preserve_empty_semantics() { + let string = packed(&[0, 1, 2]); + let out_of_bounds = UsizeCO::checked_from_start_len(99, 1).unwrap(); + + assert!(string.slice(out_of_bounds).is_empty()); + + let mut drained = string.clone(); + drained.drain_interval_assign(out_of_bounds); + assert_eq!(drained, string); +} + #[test] fn packed_attribute_encoding_round_trips_through_packed_string() { let string = PackedString::::from_chars([Symbol::Two, Symbol::Zero, Symbol::One]); From d54e24adda5ab12d1964e7c09155d6ebe1c983c0 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 01:34:20 +0000 Subject: [PATCH 69/79] harden packed boundary coverage --- src/funcs_for_packed.rs | 17 ++ src/lib.rs | 2 + src/packed_str.rs | 1 - src/packed_str/impls_for_access.rs | 3 +- src/packed_str/impls_for_conversion.rs | 2 +- src/packed_str/impls_for_ord.rs | 2 +- src/packed_str/impls_for_slice.rs | 2 +- src/packed_string.rs | 5 +- src/packed_string/funcs_for_code.rs | 9 - src/packed_string/impls_for_construction.rs | 7 +- .../impls_for_from_bits.rs | 4 +- tests/packed_char.rs | 224 ++++++++++++++++++ 12 files changed, 253 insertions(+), 25 deletions(-) create mode 100644 src/funcs_for_packed.rs diff --git a/src/funcs_for_packed.rs b/src/funcs_for_packed.rs new file mode 100644 index 0000000..886fe03 --- /dev/null +++ b/src/funcs_for_packed.rs @@ -0,0 +1,17 @@ +#[inline] +pub(crate) const fn assert_valid_width() { + assert!( + BITS > 0 && BITS <= 8, + "packed character width must be between 1 and 8" + ); +} + +#[inline] +pub(crate) const fn code_mask() -> u8 { + assert_valid_width::(); + match BITS { + 1..=7 => (1u8 << BITS) - 1, + 8 => u8::MAX, + _ => unreachable!(), + } +} diff --git a/src/lib.rs b/src/lib.rs index 277c560..b29060c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,10 +4,12 @@ extern crate alloc; mod consts_for_bits; mod funcs_for_bits; +mod funcs_for_packed; pub mod traits; pub(crate) use consts_for_bits::*; pub(crate) use funcs_for_bits::*; +pub(crate) use funcs_for_packed::*; mod bit_str; mod bit_string; diff --git a/src/packed_str.rs b/src/packed_str.rs index 193b407..507a3af 100644 --- a/src/packed_str.rs +++ b/src/packed_str.rs @@ -6,7 +6,6 @@ use crate::{BitStr, PackedString, traits::PackedChar}; /// A character-aligned borrowed view into a [`PackedString`]. #[derive(Clone, Copy)] -#[allow(dead_code)] pub struct PackedStr<'ps, C, const BITS: u8> where C: PackedChar, diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs index 0eabfaa..613e80a 100644 --- a/src/packed_str/impls_for_access.rs +++ b/src/packed_str/impls_for_access.rs @@ -19,8 +19,7 @@ where Some( C::from_code( (self.bits.get_chunk(index * usize::from(BITS)) - & u64::from(crate::packed_string::funcs_for_code::code_mask::())) - as u8, + & u64::from(crate::code_mask::())) as u8, ) .expect("PackedChar rejected a code in a PackedStr invariant"), ) diff --git a/src/packed_str/impls_for_conversion.rs b/src/packed_str/impls_for_conversion.rs index d9057bd..5e5fc38 100644 --- a/src/packed_str/impls_for_conversion.rs +++ b/src/packed_str/impls_for_conversion.rs @@ -11,7 +11,7 @@ where #[inline] pub(crate) fn from_valid_bit_str(bits: BitStr<'ps>) -> Self { let width = usize::from(BITS); - let _ = crate::packed_string::funcs_for_code::code_mask::(); + crate::assert_valid_width::(); debug_assert_eq!(bits.start() % width, 0); debug_assert_eq!(bits.bit_len() % width, 0); Self { diff --git a/src/packed_str/impls_for_ord.rs b/src/packed_str/impls_for_ord.rs index b3fe22c..b14809a 100644 --- a/src/packed_str/impls_for_ord.rs +++ b/src/packed_str/impls_for_ord.rs @@ -7,7 +7,7 @@ where C: PackedChar, { pub(crate) fn cmp_codes(&self, other: &Self) -> Ordering { - let mask = u64::from(crate::packed_string::funcs_for_code::code_mask::()); + let mask = u64::from(crate::code_mask::()); let shared_len = self.char_len().min(other.char_len()); for index in 0..shared_len { let start = index * usize::from(BITS); diff --git a/src/packed_str/impls_for_slice.rs b/src/packed_str/impls_for_slice.rs index 3d380a4..d8d9ba0 100644 --- a/src/packed_str/impls_for_slice.rs +++ b/src/packed_str/impls_for_slice.rs @@ -14,7 +14,7 @@ where .min(self.char_len()) .saturating_mul(bits); if end <= start { - return Self::from_valid_bit_str(self.bits.slice_until(0)); + return Self::from_valid_bit_str(self.bits.slice_from(start)); } let interval = UsizeCO::checked_from_start_len(start, end - start).unwrap(); Self::from_valid_bit_str(self.bits.slice(interval)) diff --git a/src/packed_string.rs b/src/packed_string.rs index 08185a6..9e4b78e 100644 --- a/src/packed_string.rs +++ b/src/packed_string.rs @@ -18,7 +18,7 @@ where marker: PhantomData C>, } -pub(crate) mod funcs_for_code; +mod funcs_for_code; mod impls_for_access; mod impls_for_construction; mod impls_for_editing; @@ -29,7 +29,8 @@ mod impls_for_iter; mod impls_for_matching; mod impls_for_ord; -use funcs_for_code::{code_mask, write_code}; +use crate::{assert_valid_width, code_mask}; +use funcs_for_code::write_code; #[cfg(test)] mod tests_for_support; diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index f829fb1..04d47bb 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -1,14 +1,5 @@ use super::*; -#[inline] -pub(crate) const fn code_mask() -> u8 { - match BITS { - 1..=7 => (1u8 << BITS) - 1, - 8 => u8::MAX, - _ => panic!("packed character width must be between 1 and 8"), - } -} - #[inline] pub(super) fn write_code(bits: &mut BitString, position: usize, code: u8) { let start = position * usize::from(BITS); diff --git a/src/packed_string/impls_for_construction.rs b/src/packed_string/impls_for_construction.rs index 386f4ae..df209f8 100644 --- a/src/packed_string/impls_for_construction.rs +++ b/src/packed_string/impls_for_construction.rs @@ -4,14 +4,9 @@ impl PackedString where C: PackedChar, { - const VALID_WIDTH: () = assert!( - BITS > 0 && BITS <= 8, - "packed character width must be between 1 and 8" - ); - #[inline] pub fn new() -> Self { - let () = Self::VALID_WIDTH; + assert_valid_width::(); Self { bits: BitString::new(), marker: PhantomData, diff --git a/src/packed_string/impls_for_construction/impls_for_from_bits.rs b/src/packed_string/impls_for_construction/impls_for_from_bits.rs index 4a90ef2..8b030d3 100644 --- a/src/packed_string/impls_for_construction/impls_for_from_bits.rs +++ b/src/packed_string/impls_for_construction/impls_for_from_bits.rs @@ -6,7 +6,7 @@ where { /// Validates and adopts an already packed bit payload. pub fn from_bits(bits: BitString) -> Option { - let () = Self::VALID_WIDTH; + assert_valid_width::(); let bits_per_char = usize::from(BITS); if bits.bit_len() % bits_per_char != 0 { return None; @@ -22,7 +22,7 @@ where #[inline] pub(crate) fn from_valid_bits(bits: BitString) -> Self { let width = usize::from(BITS); - let _ = code_mask::(); + assert_valid_width::(); debug_assert_eq!(bits.bit_len() % width, 0); Self { bits, diff --git a/tests/packed_char.rs b/tests/packed_char.rs index da54867..6d24ab3 100644 --- a/tests/packed_char.rs +++ b/tests/packed_char.rs @@ -17,6 +17,32 @@ enum Symbol { Two = 2, } +#[packed(bits = 3)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Oct { + V0 = 0, + V1 = 1, + V2 = 2, + V3 = 3, + V4 = 4, + V5 = 5, + V6 = 6, + V7 = 7, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct WideCode(u8); + +impl PackedChar<7> for WideCode { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + (code < 128).then_some(Self(code)) + } +} + fn symbol(code: u8) -> Symbol { match code { 0 => Symbol::Zero, @@ -26,16 +52,160 @@ fn symbol(code: u8) -> Symbol { } } +fn oct(code: u8) -> Oct { + match code { + 0 => Oct::V0, + 1 => Oct::V1, + 2 => Oct::V2, + 3 => Oct::V3, + 4 => Oct::V4, + 5 => Oct::V5, + 6 => Oct::V6, + 7 => Oct::V7, + _ => unreachable!(), + } +} + +fn wide(code: u8) -> WideCode { + WideCode(code) +} + fn packed(codes: &[u8]) -> PackedString { PackedString::from_chars(codes.iter().copied().map(symbol)) } +fn packed_as(codes: &[u8], decode: fn(u8) -> C) -> PackedString +where + C: PackedChar, +{ + PackedString::from_chars(codes.iter().copied().map(decode)) +} + fn matches_at(haystack: &[u8], start: usize, needle: &[u8]) -> bool { haystack .get(start..start.saturating_add(needle.len())) .is_some_and(|window| window == needle) } +fn assert_access_slice_order( + left: &[u8], + right: &[u8], + start: usize, + len: usize, + decode: fn(u8) -> C, +) where + C: PackedChar + core::fmt::Debug, +{ + let string = packed_as(left, decode); + assert_eq!(string.char_len(), left.len()); + let view = string.as_packed_str(); + for index in 0..=left.len() + 1 { + assert_eq!(string.get(index), left.get(index).copied().map(decode)); + assert_eq!(view.get(index), left.get(index).copied().map(decode)); + } + + let oracle_start = start.min(left.len()); + let oracle_end = start.saturating_add(len).min(left.len()).max(oracle_start); + let slice = string.slice(UsizeCO::checked_from_start_len(start, len).unwrap()); + assert_eq!( + slice.to_vec(), + left[oracle_start..oracle_end] + .iter() + .copied() + .map(decode) + .collect::>() + ); + + let right_string = packed_as(right, decode); + assert_eq!(string.cmp(&right_string), left.cmp(&right)); + assert_eq!( + string.as_packed_str().cmp(&right_string.as_packed_str()), + left.cmp(&right) + ); +} + +fn assert_matching(haystack: &[u8], needle: &[u8], decode: fn(u8) -> C) +where + C: PackedChar, +{ + let haystack_string = packed_as(haystack, decode); + let needle_string = packed_as(needle, decode); + let haystack_view = haystack_string.as_packed_str(); + let needle_view = needle_string.as_packed_str(); + let max_start = haystack.len().saturating_sub(needle.len()); + let expected_find = (0..=max_start).find(|&start| matches_at(haystack, start, needle)); + let expected_rfind = (0..=max_start) + .rev() + .find(|&start| matches_at(haystack, start, needle)); + + assert_eq!(haystack_view.find(needle_view), expected_find); + assert_eq!(haystack_view.rfind(needle_view), expected_rfind); + assert_eq!(haystack_view.contains(needle_view), expected_find.is_some()); +} + +fn assert_edits( + initial: &[u8], + replacement: &[u8], + insert_index: usize, + remove_index: usize, + replace_start: usize, + decode: fn(u8) -> C, +) where + C: PackedChar + core::fmt::Debug, +{ + let mut string = packed_as(initial, decode); + let mut oracle = initial.to_vec(); + + let oracle_insert_index = insert_index.min(oracle.len()); + string.insert(insert_index, decode(2)); + oracle.insert(oracle_insert_index, 2); + + if !oracle.is_empty() { + let remove_index = remove_index.min(oracle.len() - 1); + assert_eq!( + string.remove(remove_index).code(), + oracle.remove(remove_index) + ); + } + + let oracle_replace_start = replace_start.min(oracle.len()); + let replace_end = oracle_replace_start + .saturating_add(replacement.len()) + .min(oracle.len()); + string.replace_assign(replace_start, &packed_as(replacement, decode)); + oracle.splice( + oracle_replace_start..replace_end, + replacement.iter().copied(), + ); + assert_eq!( + string + .to_vec() + .iter() + .copied() + .map(|value| value.code()) + .collect::>(), + oracle + ); + + let drain_len = insert_index.max(1); + let oracle_drain_start = remove_index.min(oracle.len()); + let oracle_drain_end = remove_index + .saturating_add(drain_len) + .min(oracle.len()) + .max(oracle_drain_start); + string.drain_interval_assign(UsizeCO::checked_from_start_len(remove_index, drain_len).unwrap()); + oracle.drain(oracle_drain_start..oracle_drain_end); + assert_eq!( + string + .to_vec() + .iter() + .copied() + .map(|value| value.code()) + .collect::>(), + oracle + ); +} + proptest! { #[test] fn packed_access_slice_and_order_match_vec_oracles( @@ -129,6 +299,60 @@ proptest! { } } +proptest! { + #[test] + fn three_and_seven_bit_access_slice_and_order_cross_word_boundaries( + left3 in prop::collection::vec(0u8..=7, 22..=128), + right3 in prop::collection::vec(0u8..=7, 22..=128), + left7 in prop::collection::vec(0u8..=127, 10..=128), + right7 in prop::collection::vec(0u8..=127, 10..=128), + start in 0usize..160, + len in 1usize..80, + ) { + assert_access_slice_order::(&left3, &right3, start, len, oct); + assert_access_slice_order::(&left7, &right7, start, len, wide); + } + + #[test] + fn three_and_seven_bit_matching_crosses_word_boundaries( + haystack3 in prop::collection::vec(0u8..=7, 22..=128), + needle3 in prop::collection::vec(0u8..=7, 0..=32), + haystack7 in prop::collection::vec(0u8..=127, 10..=128), + needle7 in prop::collection::vec(0u8..=127, 0..=16), + ) { + assert_matching::(&haystack3, &needle3, oct); + assert_matching::(&haystack7, &needle7, wide); + } + + #[test] + fn three_and_seven_bit_edits_cross_word_boundaries( + initial3 in prop::collection::vec(0u8..=7, 22..=128), + replacement3 in prop::collection::vec(0u8..=7, 0..=64), + initial7 in prop::collection::vec(0u8..=127, 10..=128), + replacement7 in prop::collection::vec(0u8..=127, 0..=32), + insert_index in 0usize..160, + remove_index in 0usize..160, + replace_start in 0usize..160, + ) { + assert_edits::( + &initial3, + &replacement3, + insert_index, + remove_index, + replace_start, + oct, + ); + assert_edits::( + &initial7, + &replacement7, + insert_index, + remove_index, + replace_start, + wide, + ); + } +} + #[test] fn packed_out_of_bounds_intervals_preserve_empty_semantics() { let string = packed(&[0, 1, 2]); From 620162388436459b6942695a7a7cb4a8fbccfdb5 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 01:54:11 +0000 Subject: [PATCH 70/79] make packed edit oracle explicit --- tests/packed_char.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/packed_char.rs b/tests/packed_char.rs index 6d24ab3..9ace679 100644 --- a/tests/packed_char.rs +++ b/tests/packed_char.rs @@ -149,6 +149,7 @@ fn assert_edits( insert_index: usize, remove_index: usize, replace_start: usize, + inserted_code: u8, decode: fn(u8) -> C, ) where C: PackedChar + core::fmt::Debug, @@ -157,8 +158,8 @@ fn assert_edits( let mut oracle = initial.to_vec(); let oracle_insert_index = insert_index.min(oracle.len()); - string.insert(insert_index, decode(2)); - oracle.insert(oracle_insert_index, 2); + string.insert(insert_index, decode(inserted_code)); + oracle.insert(oracle_insert_index, inserted_code); if !oracle.is_empty() { let remove_index = remove_index.min(oracle.len() - 1); @@ -340,6 +341,7 @@ proptest! { insert_index, remove_index, replace_start, + 2, oct, ); assert_edits::( @@ -348,6 +350,7 @@ proptest! { insert_index, remove_index, replace_start, + 2, wide, ); } From cea5e2bc2cbc55596d1e334574c16dad5e2251b3 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 02:15:56 +0000 Subject: [PATCH 71/79] chore: retire codegen migration gate --- .github/workflows/gate.yml | 9 +- scripts/bench_config.py | 9 +- scripts/codegen_harness.rs | 156 ----- scripts/codegen_roots.toml | 274 --------- scripts/compare_codegen.py | 777 ------------------------- scripts/kernel_inventory.toml | 97 ---- scripts/run_codegen_comparison.py | 142 ----- scripts/tests/test_codegen.py | 937 ------------------------------ 8 files changed, 2 insertions(+), 2399 deletions(-) delete mode 100644 scripts/codegen_harness.rs delete mode 100644 scripts/codegen_roots.toml delete mode 100755 scripts/compare_codegen.py delete mode 100644 scripts/kernel_inventory.toml delete mode 100755 scripts/run_codegen_comparison.py delete mode 100644 scripts/tests/test_codegen.py diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index afe3dec..e856da8 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -22,22 +22,18 @@ jobs: - name: linux-x64-baseline os: ubuntu-latest rustflags: "" - codegen_config: x86_64-baseline - name: linux-x64-sse4.1 os: ubuntu-latest rustflags: "-C target-feature=+sse4.1" - codegen_config: x86_64-sse41 - name: linux-x64-avx2 os: ubuntu-latest rustflags: "-C target-feature=+avx2" - codegen_config: x86_64-avx2 - name: linux-arm64-neon os: ubuntu-24.04-arm rustflags: "-C target-feature=+neon" - codegen_config: aarch64-neon env: RUSTFLAGS: ${{ matrix.rustflags }} @@ -67,9 +63,6 @@ jobs: - name: Test run: cargo test --locked - - name: Test comparison tooling + - name: Test benchmark tooling if: matrix.name == 'linux-x64-baseline' run: python3 -m unittest discover -s scripts/tests -v - - - name: Compare codegen with fixed baseline - run: python3 scripts/run_codegen_comparison.py --config ${{ matrix.codegen_config }} diff --git a/scripts/bench_config.py b/scripts/bench_config.py index 17c09a0..509c677 100644 --- a/scripts/bench_config.py +++ b/scripts/bench_config.py @@ -1,11 +1,4 @@ -"""Shared constants for reproducible baseline/codegen benchmark runs.""" +"""Shared constants for reproducible benchmark runs.""" BASELINE_SHA = "c15fa1c254ea5b2bbabfe5f008d41f91e8297b89" BENCH_RUSTFLAGS = "-C target-cpu=native" - -CODEGEN_CONFIGS = { - "x86_64-baseline": ("x86_64", "-C target-cpu=x86-64"), - "x86_64-sse41": ("x86_64", "-C target-cpu=x86-64 -C target-feature=+sse4.1"), - "x86_64-avx2": ("x86_64", "-C target-cpu=x86-64 -C target-feature=+avx2"), - "aarch64-neon": ("aarch64", "-C target-cpu=generic -C target-feature=+neon"), -} diff --git a/scripts/codegen_harness.rs b/scripts/codegen_harness.rs deleted file mode 100644 index d662f21..0000000 --- a/scripts/codegen_harness.rs +++ /dev/null @@ -1,156 +0,0 @@ -#![no_std] - -use bit_string::BitString; - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_from_bool_slice(values: *const bool, len: usize) -> BitString { - let values = unsafe { core::slice::from_raw_parts(values, len) }; - BitString::from(values) -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_from_str(value: *const u8, len: usize) -> BitString { - let bytes = unsafe { core::slice::from_raw_parts(value, len) }; - BitString::try_from(unsafe { core::str::from_utf8_unchecked(bytes) }).unwrap() -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_xor(lhs: *const BitString, rhs: *const BitString) -> BitString { - unsafe { (&*lhs).xor(&*rhs).unwrap() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_and(lhs: *const BitString, rhs: *const BitString) -> BitString { - unsafe { (&*lhs).and(&*rhs).unwrap() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_or(lhs: *const BitString, rhs: *const BitString) -> BitString { - unsafe { (&*lhs).or(&*rhs).unwrap() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_not(value: *const BitString) -> BitString { - unsafe { (&*value).not() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_shl(value: *const BitString, amount: usize) -> BitString { - unsafe { (&*value).shl(amount) } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_shr(value: *const BitString, amount: usize) -> BitString { - unsafe { (&*value).shr(amount) } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_slice_from(value: *const BitString, start: usize) -> BitString { - unsafe { (&*value).slice_from(start) } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_matches_at( - lhs: *const BitString, - index: usize, - rhs: *const BitString, -) -> bool { - unsafe { (&*lhs).matches_at_string(index, &*rhs) } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_contains(lhs: *const BitString, rhs: *const BitString) -> bool { - unsafe { (&*lhs).contains_string(&*rhs) } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_find( - lhs: *const BitString, - rhs: *const BitString, -) -> Option { - unsafe { (&*lhs).find_string(&*rhs) } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_rfind( - lhs: *const BitString, - rhs: *const BitString, -) -> Option { - unsafe { (&*lhs).rfind_string(&*rhs) } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_cmp( - lhs: *const BitString, - rhs: *const BitString, -) -> core::cmp::Ordering { - unsafe { (&*lhs).cmp_string(&*rhs) } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_count_ones(value: *const BitString) -> usize { - unsafe { (&*value).count_ones() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_leading_zeros(value: *const BitString) -> usize { - unsafe { (&*value).leading_zeros() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_leading_ones(value: *const BitString) -> usize { - unsafe { (&*value).leading_ones() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_str_leading_zeros(value: *const BitString, start: usize) -> usize { - unsafe { (&*value).as_bit_str().slice_from(start).leading_zeros() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_str_leading_ones(value: *const BitString, start: usize) -> usize { - unsafe { (&*value).as_bit_str().slice_from(start).leading_ones() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_trailing_zeros(value: *const BitString) -> usize { - unsafe { (&*value).trailing_zeros() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_string_trailing_ones(value: *const BitString) -> usize { - unsafe { (&*value).trailing_ones() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_str_trailing_zeros(value: *const BitString, start: usize) -> usize { - unsafe { (&*value).as_bit_str().slice_from(start).trailing_zeros() } -} - -#[unsafe(no_mangle)] -#[inline(never)] -pub unsafe fn codegen_bit_str_trailing_ones(value: *const BitString, start: usize) -> usize { - unsafe { (&*value).as_bit_str().slice_from(start).trailing_ones() } -} diff --git a/scripts/codegen_roots.toml b/scripts/codegen_roots.toml deleted file mode 100644 index fe16537..0000000 --- a/scripts/codegen_roots.toml +++ /dev/null @@ -1,274 +0,0 @@ -[[root]] -symbol = "codegen_bit_string_from_bool_slice" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_from_str" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_xor" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_and" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_or" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_not" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_shl" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_shr" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_slice_from" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_matches_at" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_contains" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_find" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_rfind" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_cmp" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_count_ones" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_leading_zeros" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_leading_ones" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_str_leading_zeros" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_str_leading_ones" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_trailing_zeros" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_string_trailing_ones" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_str_trailing_zeros" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "codegen_bit_str_trailing_ones" -artifact = "harness" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_not_core::avx2::words" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::avx2::eq_words" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::bit_string::impls_for_construction::funcs_for_pack_bools_core::avx2::words" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_shl_core::avx2::words" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_shr_core::avx2::words" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::traits::words_edit::bits_copied::funcs_for_copy_words_shifted_core::avx2::copy_words_shifted" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::traits::words_eq::funcs_for_eq_words_unaligned_core::avx2::eq_words_unaligned" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::traits::words_ord::funcs_for_cmp_aligned_core::avx2::cmp_aligned" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::traits::words_ord::funcs_for_cmp_unaligned_core::avx2::cmp_unaligned" -artifact = "library" -targets = ["x86_64-avx2"] - -[[root]] -symbol = "bit_string::bit_string::impls_for_construction::funcs_for_pack_bools_core::sse2::words" -artifact = "library" -targets = ["x86_64-baseline", "x86_64-sse41"] - -[[root]] -symbol = "bit_string::bit_string::impls_for_construction::impls_for_from_str::funcs_for_pack_str_core::sse2::words" -artifact = "library" -targets = ["x86_64-baseline", "x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_not_core::sse2::words" -artifact = "library" -targets = ["x86_64-baseline", "x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_shl_core::sse2::words" -artifact = "library" -targets = ["x86_64-baseline", "x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_shr_core::sse2::words" -artifact = "library" -targets = ["x86_64-baseline", "x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_edit::bits_copied::funcs_for_copy_words_shifted_core::sse2::copy_words_shifted" -artifact = "library" -targets = ["x86_64-baseline", "x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_shl_core::scalar::words" -artifact = "library" -targets = ["x86_64-baseline", "x86_64-sse41", "x86_64-avx2", "aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::sse41::eq_words" -artifact = "library" -targets = ["x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_eq::funcs_for_eq_words_unaligned_core::sse41::eq_words_unaligned" -artifact = "library" -targets = ["x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_ord::funcs_for_cmp_aligned_core::sse41::cmp_aligned" -artifact = "library" -targets = ["x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_ord::funcs_for_cmp_unaligned_core::sse41::cmp_unaligned" -artifact = "library" -targets = ["x86_64-sse41"] - -[[root]] -symbol = "bit_string::traits::words_scan::funcs_for_count_ones::ssse3::count_words" -artifact = "library" -targets = ["x86_64-sse41"] - -[[root]] -symbol = "bit_string::bit_string::impls_for_construction::funcs_for_pack_bools_core::neon::words" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_not_core::neon::words" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_shl_core::neon::words" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_arith::funcs_for_shr_core::neon::words" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_edit::bits_copied::funcs_for_copy_words_shifted_core::neon::copy_words_shifted" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_eq::funcs_for_eq_words_aligned_core::neon::eq_words" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_eq::funcs_for_eq_words_unaligned_core::neon::eq_words_unaligned" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_ord::funcs_for_cmp_aligned_core::neon::cmp_aligned" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_ord::funcs_for_cmp_unaligned_core::neon::cmp_unaligned" -artifact = "library" -targets = ["aarch64-neon"] - -[[root]] -symbol = "bit_string::traits::words_scan::funcs_for_count_ones::neon::count_words" -artifact = "library" -targets = ["aarch64-neon"] diff --git a/scripts/compare_codegen.py b/scripts/compare_codegen.py deleted file mode 100755 index c8f3666..0000000 --- a/scripts/compare_codegen.py +++ /dev/null @@ -1,777 +0,0 @@ -#!/usr/bin/env python3 -"""Compare normalized disassembly for explicitly configured codegen roots.""" - -from __future__ import annotations - -import argparse -import difflib -import hashlib -import re -import subprocess -import sys -import tomllib -from dataclasses import dataclass, field -from pathlib import Path - -from bench_config import CODEGEN_CONFIGS - -FUNCTION = re.compile(r"^\s*[0-9a-f]+ <(.+)>:\s*$") -X86_INSTRUCTION = re.compile(r"^\s*([0-9a-f]+):\s+((?:[0-9a-f]{2}(?:\s+|$))+)(.*?)\s*$") -AARCH64_INSTRUCTION = re.compile(r"^\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(.*?)\s*$") -RELOCATION = re.compile(r"^\s*([0-9a-f]+):\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") -INLINE_RELOCATION = re.compile(r"\s+([0-9a-f]+):\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") -SYMBOL_TARGET = re.compile(r"(?:0x)?([0-9a-f]+)\s+<([^>]+)>") -PC_RELATIVE_X86_RELOCATIONS = {"R_X86_64_PC32", "R_X86_64_PLT32"} -X86_PC_BIAS = re.compile(r"-0x4$") -ADDEND = re.compile(r"^(?P.*?)(?P[+-]0x[0-9a-f]+)?$") -ANONYMOUS_SYMBOL_INSTANCE = re.compile( - r"(?P(?:\.Lanon|anon)\.[^.\s]+)\.(?P\d+)" - r"(?:\.llvm\.[^+\-\s]+)?(?P[+-]0x[0-9a-f]+)?$" -) -LOCAL_OBJECT_SYMBOL = re.compile(r"(?P\.LCPI\d+_\d+)(?P[+-]0x[0-9a-f]+)?$") -OBJDUMP_COMMENT = re.compile(r"\s+#\s+(?:0x)?[0-9a-f]+\s+<[^>]+>.*$") -BACKEND_MODULE = re.compile(r"::(?:scalar|sse2|ssse3|sse41|avx2|neon)::") - - -@dataclass -class Instruction: - address: int - encoded: bytes - assembly: str - relocations: list[tuple[int, str, str]] = field(default_factory=list) - member: str = "" - - -@dataclass(frozen=True) -class Relocation: - offset: int - kind: str - raw_target: str - token: str - anonymous: bool - resolved: bool - instruction_index: int - addend: str = "" - nested_addends: tuple[str, ...] = () - nested_alias_partition: tuple[int, ...] = () - metadata: bool = False - - -@dataclass(frozen=True) -class SectionRelocation: - offset: int - kind: str - target: str - - -@dataclass(frozen=True) -class SectionData: - member: str - name: str - data: bytes - relocations: tuple[SectionRelocation, ...] - - -@dataclass(frozen=True) -class ResolvedTarget: - token: str - anonymous: bool - resolved: bool - addend: str = "" - nested_addends: tuple[str, ...] = () - nested_alias_partition: tuple[int, ...] = () - metadata: bool = False - - -class ArtifactSections: - """Small, conservative index of sections needed by anonymous relocations.""" - - SECTION_HEADER = re.compile(r"^Contents of section (.+):$") - FILE_FORMAT = re.compile(r"^(?P.+):\s+file format .+$") - RELOCATION_HEADER = re.compile(r"^RELOCATION RECORDS FOR \[(.+)\]:$") - RELOCATION_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+(R_[A-Z0-9_]+)\s+(.+?)\s*$") - SYMBOL_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+\S+\s+\S+\s+(\S+)\s+([0-9a-f]+)\s+(.+?)\s*$") - LOCAL_SYMBOL_ENTRY = re.compile(r"^\s*([0-9a-f]+)\s+l\s+(\S+)\s+([0-9a-f]+)\s+(\.LCPI\d+_\d+)\s*$") - - def __init__( - self, - sections: dict[tuple[str, str], SectionData], - arch: str, - symbols: dict[tuple[str, str], tuple[str, int, int]] | None = None, - ) -> None: - self.sections = sections - self.arch = arch - self.symbols = symbols or {} - - @classmethod - def from_objdump( - cls, - output: str, - arch: str, - symbols_output: str = "", - ) -> ArtifactSections: - member = "" - section_name: str | None = None - section_bytes: dict[tuple[str, str], bytearray] = {} - section_relocations: dict[tuple[str, str], list[SectionRelocation]] = {} - relocation_section: str | None = None - for line in output.splitlines(): - file_format = cls.FILE_FORMAT.match(line) - if file_format: - member = file_format.group("member") - section_name = None - relocation_section = None - continue - section = cls.SECTION_HEADER.match(line) - if section: - section_name = section.group(1) - section_bytes.setdefault((member, section_name), bytearray()) - relocation_section = None - continue - relocation_header = cls.RELOCATION_HEADER.match(line) - if relocation_header: - relocation_section = relocation_header.group(1) - section_relocations.setdefault((member, relocation_section), []) - section_name = None - continue - relocation = cls.RELOCATION_ENTRY.match(line) - if relocation and relocation_section is not None: - section_relocations[(member, relocation_section)].append( - SectionRelocation(int(relocation.group(1), 16), relocation.group(2), relocation.group(3)) - ) - continue - if section_name is None: - continue - fields = line.split() - if len(fields) < 2 or not re.fullmatch(r"[0-9a-f]+", fields[0]): - continue - chunks: list[str] = [] - for chunk_field in fields[1:5]: - if ( - len(chunk_field) in {2, 4, 6, 8} - and len(chunk_field) % 2 == 0 - and re.fullmatch(r"[0-9a-fA-F]+", chunk_field) - ): - chunks.append(chunk_field) - else: - break - if not chunks: - continue - data = section_bytes[(member, section_name)] - offset = int(fields[0], 16) - encoded = bytes.fromhex("".join(chunks)) - end = offset + len(encoded) - if len(data) < end: - data.extend(b"\0" * (end - len(data))) - data[offset:end] = encoded - sections = { - key: SectionData(key[0], key[1], bytes(data), tuple(section_relocations.get(key, ()))) - for key, data in section_bytes.items() - } - symbols: dict[tuple[str, str], tuple[str, int, int]] = {} - member = "" - for line in symbols_output.splitlines(): - file_format = cls.FILE_FORMAT.match(line) - if file_format: - member = file_format.group("member") - continue - symbol = cls.SYMBOL_ENTRY.match(line) - if symbol and symbol.group(2) not in {"*UND*", "*ABS*"}: - name = symbol.group(4).strip() - if name.startswith(".hidden "): - name = name.removeprefix(".hidden ") - symbols[(member, name)] = ( - symbol.group(2), - int(symbol.group(1), 16), - int(symbol.group(3), 16), - ) - continue - local_symbol = cls.LOCAL_SYMBOL_ENTRY.match(line) - if local_symbol: - symbols[(member, local_symbol.group(4))] = ( - local_symbol.group(2), - int(local_symbol.group(1), 16), - int(local_symbol.group(3), 16), - ) - return cls(sections, arch, symbols) - - def _find(self, member: str, name: str) -> SectionData | None: - exact = self.sections.get((member, name)) - if exact is not None: - return exact - candidates = [ - section - for (candidate_member, candidate_name), section in self.sections.items() - if candidate_member == member - and (candidate_name == name or candidate_name.endswith(f".{name}")) - ] - if len(candidates) == 1: - return candidates[0] - symbol = self.symbols.get((member, name)) - if symbol is None: - return None - symbol_section, offset, size = symbol - section = self.sections.get((member, symbol_section)) - if section is None or offset < 0 or offset > len(section.data): - return None - if size == 0 and _local_object_instance(name) is not None: - next_offsets = [ - candidate_offset - for (candidate_member, candidate_name), (candidate_section, candidate_offset, _) in self.symbols.items() - if candidate_member == member - and candidate_section == symbol_section - and candidate_name != name - and _local_object_instance(candidate_name) is not None - and candidate_offset > offset - ] - size = min(next_offsets, default=len(section.data)) - offset - if offset + size > len(section.data): - return None - relocations = tuple( - SectionRelocation(relocation.offset - offset, relocation.kind, relocation.target) - for relocation in section.relocations - if offset <= relocation.offset < offset + size - ) - return SectionData(member, symbol_section, section.data[offset : offset + size], relocations) - - @staticmethod - def _split_target(value: str) -> tuple[str, str]: - match = ADDEND.match(value.strip()) - assert match is not None - return match.group("base"), match.group("addend") or "" - - @staticmethod - def _addend(value: str) -> int: - return int(value, 16) if value.startswith("+0x") else -int(value[3:], 16) - - def _string_target(self, member: str, name: str, addend: str) -> str | None: - if not name.startswith(".rodata.str"): - return None - section = self._find(member, name) - if section is None: - return None - offset = self._addend(addend) if addend else 0 - if offset < 0 or offset >= len(section.data): - return None - value = section.data[offset:] - return value.split(b"\0", 1)[0].hex() - - def _fingerprint_details( - self, section: SectionData, member: str, anonymous_depth: int = 0 - ) -> tuple[str, tuple[str, ...], tuple[int, ...]] | None: - data = bytearray(section.data) - descriptors: list[str] = [] - section_identity = _anonymous_instance(section.name) - section_class = section_identity[0] if section_identity is not None else section.name - widths = { - "R_X86_64_64": 8, - "R_X86_64_32": 4, - "R_X86_64_PC32": 4, - "R_X86_64_PLT32": 4, - "R_X86_64_GOTPCREL": 4, - "R_AARCH64_ABS64": 8, - "R_AARCH64_ADR_PREL_PG_HI21": 4, - "R_AARCH64_ADD_ABS_LO12_NC": 4, - "R_AARCH64_LDST64_ABS_LO12_NC": 4, - } - anonymous_aliases: dict[str, int] = {} - nested_addends: list[str] = [] - nested_alias_partition: list[int] = [] - for relocation in section.relocations: - width = widths.get(relocation.kind) - if width is None or relocation.offset < 0 or relocation.offset + width > len(data): - return None - data[relocation.offset : relocation.offset + width] = b"\0" * width - target = relocation.target - if target in {"", ".text"}: - descriptor_target = target - else: - target_name, addend = self._split_target(target) - anonymous = _anonymous_instance(target_name) - local_object = _local_object_instance(target_name) - if anonymous is not None or local_object is not None: - # Deliberately support exactly one nested anonymous level. - # A nested anonymous target at this level would require a - # recursive object graph, so remain fail-closed. - if anonymous_depth >= 1: - return None - nested = self._find(member, target_name) - if nested is None: - return None - nested_details = self._fingerprint_details(nested, member, anonymous_depth + 1) - if nested_details is None: - return None - nested_fingerprint, nested_child_addends, nested_child_aliases = nested_details - # Keep the concrete target symbol here so distinct - # anonymous objects with the same stem remain distinct. - # Their absolute ordinals are normalized by encounter - # order, which tolerates compiler renumbering. - alias_key = f"{member}:{target_name}" - alias = anonymous_aliases.setdefault(alias_key, len(anonymous_aliases)) - descriptor_target = f"anonymous:{nested_fingerprint}:alias{alias}:{addend}" - nested_addends.append(addend) - nested_addends.extend(nested_child_addends) - nested_alias_partition.append(alias) - nested_alias_partition.extend(nested_child_aliases) - else: - string = self._string_target(member, target_name, addend) - descriptor_target = f"string:{string}" if string is not None else f"symbol:{target_name}{addend}" - descriptors.append(f"{relocation.offset}:{relocation.kind}:{descriptor_target}") - payload = section_class.encode() + b"\0" + bytes(data) + b"\0" + "\n".join(descriptors).encode() - return hashlib.sha256(payload).hexdigest(), tuple(nested_addends), tuple(nested_alias_partition) - - def _fingerprint(self, section: SectionData, member: str, anonymous_depth: int = 0) -> str | None: - details = self._fingerprint_details(section, member, anonymous_depth) - return details[0] if details is not None else None - - def resolve(self, member: str, kind: str, value: str) -> ResolvedTarget: - target = value.strip() - if kind in PC_RELATIVE_X86_RELOCATIONS: - target = X86_PC_BIAS.sub("", target) - anonymous = _anonymous_instance(target) - local_object = _local_object_instance(target) - if anonymous is None and local_object is None: - name, addend = self._split_target(target) - string = self._string_target(member, name, addend) - token = f"string:{string}" if string is not None else f"symbol:{name}{addend}" - return ResolvedTarget(token, False, True, addend) - name, addend = self._split_target(target) - section = self._find(member, name) - if section is None: - object_name = anonymous[0] if anonymous is not None else name - return ResolvedTarget( - f"anonymous-unresolved:{object_name}", True, False, addend, metadata=anonymous is not None - ) - details = self._fingerprint_details(section, member) - if details is None: - object_name = anonymous[0] if anonymous is not None else name - return ResolvedTarget( - f"anonymous-unresolved:{object_name}", True, False, addend, metadata=anonymous is not None - ) - fingerprint, nested_addends, nested_alias_partition = details - return ResolvedTarget( - f"anonymous:{fingerprint}", - True, - True, - addend, - nested_addends, - nested_alias_partition, - anonymous is not None, - ) - - -@dataclass(frozen=True) -class Function: - instructions: tuple[str, ...] - encoded: bytes - calls: int - branches: int - executable: tuple[str, ...] = () - relocations: tuple[Relocation, ...] = () - alias_partition: tuple[int, ...] = () - - @property - def digest(self) -> str: - return hashlib.sha256("\n".join(self.executable).encode()).hexdigest()[:16] - - -@dataclass(frozen=True) -class Root: - symbol: str - artifact: str - targets: frozenset[str] - - -def classify(old: Function, new: Function) -> str: - if old.executable != new.executable: - return "CODEGEN CHANGED" - if old.encoded != new.encoded: - return "CODEGEN ENCODING CHANGED" - if _same_relocations(old.relocations, new.relocations) and old.alias_partition == new.alias_partition: - return "CODEGEN IDENTICAL" - if _metadata_only_relocations(old, new): - return "CODEGEN METADATA CHANGED" - return "CODEGEN CHANGED" - - -def _same_relocations(old: tuple[Relocation, ...], new: tuple[Relocation, ...]) -> bool: - return all( - before.offset == after.offset - and before.instruction_index == after.instruction_index - and before.kind == after.kind - and before.token == after.token - and before.anonymous == after.anonymous - and before.resolved == after.resolved - and before.addend == after.addend - and before.nested_addends == after.nested_addends - and before.nested_alias_partition == after.nested_alias_partition - and before.metadata == after.metadata - and (not before.anonymous or before.resolved) - for before, after in zip(old, new) - ) and len(old) == len(new) - - -def _metadata_only_relocations(old: Function, new: Function) -> bool: - if old.alias_partition != new.alias_partition or len(old.relocations) != len(new.relocations): - return False - changed = False - for before, after in zip(old.relocations, new.relocations): - if ( - before.offset != after.offset - or before.instruction_index != after.instruction_index - or before.kind != after.kind - or before.anonymous != after.anonymous - or before.addend != after.addend - or before.metadata != after.metadata - ): - return False - if ( - (before.nested_addends != after.nested_addends or before.nested_alias_partition != after.nested_alias_partition) - and not _anonymous_wrapper_representation_changed(before, after) - ): - return False - if before.token != after.token: - if not before.anonymous or not before.resolved or not after.resolved or not before.metadata: - return False - changed = True - return changed - - -def _anonymous_instance(value: str) -> tuple[str, int, str] | None: - match = ANONYMOUS_SYMBOL_INSTANCE.search(value) - if match is None: - return None - key = value[: match.start("stem")] + match.group("stem") - return key, int(match.group("instance")), match.group("addend") or "" - - -def _anonymous_wrapper_representation_changed(before: Relocation, after: Relocation) -> bool: - """Allow rustc to flatten a single anonymous metadata wrapper.""" - - def style(value: str) -> str: - value = value.strip() - if value.startswith("anon."): - return "llvm" - if ".Lanon." in value: - return "section" - return "other" - - return ( - {style(before.raw_target), style(after.raw_target)} == {"llvm", "section"} - and {before.nested_addends, after.nested_addends} == {(), ("",)} - and {before.nested_alias_partition, after.nested_alias_partition} == {(), (0,)} - ) - - -def _local_object_instance(value: str) -> tuple[str, str] | None: - match = LOCAL_OBJECT_SYMBOL.fullmatch(value.strip()) - if match is None: - return None - return match.group("name"), match.group("addend") or "" - - -def _relocation_target(kind: str, value: str, anonymous_bases: dict[str, int]) -> str: - value = value.strip() - if kind in PC_RELATIVE_X86_RELOCATIONS: - value = X86_PC_BIAS.sub("", value) - anonymous = _anonymous_instance(value) - if anonymous is None: - return value - key, instance, addend = anonymous - base = anonymous_bases[key] - return f"{key}.@{instance - base}{addend}" - - -def _is_call(mnemonic: str, arch: str) -> bool: - return mnemonic.startswith("call") if arch == "x86_64" else mnemonic in {"bl", "blr"} - - -def _is_conditional_branch(mnemonic: str, arch: str) -> bool: - if arch == "x86_64": - return mnemonic.startswith("j") and mnemonic not in {"jmp", "jmpq"} - return mnemonic.startswith("b.") or mnemonic in {"cbz", "cbnz", "tbz", "tbnz"} - - -def _has_control_target(mnemonic: str, arch: str) -> bool: - if arch == "x86_64": - return mnemonic.startswith(("j", "call", "loop")) - return mnemonic in {"b", "bl", "blr", "br", "cbz", "cbnz", "tbz", "tbnz"} or mnemonic.startswith("b.") - - -def _normalize(instructions: list[Instruction], arch: str, sections: ArtifactSections | None = None) -> Function: - indexes = {instruction.address: index for index, instruction in enumerate(instructions)} - anonymous_bases: dict[str, int] = {} - for instruction in instructions: - for _, _, target in instruction.relocations: - anonymous = _anonymous_instance(target.strip()) - if anonymous is None: - continue - key, instance, _ = anonymous - anonymous_bases[key] = min(instance, anonymous_bases.get(key, instance)) - normalized: list[str] = [] - executable: list[str] = [] - relocations: list[Relocation] = [] - aliases: dict[str, int] = {} - alias_partition: list[int] = [] - calls = branches = 0 - for instruction_index, instruction in enumerate(instructions): - assembly = OBJDUMP_COMMENT.sub("", instruction.assembly).strip() - if not assembly: - continue - parts = assembly.split(None, 1) - mnemonic = parts[0] - operands = parts[1] if len(parts) == 2 else "" - calls += _is_call(mnemonic, arch) - branches += _is_conditional_branch(mnemonic, arch) - if _has_control_target(mnemonic, arch): - match = SYMBOL_TARGET.search(operands) - if match: - address = int(match.group(1), 16) - symbol = match.group(2) - target = f"@instruction_{indexes[address]}" if address in indexes and "+0x" in symbol else f"<{symbol}>" - operands = SYMBOL_TARGET.sub(target, operands) - relocation_text: list[str] = [] - for relocation_offset, kind, target in instruction.relocations: - if sections is None: - token = _relocation_target(kind, target, anonymous_bases) - anonymous = _anonymous_instance(target) is not None - resolved = True - addend = _anonymous_instance(target)[2] if anonymous else "" - nested_addends = () - nested_alias_partition = () - metadata = anonymous - else: - resolved_target = sections.resolve(instruction.member, kind, target) - token = resolved_target.token - anonymous = resolved_target.anonymous - resolved = resolved_target.resolved - addend = resolved_target.addend - nested_addends = resolved_target.nested_addends - nested_alias_partition = resolved_target.nested_alias_partition - metadata = resolved_target.metadata - relocation_text.append(f"[{kind}:{token}]") - relocations.append( - Relocation( - relocation_offset, - kind, - target, - token, - anonymous, - resolved, - instruction_index, - addend, - nested_addends, - nested_alias_partition, - metadata, - ) - ) - if anonymous: - alias_target = X86_PC_BIAS.sub("", target) if kind in PC_RELATIVE_X86_RELOCATIONS else target - alias_key, _ = ArtifactSections._split_target(alias_target) - if kind in PC_RELATIVE_X86_RELOCATIONS: - alias_key = X86_PC_BIAS.sub("", alias_key) - alias = aliases.setdefault(f"{instruction.member}:{alias_key}", len(aliases)) - alias_partition.append(alias) - executable.append(" ".join(f"{mnemonic} {operands}".split())) - normalized.append(" ".join(f"{mnemonic} {operands} {' '.join(relocation_text)}".split())) - return Function( - tuple(normalized), - b"".join(item.encoded for item in instructions), - calls, - branches, - tuple(executable), - tuple(relocations), - tuple(alias_partition), - ) - - -def parse_objdump(output: str, arch: str, sections: ArtifactSections | None = None) -> dict[str, Function]: - instruction_pattern = X86_INSTRUCTION if arch == "x86_64" else AARCH64_INSTRUCTION - raw: dict[str, list[Instruction]] = {} - current: str | None = None - current_member = "" - last_instruction: Instruction | None = None - for line in output.splitlines(): - file_format = ArtifactSections.FILE_FORMAT.match(line) - if file_format: - current_member = file_format.group("member") - continue - label = FUNCTION.match(line) - if label: - current = label.group(1) - if current in raw: - raise ValueError(f"duplicate disassembly symbol: {current}") - raw[current] = [] - last_instruction = None - continue - if current is None: - continue - relocation = RELOCATION.match(line) - if relocation: - if last_instruction is not None: - relocation_offset = int(relocation.group(1), 16) - last_instruction.address - last_instruction.relocations.append((relocation_offset, relocation.group(2), relocation.group(3))) - continue - match = instruction_pattern.match(line) - if not match: - continue - encoded = bytes.fromhex(match.group(2)) - assembly = match.group(3).strip() - relocations: list[tuple[int, str, str]] = [] - inline_relocation = INLINE_RELOCATION.search(assembly) - if inline_relocation is not None: - assembly = assembly[: inline_relocation.start()].rstrip() - relocation_offset = int(inline_relocation.group(1), 16) - int(match.group(1), 16) - relocations.append((relocation_offset, inline_relocation.group(2), inline_relocation.group(3))) - if not assembly: - continue - last_instruction = Instruction(int(match.group(1), 16), encoded, assembly, relocations, current_member) - raw[current].append(last_instruction) - return {name: _normalize(items, arch, sections) for name, items in raw.items() if items} - - -def load_functions(artifact: Path, objdump: str, arch: str) -> dict[str, Function]: - output = subprocess.check_output([objdump, "-drwC", str(artifact)], text=True, errors="replace") - section_output = subprocess.check_output([objdump, "-sr", str(artifact)], text=True, errors="replace") - symbol_output = subprocess.check_output([objdump, "-t", str(artifact)], text=True, errors="replace") - sections = ArtifactSections.from_objdump(section_output, arch, symbol_output) - return parse_objdump(output, arch, sections) - - -def load_roots(path: Path) -> list[Root]: - data = tomllib.loads(path.read_text(encoding="utf-8")) - roots = [Root(item["symbol"], item["artifact"], frozenset(item["targets"])) for item in data.get("root", [])] - identities = [(root.artifact, root.symbol) for root in roots] - if len(identities) != len(set(identities)): - raise ValueError(f"{path}: duplicate codegen root") - unknown_targets = set().union(*(root.targets for root in roots)) - CODEGEN_CONFIGS.keys() - if unknown_targets: - raise ValueError(f"{path}: unknown target configurations: {sorted(unknown_targets)}") - return roots - - -def load_kernel_modules(path: Path) -> tuple[str, ...]: - data = tomllib.loads(path.read_text(encoding="utf-8")) - modules = [] - for kernel in data.get("kernel", []): - source = Path(kernel["path"]) - if source.parts[0] != "src" or source.suffix != ".rs": - raise ValueError(f"{path}: invalid kernel source path: {source}") - modules.append("bit_string::" + "::".join(source.with_suffix("").parts[1:])) - return tuple(modules) - - -def load_kernel_caller_modules(path: Path) -> tuple[str, ...]: - data = tomllib.loads(path.read_text(encoding="utf-8")) - modules = [] - for kernel in data.get("kernel", []): - if kernel.get("coverage") != "caller": - continue - source = Path(kernel["path"]) - modules.append("bit_string::" + "::".join(source.with_suffix("").parts[1:])) - return tuple(modules) - - -def independent_backend_symbols(functions: dict[str, Function], modules: tuple[str, ...]) -> set[str]: - return { - symbol - for symbol in functions - if BACKEND_MODULE.search(symbol) and any(symbol.startswith(f"{module}::") for module in modules) - } - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("baseline", type=Path) - parser.add_argument("current", type=Path) - parser.add_argument("--roots", type=Path, required=True) - parser.add_argument("--artifact", required=True) - parser.add_argument("--inventory", type=Path) - parser.add_argument("--target", required=True) - parser.add_argument("--arch", choices=("x86_64", "aarch64"), required=True) - parser.add_argument("--objdump", default="objdump") - parser.add_argument("--strict-metadata", action="store_true") - args = parser.parse_args() - before = load_functions(args.baseline, args.objdump, args.arch) - after = load_functions(args.current, args.objdump, args.arch) - roots = [root for root in load_roots(args.roots) if root.artifact == args.artifact] - identical = metadata_changed = encoding_changed = changed = errors = skipped = 0 - topology_changed = 0 - caller_covered_modules: tuple[str, ...] = () - if args.artifact == "library": - if args.inventory is None: - parser.error("--inventory is required for library artifacts") - modules = load_kernel_modules(args.inventory) - caller_covered_modules = load_kernel_caller_modules(args.inventory) - registered = {root.symbol for root in roots if args.target in root.targets} - discovered = independent_backend_symbols(before, modules) | independent_backend_symbols(after, modules) - for symbol in sorted(discovered - registered): - errors += 1 - print(f"ERROR: independent backend symbol is not a configured root: {symbol}", file=sys.stderr) - for root in roots: - if args.target not in root.targets: - skipped += 1 - continue - old, new = before.get(root.symbol), after.get(root.symbol) - if old is None or new is None: - if ( - args.artifact == "library" - and old is not None - and new is None - and BACKEND_MODULE.search(root.symbol) - and any(root.symbol.startswith(f"{module}::") for module in caller_covered_modules) - ): - topology_changed += 1 - print(f"{root.symbol}: CODEGEN TOPOLOGY CHANGED") - print(" independent backend symbol disappeared; caller coverage remains required") - continue - errors += 1 - where = "both" if old is None and new is None else "baseline" if old is None else "current" - print(f"ERROR: expected symbol missing in {where}: {root.symbol}", file=sys.stderr) - continue - same_instructions = old.instructions == new.instructions - status = classify(old, new) - if status == "CODEGEN IDENTICAL": - identical += 1 - elif status == "CODEGEN METADATA CHANGED": - metadata_changed += 1 - elif status == "CODEGEN ENCODING CHANGED": - encoding_changed += 1 - else: - changed += 1 - print(f"{root.symbol}: {status}") - if status == "CODEGEN METADATA CHANGED": - print(f"WARNING: {root.symbol} has metadata-only codegen changes", file=sys.stderr) - print(f" text bytes {len(old.encoded)} -> {len(new.encoded)}") - print(f" instructions {len(old.instructions)} -> {len(new.instructions)}") - print(f" calls {old.calls} -> {new.calls}") - print(f" branches {old.branches} -> {new.branches}") - print(f" asm hash {old.digest} -> {new.digest}") - if status == "CODEGEN METADATA CHANGED": - for old_relocation, new_relocation in zip(old.relocations, new.relocations): - if old_relocation.token != new_relocation.token: - print(f" metadata {old_relocation.raw_target} -> {new_relocation.raw_target}") - print(f" semantic {old_relocation.token} -> {new_relocation.token}") - if new.calls > old.calls: - print(f"WARNING: {root.symbol} call count increased from {old.calls} to {new.calls}", file=sys.stderr) - if not same_instructions: - print( - "\n".join( - difflib.unified_diff(old.instructions, new.instructions, "baseline", "current", lineterm="") - ) - ) - print("\nCODEGEN SUMMARY") - print(f"compared : {identical + metadata_changed + encoding_changed + changed}") - print(f"identical : {identical}") - print(f"metadata changed : {metadata_changed}") - print(f"encoding changed : {encoding_changed}") - print(f"changed : {changed}") - print(f"topology changed : {topology_changed}") - print(f"errors : {errors}") - print(f"skipped : {skipped}") - failed = encoding_changed or changed or errors or (args.strict_metadata and metadata_changed) - return 0 if not failed else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/kernel_inventory.toml b/scripts/kernel_inventory.toml deleted file mode 100644 index 714aa1b..0000000 --- a/scripts/kernel_inventory.toml +++ /dev/null @@ -1,97 +0,0 @@ -## Each entry records a stable caller observation root. Backend functions that -## remain as independent symbols require separate library roots. - -[[kernel]] -path = "src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_from_bool_slice"] - -[[kernel]] -path = "src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_from_str"] - -[[kernel]] -path = "src/traits/words_arith/funcs_for_binary_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_and", "codegen_bit_string_or", "codegen_bit_string_xor"] - -[[kernel]] -path = "src/traits/words_arith/funcs_for_not_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_not"] - -[[kernel]] -path = "src/traits/words_arith/funcs_for_shl_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_shl"] - -[[kernel]] -path = "src/traits/words_arith/funcs_for_shr_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_shr"] - -[[kernel]] -path = "src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_slice_from"] - -[[kernel]] -path = "src/traits/words_eq/funcs_for_eq_words_aligned_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_matches_at"] - -[[kernel]] -path = "src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_matches_at"] - -[[kernel]] -path = "src/traits/words_find/funcs_for_contains_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_contains"] - -[[kernel]] -path = "src/traits/words_find/funcs_for_find_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_find"] - -[[kernel]] -path = "src/traits/words_find/funcs_for_rfind_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_rfind"] - -[[kernel]] -path = "src/traits/words_ord/funcs_for_cmp_aligned_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_cmp"] - -[[kernel]] -path = "src/traits/words_ord/funcs_for_cmp_unaligned_core.rs" -coverage = "caller" -roots = ["codegen_bit_string_cmp"] - -[[kernel]] -path = "src/traits/words_scan/funcs_for_count_ones.rs" -coverage = "caller" -roots = ["codegen_bit_string_count_ones"] - -[[kernel]] -path = "src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs" -coverage = "caller" -roots = [ - "codegen_bit_string_leading_zeros", - "codegen_bit_string_leading_ones", - "codegen_bit_str_leading_zeros", - "codegen_bit_str_leading_ones", -] - -[[kernel]] -path = "src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs" -coverage = "caller" -roots = [ - "codegen_bit_string_trailing_zeros", - "codegen_bit_string_trailing_ones", - "codegen_bit_str_trailing_zeros", - "codegen_bit_str_trailing_ones", -] diff --git a/scripts/run_codegen_comparison.py b/scripts/run_codegen_comparison.py deleted file mode 100755 index d2d7e7d..0000000 --- a/scripts/run_codegen_comparison.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 -"""Build fixed-feature baseline/current artifacts and compare their codegen.""" - -from __future__ import annotations - -import argparse -import atexit -import os -import platform -import shlex -import subprocess -import sys -import tempfile -from pathlib import Path - -from bench_config import BASELINE_SHA, CODEGEN_CONFIGS - - -def run(command: list[str], cwd: Path, env: dict[str, str]) -> None: - print(f"[{cwd}] {' '.join(command)}", flush=True) - completed = subprocess.run(command, cwd=cwd, env=env, check=False) - if completed.returncode: - raise SystemExit(completed.returncode) - - -def build_library(worktree: Path, target_dir: Path, rustflags: str) -> Path: - env = os.environ.copy() - env["RUSTFLAGS"] = rustflags - env["CARGO_TARGET_DIR"] = str(target_dir) - run(["cargo", "build", "--release", "--lib"], worktree, env) - return target_dir / "release" / "libbit_string.rlib" - - -def materialize_harness(root: Path, output: Path, revision: str) -> Path: - source = output.with_suffix(".rs") - source.write_bytes(subprocess.check_output(["git", "show", f"{revision}:scripts/codegen_harness.rs"], cwd=root)) - return source - - -def build_harness(root: Path, library: Path, output: Path, rustflags: str, revision: str) -> None: - dependency_dir = library.parent / "deps" - source = materialize_harness(root, output, revision) - command = [ - "rustc", - str(source), - "--crate-name", - "bit_string_codegen_harness", - "--crate-type=lib", - "--edition=2024", - "-O", - "--emit=obj", - "-o", - str(output), - "--extern", - f"bit_string={library}", - "-L", - f"dependency={dependency_dir}", - *shlex.split(rustflags), - ] - run(command, root, os.environ.copy()) - - -def compare( - root: Path, - baseline: Path, - current: Path, - artifact: str, - target: str, - arch: str, - strict_metadata: bool, -) -> int: - command = [ - sys.executable, - str(root / "scripts" / "compare_codegen.py"), - str(baseline), - str(current), - "--roots", - str(root / "scripts" / "codegen_roots.toml"), - "--artifact", - artifact, - "--inventory", - str(root / "scripts" / "kernel_inventory.toml"), - "--target", - target, - "--arch", - arch, - ] - if strict_metadata: - command.append("--strict-metadata") - return subprocess.run(command, cwd=root, check=False).returncode - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--baseline", default=BASELINE_SHA) - parser.add_argument("--config", choices=sorted(CODEGEN_CONFIGS), action="append") - parser.add_argument("--strict-metadata", action="store_true") - args = parser.parse_args() - root = Path(__file__).resolve().parents[1] - current_revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() - output = root / "target" / "codegen-comparison" - output.mkdir(parents=True, exist_ok=True) - baseline_worktree = Path(tempfile.mkdtemp(prefix="bit-string-codegen-baseline-")) - baseline_worktree.rmdir() - run(["git", "worktree", "add", "--detach", str(baseline_worktree), args.baseline], root, os.environ.copy()) - - def cleanup_worktree() -> None: - subprocess.run( - ["git", "worktree", "remove", "--force", str(baseline_worktree)], - cwd=root, - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - atexit.register(cleanup_worktree) - host_arch = "aarch64" if platform.machine() in {"aarch64", "arm64"} else "x86_64" - configs = args.config or [name for name, (arch, _) in CODEGEN_CONFIGS.items() if arch == host_arch] - failed = False - for config in configs: - arch, rustflags = CODEGEN_CONFIGS[config] - if arch != host_arch: - print(f"ERROR: {config} requires {arch}, but host architecture is {host_arch}", file=sys.stderr) - return 2 - config_dir = output / config - baseline_library = build_library(baseline_worktree, config_dir / "baseline-target", rustflags) - current_library = build_library(root, config_dir / "current-target", rustflags) - baseline_harness = config_dir / "baseline-harness.o" - current_harness = config_dir / "current-harness.o" - build_harness(root, baseline_library, baseline_harness, rustflags, current_revision) - build_harness(root, current_library, current_harness, rustflags, current_revision) - failed |= compare( - root, baseline_harness, current_harness, "harness", config, arch, args.strict_metadata - ) != 0 - failed |= compare( - root, baseline_library, current_library, "library", config, arch, args.strict_metadata - ) != 0 - return int(failed) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/tests/test_codegen.py b/scripts/tests/test_codegen.py deleted file mode 100644 index c63a7cd..0000000 --- a/scripts/tests/test_codegen.py +++ /dev/null @@ -1,937 +0,0 @@ -from __future__ import annotations - -import contextlib -import io -import sys -import tempfile -import tomllib -import unittest -from pathlib import Path -from unittest import mock - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -import compare_codegen -import run_codegen_comparison - - -def parse(body: str, arch: str = "x86_64") -> compare_codegen.Function: - return compare_codegen.parse_objdump(body, arch)["root"] - - -def parse_with_sections( - body: str, sections: str, arch: str = "x86_64", symbols: str = "" -) -> compare_codegen.Function: - index = compare_codegen.ArtifactSections.from_objdump(sections, arch, symbols) - return compare_codegen.parse_objdump(body, arch, index)["root"] - - -class CodegenParserTests(unittest.TestCase): - SECTION_PREFIX = ( - "fixture.o: file format elf64-x86-64\n" - "Contents of section .rodata.str1.1:\n" - " 0000 66696c65 2e727300 file.rs.\n" - ) - - @classmethod - def anonymous_section(cls, ordinal: int, line: int = 1, column: int = 2) -> str: - line_bytes = line.to_bytes(4, "little").hex() - column_bytes = column.to_bytes(4, "little").hex() - return ( - f"Contents of section .data.rel.ro..Lanon.hash.{ordinal}:\n" - f" 0000 00000000 00000000 {line_bytes} {column_bytes} ........\n" - "RELOCATION RECORDS FOR [.data.rel.ro..Lanon.hash." - f"{ordinal}]:\n" - "OFFSET TYPE VALUE\n" - "0000000000000000 R_X86_64_64 .rodata.str1.1+0x0000000000000000\n" - ) - - def test_anonymous_section_content_ignores_ordinal(self) -> None: - old_sections = self.SECTION_PREFIX + self.anonymous_section(1) - new_sections = self.SECTION_PREFIX + self.anonymous_section(9) - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", - old_sections, - ) - new = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.9-0x4\n", - new_sections, - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") - - @staticmethod - def nested_sections( - outer_ordinal: int, - nested_ordinal: int, - nested_value: int = 1, - nested_addend: str = "+0x0", - ) -> str: - nested_name = f".data.rel.ro..Lanon.nested.{nested_ordinal}" - outer_name = f".data.rel.ro..Lanon.outer.{outer_ordinal}" - result = ( - "fixture.o: file format elf64-x86-64\n" - f"Contents of section {nested_name}:\n" - f" 0000 {nested_value & 0xffffffff:08x} 00000000 ........\n" - f"Contents of section {outer_name}:\n" - " 0000 00000000 00000000 ........\n" - f"RELOCATION RECORDS FOR [{outer_name}]:\n" - "OFFSET TYPE VALUE\n" - f"0000000000000000 R_X86_64_64 {nested_name}{nested_addend}\n" - ) - return result - - def nested_root(self, outer_ordinal: int, addend: str = "+0x0") -> str: - return ( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - f" 3: R_X86_64_PC32 .data.rel.ro..Lanon.outer.{outer_ordinal}{addend}\n" - ) - - def test_one_level_nested_ordinal_change_is_identical(self) -> None: - old = parse_with_sections(self.nested_root(1), self.nested_sections(1, 2)) - new = parse_with_sections(self.nested_root(9), self.nested_sections(9, 8)) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") - - def test_nested_semantic_content_change_is_metadata_change(self) -> None: - old = parse_with_sections(self.nested_root(1), self.nested_sections(1, 2, nested_value=1)) - new = parse_with_sections(self.nested_root(9), self.nested_sections(9, 8, nested_value=2)) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN METADATA CHANGED") - - def test_nested_addend_change_is_not_metadata(self) -> None: - old = parse_with_sections(self.nested_root(1), self.nested_sections(1, 2, nested_addend="+0x0")) - new = parse_with_sections(self.nested_root(9), self.nested_sections(9, 8, nested_addend="+0x8")) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_nested_alias_partition_is_preserved(self) -> None: - def sections(outer_ordinal: int, first: int, second: int) -> str: - outer = f".data.rel.ro..Lanon.outer.{outer_ordinal}" - first_name = f".data.rel.ro..Lanon.nested.{first}" - second_name = f".data.rel.ro..Lanon.nested.{second}" - return ( - "fixture.o: file format elf64-x86-64\n" - f"Contents of section {first_name}:\n" - " 0000 01000000 00000000 ........\n" - + ( - f"Contents of section {second_name}:\n" - " 0000 01000000 00000000 ........\n" - if second != first - else "" - ) - + f"Contents of section {outer}:\n" - " 0000 00000000 00000000 00000000 00000000 ................\n" - f"RELOCATION RECORDS FOR [{outer}]:\n" - "OFFSET TYPE VALUE\n" - f"0000000000000000 R_X86_64_64 {first_name}+0x0\n" - f"0000000000000008 R_X86_64_64 {second_name}+0x0\n" - ) - - old = parse_with_sections(self.nested_root(1), sections(1, 2, 2)) - new = parse_with_sections(self.nested_root(9), sections(9, 8, 9)) - self.assertTrue(old.relocations[0].resolved) - self.assertTrue(new.relocations[0].resolved) - self.assertEqual(old.relocations[0].nested_alias_partition, (0, 0)) - self.assertEqual(new.relocations[0].nested_alias_partition, (0, 1)) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - same_alias = parse_with_sections(self.nested_root(9), sections(9, 8, 8)) - self.assertTrue(same_alias.relocations[0].resolved) - self.assertEqual(same_alias.relocations[0].nested_alias_partition, (0, 0)) - self.assertEqual(compare_codegen.classify(old, same_alias), "CODEGEN IDENTICAL") - - def test_second_level_nested_anonymous_is_conservative(self) -> None: - inner = ".data.rel.ro..Lanon.inner.3" - nested = ".data.rel.ro..Lanon.nested.2" - outer = ".data.rel.ro..Lanon.outer.1" - sections = ( - "fixture.o: file format elf64-x86-64\n" - f"Contents of section {inner}:\n" - " 0000 00000000 00000000 ........\n" - f"Contents of section {nested}:\n" - " 0000 00000000 00000000 ........\n" - f"RELOCATION RECORDS FOR [{nested}]:\n" - "OFFSET TYPE VALUE\n" - f"0000000000000000 R_X86_64_64 {inner}+0x0\n" - f"Contents of section {outer}:\n" - " 0000 00000000 00000000 ........\n" - f"RELOCATION RECORDS FOR [{outer}]:\n" - "OFFSET TYPE VALUE\n" - f"0000000000000000 R_X86_64_64 {nested}+0x0\n" - ) - old = parse_with_sections(self.nested_root(1), sections) - self.assertFalse(old.relocations[0].resolved) - self.assertEqual(compare_codegen.classify(old, old), "CODEGEN CHANGED") - - def test_nested_target_does_not_cross_archive_members(self) -> None: - outer = ".data.rel.ro..Lanon.outer.1" - nested = ".data.rel.ro..Lanon.nested.2" - sections = ( - "member-a.o: file format elf64-x86-64\n" - f"Contents of section {outer}:\n" - " 0000 00000000 00000000 ........\n" - f"RELOCATION RECORDS FOR [{outer}]:\n" - "OFFSET TYPE VALUE\n" - f"0000000000000000 R_X86_64_64 {nested}+0x0\n" - "member-b.o: file format elf64-x86-64\n" - f"Contents of section {nested}:\n" - " 0000 01000000 00000000 ........\n" - ) - index = compare_codegen.ArtifactSections.from_objdump(sections, "x86_64") - resolved = index.resolve("member-a.o", "R_X86_64_PC32", outer) - self.assertFalse(resolved.resolved) - - def test_relocation_site_change_is_not_ignored(self) -> None: - old = parse( - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 foo-0x4\n" - " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax" - ) - new = parse( - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " a: R_X86_64_PC32 foo-0x4" - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_relocation_offset_change_is_not_ignored(self) -> None: - old = parse( - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 foo-0x4" - ) - new = parse( - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 4: R_X86_64_PC32 foo-0x4" - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_anonymous_section_content_change_is_metadata_change(self) -> None: - old_sections = self.SECTION_PREFIX + self.anonymous_section(1, line=1) - new_sections = self.SECTION_PREFIX + self.anonymous_section(9, line=3) - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", - old_sections, - ) - new = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.9-0x4\n", - new_sections, - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN METADATA CHANGED") - - def test_anonymous_section_alias_partition_is_preserved(self) -> None: - sections = self.SECTION_PREFIX + self.anonymous_section(1) + self.anonymous_section(2) - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n" - " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " a: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", - sections, - ) - new = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n" - " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " a: R_X86_64_PC32 .data.rel.ro..Lanon.hash.2-0x4\n", - sections, - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_unresolved_anonymous_section_is_conservative(self) -> None: - sections = "fixture.o: file format elf64-x86-64\n" - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", - sections, - ) - new = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.9-0x4\n", - sections, - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_named_relocation_change_is_not_metadata(self) -> None: - old = parse( - "0000 :\n" - " 0: e8 00 00 00 00 call 5 \n" - " 1: R_X86_64_PLT32 foo-0x4" - ) - new = parse( - "0000 :\n" - " 0: e8 00 00 00 00 call 5 \n" - " 1: R_X86_64_PLT32 bar-0x4" - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_relocation_addend_change_is_not_metadata(self) -> None: - old = parse( - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .rodata+0x20" - ) - new = parse( - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .rodata+0x40" - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_aarch64_anonymous_section_identity_ignores_ordinal(self) -> None: - def sections(ordinal: int) -> str: - return ( - "fixture.o: file format elf64-littleaarch64\n" - "Contents of section .rodata.str1.1:\n" - " 0000 66696c65 2e727300 file.rs.\n" - f"Contents of section .data.rel.ro..Lanon.hash.{ordinal}:\n" - " 0000 00000000 00000000 01000000 02000000 ........\n" - f"RELOCATION RECORDS FOR [.data.rel.ro..Lanon.hash.{ordinal}]:\n" - "OFFSET TYPE VALUE\n" - "0000000000000000 R_AARCH64_ABS64 .rodata.str1.1+0x0000000000000000\n" - ) - - old = parse_with_sections( - "fixture.o: file format elf64-littleaarch64\n" - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 \n" - " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.1\n", - sections(1), - "aarch64", - ) - new = parse_with_sections( - "fixture.o: file format elf64-littleaarch64\n" - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 \n" - " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.9\n", - sections(9), - "aarch64", - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") - - def test_anonymous_addend_change_is_not_metadata(self) -> None: - sections = self.SECTION_PREFIX + self.anonymous_section(1) - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1+0x0", - sections, - ) - new = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1+0x8", - sections, - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_alias_identity_excludes_addend(self) -> None: - sections = ( - self.SECTION_PREFIX - + self.anonymous_section(1) - + self.anonymous_section(2) - + self.anonymous_section(3) - ) - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1+0x0\n" - " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " a: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1+0x8", - sections, - ) - new = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.2+0x0\n" - " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " a: R_X86_64_PC32 .data.rel.ro..Lanon.hash.3+0x8", - sections, - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN CHANGED") - - def test_section_class_is_part_of_anonymous_identity(self) -> None: - old_sections = self.SECTION_PREFIX + self.anonymous_section(1) - new_sections = self.SECTION_PREFIX + self.anonymous_section(9).replace( - ".data.rel.ro..Lanon", ".rodata..Lanon" - ) - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4", - old_sections, - ) - new = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .rodata..Lanon.hash.9-0x4", - new_sections, - ) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN METADATA CHANGED") - - def test_section_lookup_does_not_cross_archive_members(self) -> None: - sections = ( - "member-a.o: file format elf64-x86-64\n" - "member-b.o: file format elf64-x86-64\n" - "Contents of section .data.rel.ro..Lanon.hash.1:\n" - " 0000 00000000 00000000 01000000 02000000 ........\n" - ) - index = compare_codegen.ArtifactSections.from_objdump(sections, "x86_64") - resolved = index.resolve("member-a.o", "R_X86_64_PC32", ".data.rel.ro..Lanon.hash.1-0x4") - self.assertFalse(resolved.resolved) - - def test_section_bytes_ignore_ascii_column_after_four_groups(self) -> None: - output = ( - "fixture.o: file format elf64-x86-64\n" - "Contents of section .rodata:\n" - " 0000 00000000 00000000 00000000 00000000 deadbeef\n" - ) - index = compare_codegen.ArtifactSections.from_objdump(output, "x86_64") - section = index.sections[("fixture.o", ".rodata")] - self.assertEqual(section.data, b"\0" * 16) - - def test_lcpi_symbol_ordinal_change_uses_object_content(self) -> None: - def sections(symbol: str, value: str) -> tuple[str, str]: - body = ( - "fixture.o: file format elf64-x86-64\n" - "Contents of section .rodata.cst16:\n" - f" 0000 {value} 00000000 00000000 00000000 ................\n" - ) - symbols = ( - "fixture.o: file format elf64-x86-64\n\n" - "SYMBOL TABLE:\n" - f"0000000000000000 l .rodata.cst16\t0000000000000010 {symbol}\n" - ) - return body, symbols - - old_sections, old_symbols = sections(".LCPI13_0", "01020304") - new_sections, new_symbols = sections(".LCPI11_0", "01020304") - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .LCPI13_0-0x4\n", - old_sections, - symbols=old_symbols, - ) - new = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .LCPI11_0-0x4\n", - new_sections, - symbols=new_symbols, - ) - self.assertTrue(old.relocations[0].resolved) - self.assertTrue(new.relocations[0].resolved) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN IDENTICAL") - - def test_lcpi_content_change_is_not_ignored(self) -> None: - def make(value: str) -> compare_codegen.Function: - sections = ( - "fixture.o: file format elf64-x86-64\n" - "Contents of section .rodata.cst16:\n" - f" 0000 {value} 00000000 00000000 00000000 ................\n" - ) - symbols = ( - "fixture.o: file format elf64-x86-64\n\n" - "SYMBOL TABLE:\n" - "0000000000000000 l .rodata.cst16\t0000000000000010 .LCPI13_0\n" - ) - return parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .LCPI13_0-0x4\n", - sections, - symbols=symbols, - ) - - self.assertEqual(compare_codegen.classify(make("01020304"), make("05060708")), "CODEGEN CHANGED") - - def test_rustc_anonymous_representation_change_is_metadata_only(self) -> None: - def function( - raw_target: str, - token: str, - nested_addends: tuple[str, ...], - aliases: tuple[int, ...], - ) -> compare_codegen.Function: - relocation = compare_codegen.Relocation( - 3, - "R_X86_64_PC32", - raw_target, - token, - True, - True, - 0, - "-0x4", - nested_addends, - aliases, - True, - ) - return compare_codegen.Function( - ("lea %rax",), - b"\x48\x8d\x05\0\0\0\0", - 0, - 0, - ("lea %rax",), - (relocation,), - (0,), - ) - - old = function("anon.hash.20.llvm.123-0x4", "anonymous:old", ("",), (0,)) - new = function(".data.rel.ro..Lanon.hash.20-0x4", "anonymous:new", (), ()) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN METADATA CHANGED") - def test_call_target_identity_is_preserved(self) -> None: - foo = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 foo-0x4") - bar = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 bar-0x4") - self.assertNotEqual(foo.instructions, bar.instructions) - - def test_local_branch_uses_target_instruction_index(self) -> None: - old = parse("0000 :\n 0: 74 02 je 4 \n 2: 90 nop\n 4: c3 ret") - relocated = parse("0000 :\n 0: 74 08 je a \n 2: 66 90 xchg %ax,%ax\n a: c3 ret") - different = parse("0000 :\n 0: 74 00 je 2 \n 2: 90 nop\n 4: c3 ret") - self.assertEqual(old.instructions[0], relocated.instructions[0]) - self.assertNotEqual(old.instructions[0], different.instructions[0]) - - def test_relocation_is_not_an_instruction(self) -> None: - function = parse("0000 :\n 0: e8 00 00 00 00 call 5 \n 1: R_X86_64_PLT32 foo-0x4\n 5: c3 ret") - self.assertEqual(len(function.instructions), 2) - self.assertEqual(function.instructions[0], "call [R_X86_64_PLT32:foo]") - - def test_semantic_relocation_addend_is_preserved(self) -> None: - first = parse("0000 :\n 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n 3: R_X86_64_PC32 .rodata+0x20") - second = parse("0000 :\n 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n 3: R_X86_64_PC32 .rodata+0x40") - self.assertNotEqual(first.instructions, second.instructions) - - def test_anonymous_relocation_instance_is_ignored(self) -> None: - first = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 \n" - " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.abcdef0123456789.67", - "aarch64", - ) - second = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 \n" - " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.abcdef0123456789.68", - "aarch64", - ) - self.assertEqual(first.instructions, second.instructions) - - def test_anonymous_relocation_instance_with_addend_is_ignored(self) -> None: - first = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 \n" - " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash-with-extra.67+0x20", - "aarch64", - ) - second = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 \n" - " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash-with-extra.68+0x20", - "aarch64", - ) - self.assertEqual(first.instructions, second.instructions) - self.assertIn("+0x20", first.instructions[0]) - - different_addend = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 \n" - " 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash-with-extra.68+0x40", - "aarch64", - ) - self.assertNotEqual(first.instructions, different_addend.instructions) - - def test_inline_anonymous_relocations_allow_uniform_ordinal_shift(self) -> None: - first = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.67\n" - " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n" - " 8: 90000000 adrp x2, 0 8: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.68\n", - "aarch64", - ) - shifted = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.68\n" - " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.70\n" - " 8: 90000000 adrp x2, 0 8: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n", - "aarch64", - ) - self.assertEqual(first.instructions, shifted.instructions) - - def test_inline_anonymous_relocations_preserve_alias_relationships(self) -> None: - aliased = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.67\n" - " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.67\n" - " 8: 90000000 adrp x2, 0 8: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n", - "aarch64", - ) - no_longer_aliased = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.68\n" - " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n" - " 8: 90000000 adrp x2, 0 8: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.70\n", - "aarch64", - ) - self.assertNotEqual(aliased.instructions, no_longer_aliased.instructions) - - def test_inline_anonymous_relocations_preserve_relative_gaps(self) -> None: - first = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.67\n" - " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.69\n", - "aarch64", - ) - different_gap = parse( - "0000000000000000 :\n" - " 0: 90000000 adrp x0, 0 0: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.68\n" - " 4: 90000000 adrp x1, 0 4: R_AARCH64_ADR_PREL_PG_HI21 .data.rel.ro..Lanon.hash.71\n", - "aarch64", - ) - self.assertNotEqual(first.instructions, different_gap.instructions) - - def test_x86_anonymous_relocations_allow_llvm_suffix_changes(self) -> None: - first = parse( - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax 0: R_X86_64_PC32 anon.hash.67.llvm.123\n" - " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax 7: R_X86_64_PC32 anon.hash.69.llvm.123\n" - ) - shifted = parse( - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax 0: R_X86_64_PC32 anon.hash.68.llvm.456\n" - " 7: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax 7: R_X86_64_PC32 anon.hash.70.llvm.456\n" - ) - self.assertEqual(first.instructions, shifted.instructions) - - def test_duplicate_disassembly_symbol_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "duplicate disassembly symbol"): - compare_codegen.parse_objdump("0000 :\n 0: c3 ret\n0000 :\n 0: c3 ret", "x86_64") - - def test_encoding_size_change_is_not_identical(self) -> None: - old = parse("0000 :\n 0: 90 nop\n 1: c3 ret") - new = parse("0000 :\n 0: 66 90 nop\n 2: c3 ret") - self.assertEqual(old.instructions, new.instructions) - self.assertEqual(compare_codegen.classify(old, new), "CODEGEN ENCODING CHANGED") - - def test_aarch64_calls_and_branches(self) -> None: - function = parse( - "0000000000000000 :\n" - " 0: 94000000 bl 0 \n" - " 0: R_AARCH64_CALL26 callee\n" - " 4: 54000040 b.eq c \n" - " 8: 91000400 add x0, x0, #0x1\n" - " c: d65f03c0 ret", - "aarch64", - ) - self.assertEqual(function.calls, 1) - self.assertEqual(function.branches, 1) - self.assertIn("@instruction_3", function.instructions[1]) - self.assertIn("#0x1", function.instructions[2]) - self.assertIn("[R_AARCH64_CALL26:callee]", function.instructions[0]) - - def test_expected_missing_symbol_fails(self) -> None: - with tempfile.TemporaryDirectory() as directory: - roots = Path(directory) / "roots.toml" - roots.write_text('[[root]]\nsymbol="expected"\nartifact="harness"\ntargets=["x86_64-avx2"]\n') - argv = [ - "compare_codegen.py", - "baseline.o", - "current.o", - "--roots", - str(roots), - "--artifact", - "harness", - "--target", - "x86_64-avx2", - "--arch", - "x86_64", - ] - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object(compare_codegen, "load_functions", return_value={}), - contextlib.redirect_stdout(io.StringIO()), - contextlib.redirect_stderr(io.StringIO()), - ): - self.assertNotEqual(compare_codegen.main(), 0) - - def test_covered_backend_disappearing_is_topology_change(self) -> None: - function = parse("0000 :\n 0: c3 ret") - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - roots = root / "roots.toml" - roots.write_text( - '[[root]]\nsymbol="bit_string::foo::avx2::words"\n' - 'artifact="library"\ntargets=["x86_64-avx2"]\n' - ) - inventory = root / "inventory.toml" - inventory.write_text( - '[[kernel]]\npath="src/foo.rs"\ncoverage="caller"\nroots=["caller"]\n' - ) - argv = [ - "compare_codegen.py", - "baseline.o", - "current.o", - "--roots", - str(roots), - "--artifact", - "library", - "--inventory", - str(inventory), - "--target", - "x86_64-avx2", - "--arch", - "x86_64", - ] - stdout = io.StringIO() - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object( - compare_codegen, - "load_functions", - side_effect=[{"bit_string::foo::avx2::words": function}, {}], - ), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(io.StringIO()), - ): - self.assertEqual(compare_codegen.main(), 0) - self.assertIn("CODEGEN TOPOLOGY CHANGED", stdout.getvalue()) - - def test_caller_root_disappearing_still_fails(self) -> None: - function = parse("0000 :\n 0: c3 ret") - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - roots = root / "roots.toml" - roots.write_text( - '[[root]]\nsymbol="caller"\nartifact="harness"\ntargets=["x86_64-avx2"]\n' - ) - argv = [ - "compare_codegen.py", - "baseline.o", - "current.o", - "--roots", - str(roots), - "--artifact", - "harness", - "--target", - "x86_64-avx2", - "--arch", - "x86_64", - ] - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object( - compare_codegen, - "load_functions", - side_effect=[{"caller": function}, {}], - ), - contextlib.redirect_stdout(io.StringIO()), - contextlib.redirect_stderr(io.StringIO()), - ): - self.assertNotEqual(compare_codegen.main(), 0) - - def test_new_unregistered_backend_still_fails(self) -> None: - function = parse("0000 :\n 0: c3 ret") - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - roots = root / "roots.toml" - roots.write_text('[[root]]\nsymbol="caller"\nartifact="library"\ntargets=["x86_64-avx2"]\n') - inventory = root / "inventory.toml" - inventory.write_text( - '[[kernel]]\npath="src/foo.rs"\ncoverage="caller"\nroots=["caller"]\n' - ) - argv = [ - "compare_codegen.py", - "baseline.o", - "current.o", - "--roots", - str(roots), - "--artifact", - "library", - "--inventory", - str(inventory), - "--target", - "x86_64-avx2", - "--arch", - "x86_64", - ] - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object( - compare_codegen, - "load_functions", - side_effect=[{}, {"bit_string::foo::avx2::words": function}], - ), - contextlib.redirect_stdout(io.StringIO()), - contextlib.redirect_stderr(io.StringIO()), - ): - self.assertNotEqual(compare_codegen.main(), 0) - - def test_metadata_policy_exit_codes(self) -> None: - old = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.1-0x4\n", - self.SECTION_PREFIX + self.anonymous_section(1, line=1), - ) - metadata = parse_with_sections( - "fixture.o: file format elf64-x86-64\n" - "0000 :\n" - " 0: 48 8d 05 00 00 00 00 lea 0x0(%rip),%rax\n" - " 3: R_X86_64_PC32 .data.rel.ro..Lanon.hash.9-0x4\n", - self.SECTION_PREFIX + self.anonymous_section(9, line=2), - ) - changed = parse("0000 :\n 0: 90 nop\n 1: c3 ret") - old_encoding = parse("0000 :\n 0: 90 nop\n 1: c3 ret") - new_encoding = parse("0000 :\n 0: 66 90 nop\n 2: c3 ret") - - def run_main(before: compare_codegen.Function, after: compare_codegen.Function, strict: bool) -> int: - with tempfile.TemporaryDirectory() as directory: - roots = Path(directory) / "roots.toml" - roots.write_text('[[root]]\nsymbol="root"\nartifact="harness"\ntargets=["x86_64-avx2"]\n') - argv = [ - "compare_codegen.py", - "baseline.o", - "current.o", - "--roots", - str(roots), - "--artifact", - "harness", - "--target", - "x86_64-avx2", - "--arch", - "x86_64", - ] - if strict: - argv.append("--strict-metadata") - with ( - mock.patch.object(sys, "argv", argv), - mock.patch.object( - compare_codegen, - "load_functions", - side_effect=[{"root": before}, {"root": after}], - ), - contextlib.redirect_stdout(io.StringIO()), - contextlib.redirect_stderr(io.StringIO()), - ): - return compare_codegen.main() - - self.assertEqual(run_main(old, metadata, False), 0) - self.assertNotEqual(run_main(old, metadata, True), 0) - self.assertNotEqual(run_main(changed, old, False), 0) - self.assertNotEqual(run_main(changed, old, True), 0) - self.assertNotEqual(run_main(old_encoding, new_encoding, False), 0) - self.assertNotEqual(run_main(old_encoding, new_encoding, True), 0) - - def test_runner_forwards_strict_metadata(self) -> None: - with mock.patch.object(run_codegen_comparison.subprocess, "run", return_value=mock.Mock(returncode=0)) as run: - result = run_codegen_comparison.compare( - Path("."), - Path("baseline.o"), - Path("current.o"), - "harness", - "x86_64-avx2", - "x86_64", - True, - ) - self.assertEqual(result, 0) - self.assertIn("--strict-metadata", run.call_args.args[0]) - - def test_inventory_has_root_for_each_kernel(self) -> None: - scripts = Path(__file__).resolve().parents[1] - roots = {root.symbol for root in compare_codegen.load_roots(scripts / "codegen_roots.toml")} - inventory = tomllib.loads((scripts / "kernel_inventory.toml").read_text())["kernel"] - self.assertEqual(len(inventory), 17) - for kernel in inventory: - self.assertEqual(kernel["coverage"], "caller") - self.assertTrue(kernel["roots"]) - self.assertLessEqual(set(kernel["roots"]), roots) - - binary = next( - kernel - for kernel in inventory - if kernel["path"] == "src/traits/words_arith/funcs_for_binary_core.rs" - ) - self.assertEqual( - set(binary["roots"]), - {"codegen_bit_string_and", "codegen_bit_string_or", "codegen_bit_string_xor"}, - ) - - leading = next( - kernel - for kernel in inventory - if kernel["path"] == "src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs" - ) - self.assertEqual( - set(leading["roots"]), - { - "codegen_bit_string_leading_zeros", - "codegen_bit_string_leading_ones", - "codegen_bit_str_leading_zeros", - "codegen_bit_str_leading_ones", - }, - ) - - trailing = next( - kernel - for kernel in inventory - if kernel["path"] == "src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs" - ) - self.assertEqual( - set(trailing["roots"]), - { - "codegen_bit_string_trailing_zeros", - "codegen_bit_string_trailing_ones", - "codegen_bit_str_trailing_zeros", - "codegen_bit_str_trailing_ones", - }, - ) - - def test_independent_backend_symbols_are_discovered_from_inventory_modules(self) -> None: - function = parse("0000 :\n 0: c3 ret") - functions = { - "bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words": function, - "bit_string::traits::words_scan::unrelated::avx2::count_words": function, - "bit_string::traits::words_scan::funcs_for_count_ones::entry": function, - } - discovered = compare_codegen.independent_backend_symbols( - functions, - ("bit_string::traits::words_scan::funcs_for_count_ones",), - ) - self.assertEqual( - discovered, - {"bit_string::traits::words_scan::funcs_for_count_ones::avx2::count_words"}, - ) - - -if __name__ == "__main__": - unittest.main() From 6f85c110535ab013221e1088b13e4aabf76de517 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 02:30:51 +0000 Subject: [PATCH 72/79] prepare 0.6.0 release --- .github/workflows/release.yml | 21 +++++++++++++++++++++ Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- derive/Cargo.toml | 2 +- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 30ad63a..bd924c9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,28 @@ concurrency: cancel-in-progress: false jobs: + publish-derive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: dtolnay/rust-toolchain@stable + + - name: Test + run: cargo test --locked + + - name: Publish derive crate + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + run: | + if [ "${{ inputs.mode || 'real' }}" = "dryrun" ]; then + cargo publish --locked --dry-run --manifest-path derive/Cargo.toml + else + cargo publish --locked --manifest-path derive/Cargo.toml + fi + release: + needs: publish-derive uses: jcfangc/ci/.github/workflows/rust-crates-release.yml@v0 with: mode: ${{ inputs.mode || 'real' }} diff --git a/Cargo.lock b/Cargo.lock index 7a3e6ab..2c910b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,7 +49,7 @@ dependencies = [ [[package]] name = "bit-string" -version = "0.5.0" +version = "0.6.0" dependencies = [ "bit-string-derive", "bitvec_simd", @@ -61,7 +61,7 @@ dependencies = [ [[package]] name = "bit-string-derive" -version = "0.5.0" +version = "0.6.0" dependencies = [ "proc-macro-crate", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 5c9ab82..9c8b41f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bit-string" -version = "0.5.0" +version = "0.6.0" edition = "2024" description = "A compact owned bit string type with editing, matching, and bitwise operations." readme = "README.md" @@ -14,7 +14,7 @@ exclude = ["/benches", "/scripts", "/src/**/tests_for_*", "/tests", ".github/"] members = ["derive"] [dependencies] -bit-string-derive = { version = "0.5.0", path = "derive" } +bit-string-derive = { version = "0.6.0", path = "derive" } int-interval = "0.9.6" witnessed = "0.8.0" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 08ff9b5..6dc08ef 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bit-string-derive" -version = "0.5.0" +version = "0.6.0" edition = "2024" description = "Procedural macros for bit-string" repository = "https://github.com/jcfangc/bit-string" From 7ba78d028d85f1def824292f85380d05a7561264 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 02:40:47 +0000 Subject: [PATCH 73/79] inherit release version from workspace --- Cargo.toml | 10 ++++++++-- derive/Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9c8b41f..d8a524e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bit-string" -version = "0.6.0" +version.workspace = true edition = "2024" description = "A compact owned bit string type with editing, matching, and bitwise operations." readme = "README.md" @@ -13,8 +13,14 @@ exclude = ["/benches", "/scripts", "/src/**/tests_for_*", "/tests", ".github/"] [workspace] members = ["derive"] +[workspace.package] +version = "0.6.0" + +[workspace.dependencies] +bit-string-derive = { version = "0.6", path = "derive" } + [dependencies] -bit-string-derive = { version = "0.6.0", path = "derive" } +bit-string-derive.workspace = true int-interval = "0.9.6" witnessed = "0.8.0" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 6dc08ef..1005bcc 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bit-string-derive" -version = "0.6.0" +version.workspace = true edition = "2024" description = "Procedural macros for bit-string" repository = "https://github.com/jcfangc/bit-string" From ef6392e6ce8e9ef331b6c45bdc4f76882d9659a9 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 02:50:38 +0000 Subject: [PATCH 74/79] replace int-interval dependency --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- README.md | 4 ++-- benches/bit_ops_leading.rs | 2 +- benches/bit_ops_trailing.rs | 2 +- benches/editing_replace_interval.rs | 2 +- benches/editing_slice.rs | 2 +- benches/matching_matches_at.rs | 2 +- src/bit_str.rs | 2 +- src/bit_str/impls_for_access/tests_for_get.rs | 2 +- src/bit_str/impls_for_access/tests_for_get_chunk.rs | 2 +- .../impls_for_leading_zeros/tests_for_leading_ones.rs | 2 +- .../impls_for_leading_zeros/tests_for_leading_zeros.rs | 2 +- .../impls_for_trailing_zeros/tests_for_trailing_ones.rs | 2 +- .../impls_for_trailing_zeros/tests_for_trailing_zeros.rs | 2 +- src/bit_str/impls_for_iter/tests_for_iter.rs | 2 +- src/bit_str/impls_for_slice.rs | 2 +- src/bit_str/impls_for_slice/tests_for_slice.rs | 2 +- src/bit_string/impls_for_editing/impls_for_drain.rs | 2 +- .../impls_for_drain/tests_for_drain_interval.rs | 2 +- src/bit_string/impls_for_editing/impls_for_replace.rs | 2 +- .../impls_for_replace/tests_for_replace_interval.rs | 2 +- src/bit_string/impls_for_editing/impls_for_slice.rs | 2 +- .../impls_for_editing/impls_for_slice/tests_for_slice.rs | 2 +- src/bit_string/impls_for_editing/tests_for_invariants.rs | 2 +- src/bit_string/tests_for_proptest.rs | 2 +- src/packed_str/impls_for_slice.rs | 2 +- src/packed_string/impls_for_editing/impls_for_drain.rs | 2 +- src/packed_string/impls_for_editing/impls_for_replace.rs | 2 +- src/packed_string/impls_for_editing/impls_for_slice.rs | 2 +- tests/adversarial/tests_for_access.rs | 2 +- tests/adversarial/tests_for_bitstr.rs | 2 +- tests/adversarial/tests_for_bugs.rs | 2 +- tests/adversarial/tests_for_count.rs | 2 +- tests/adversarial/tests_for_fmt.rs | 2 +- tests/adversarial/tests_for_iter.rs | 2 +- tests/adversarial/tests_for_leading_trailing.rs | 2 +- tests/adversarial/tests_for_matching.rs | 2 +- tests/adversarial/tests_for_ord_hash.rs | 2 +- tests/adversarial/tests_for_predicates.rs | 2 +- tests/adversarial/tests_for_replace.rs | 2 +- tests/adversarial/tests_for_slice_drain.rs | 2 +- tests/packed_char.rs | 2 +- 43 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c910b9..2f9184a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,7 +54,7 @@ dependencies = [ "bit-string-derive", "bitvec_simd", "codspeed-divan-compat", - "int-interval", + "int-intervals", "proptest", "witnessed", ] @@ -329,10 +329,10 @@ dependencies = [ ] [[package]] -name = "int-interval" -version = "0.9.6" +name = "int-intervals" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9801bebb6a8355e9a7bcc42ce9cc910b045bd5217cb750b8b9e0528eb1842e37" +checksum = "bbae229a6c2c8562cc2bee9365152c5787dd6ec7f42404fbd3563f2b437c4094" [[package]] name = "itertools" diff --git a/Cargo.toml b/Cargo.toml index d8a524e..ad4560d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ bit-string-derive = { version = "0.6", path = "derive" } [dependencies] bit-string-derive.workspace = true -int-interval = "0.9.6" +int-intervals = "0.1" witnessed = "0.8.0" [lib] diff --git a/README.md b/README.md index 24b8ae7..0ace4b9 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ let v: BitString = (0..8).map(|i| i % 2 == 0).collect(); ```rust use bit_string::BitString; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; let bits = BitString::try_from("11001010").unwrap(); @@ -123,7 +123,7 @@ bits.truncate(4); // "1001" bits.split_off(2); // → BitString "01", bits = "10" // Range operations -use int_interval::UsizeCO; +use int_intervals::UsizeCO; let interval = UsizeCO::try_new(1, 3).unwrap(); let replaced = bits.replace_interval(interval, &BitString::ones(2)); // "111" let drained = replaced.drain_interval(UsizeCO::try_new(0, 2).unwrap()); // "1" diff --git a/benches/bit_ops_leading.rs b/benches/bit_ops_leading.rs index 31b3706..d50ac20 100644 --- a/benches/bit_ops_leading.rs +++ b/benches/bit_ops_leading.rs @@ -1,7 +1,7 @@ use bit_string::BitString; use bitvec_simd::BitVec; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/benches/bit_ops_trailing.rs b/benches/bit_ops_trailing.rs index ec64823..d6c7758 100644 --- a/benches/bit_ops_trailing.rs +++ b/benches/bit_ops_trailing.rs @@ -1,6 +1,6 @@ use bit_string::BitString; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/benches/editing_replace_interval.rs b/benches/editing_replace_interval.rs index e642304..5fed38a 100644 --- a/benches/editing_replace_interval.rs +++ b/benches/editing_replace_interval.rs @@ -1,6 +1,6 @@ use bit_string::BitString; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/benches/editing_slice.rs b/benches/editing_slice.rs index 296518f..774f7d8 100644 --- a/benches/editing_slice.rs +++ b/benches/editing_slice.rs @@ -1,6 +1,6 @@ use bit_string::BitString; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/benches/matching_matches_at.rs b/benches/matching_matches_at.rs index 6bb4cd9..8f6efc6 100644 --- a/benches/matching_matches_at.rs +++ b/benches/matching_matches_at.rs @@ -1,6 +1,6 @@ use bit_string::BitString; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/src/bit_str.rs b/src/bit_str.rs index ee725f1..5119c87 100644 --- a/src/bit_str.rs +++ b/src/bit_str.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_access/tests_for_get.rs b/src/bit_str/impls_for_access/tests_for_get.rs index fcb165a..2ab1496 100644 --- a/src/bit_str/impls_for_access/tests_for_get.rs +++ b/src/bit_str/impls_for_access/tests_for_get.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitStr; use crate::BitString; diff --git a/src/bit_str/impls_for_access/tests_for_get_chunk.rs b/src/bit_str/impls_for_access/tests_for_get_chunk.rs index c0360b8..628a0b2 100644 --- a/src/bit_str/impls_for_access/tests_for_get_chunk.rs +++ b/src/bit_str/impls_for_access/tests_for_get_chunk.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitStr; use crate::BitString; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_ones.rs b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_ones.rs index 98a86eb..c28c043 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_ones.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_ones.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_zeros.rs b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_zeros.rs index f3dd482..b60756f 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_zeros.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_zeros.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs index 21eea89..892c381 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs index ae8d5a1..0cddb2f 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_iter/tests_for_iter.rs b/src/bit_str/impls_for_iter/tests_for_iter.rs index 1154ebf..0f85c64 100644 --- a/src/bit_str/impls_for_iter/tests_for_iter.rs +++ b/src/bit_str/impls_for_iter/tests_for_iter.rs @@ -1,5 +1,5 @@ use alloc::vec::Vec; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_slice.rs b/src/bit_str/impls_for_slice.rs index 9b89e99..25af156 100644 --- a/src/bit_str/impls_for_slice.rs +++ b/src/bit_str/impls_for_slice.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use super::*; diff --git a/src/bit_str/impls_for_slice/tests_for_slice.rs b/src/bit_str/impls_for_slice/tests_for_slice.rs index b6f3b27..526a734 100644 --- a/src/bit_str/impls_for_slice/tests_for_slice.rs +++ b/src/bit_str/impls_for_slice/tests_for_slice.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_string/impls_for_editing/impls_for_drain.rs b/src/bit_string/impls_for_editing/impls_for_drain.rs index a8c7cf9..0c3a179 100644 --- a/src/bit_string/impls_for_editing/impls_for_drain.rs +++ b/src/bit_string/impls_for_editing/impls_for_drain.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::funcs_for_bits::*; use crate::traits::*; diff --git a/src/bit_string/impls_for_editing/impls_for_drain/tests_for_drain_interval.rs b/src/bit_string/impls_for_editing/impls_for_drain/tests_for_drain_interval.rs index 39b7272..cb88077 100644 --- a/src/bit_string/impls_for_editing/impls_for_drain/tests_for_drain_interval.rs +++ b/src/bit_string/impls_for_editing/impls_for_drain/tests_for_drain_interval.rs @@ -1,5 +1,5 @@ use alloc::string::ToString; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_string/impls_for_editing/impls_for_replace.rs b/src/bit_string/impls_for_editing/impls_for_replace.rs index cc5e184..809aba5 100644 --- a/src/bit_string/impls_for_editing/impls_for_replace.rs +++ b/src/bit_string/impls_for_editing/impls_for_replace.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use witnessed::{WitnessExt, Witnessed}; use crate::funcs_for_bits::*; diff --git a/src/bit_string/impls_for_editing/impls_for_replace/tests_for_replace_interval.rs b/src/bit_string/impls_for_editing/impls_for_replace/tests_for_replace_interval.rs index b0a138b..f77cf4c 100644 --- a/src/bit_string/impls_for_editing/impls_for_replace/tests_for_replace_interval.rs +++ b/src/bit_string/impls_for_editing/impls_for_replace/tests_for_replace_interval.rs @@ -1,7 +1,7 @@ use crate::BitString; use alloc::string::ToString; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[inline] fn iv(start: usize, end_excl: usize) -> UsizeCO { diff --git a/src/bit_string/impls_for_editing/impls_for_slice.rs b/src/bit_string/impls_for_editing/impls_for_slice.rs index d474f1a..8016d06 100644 --- a/src/bit_string/impls_for_editing/impls_for_slice.rs +++ b/src/bit_string/impls_for_editing/impls_for_slice.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::funcs_for_bits::*; use crate::traits::*; diff --git a/src/bit_string/impls_for_editing/impls_for_slice/tests_for_slice.rs b/src/bit_string/impls_for_editing/impls_for_slice/tests_for_slice.rs index 56a6e2f..5753cc0 100644 --- a/src/bit_string/impls_for_editing/impls_for_slice/tests_for_slice.rs +++ b/src/bit_string/impls_for_editing/impls_for_slice/tests_for_slice.rs @@ -1,5 +1,5 @@ use alloc::string::ToString; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_string/impls_for_editing/tests_for_invariants.rs b/src/bit_string/impls_for_editing/tests_for_invariants.rs index c6002d8..e5965c4 100644 --- a/src/bit_string/impls_for_editing/tests_for_invariants.rs +++ b/src/bit_string/impls_for_editing/tests_for_invariants.rs @@ -5,7 +5,7 @@ use alloc::string::ToString; use crate::BitString; use crate::word_len; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; /// Verify that `self.as_words().len() == word_len(self.bit_len())`. fn assert_invariants(bits: &BitString) { diff --git a/src/bit_string/tests_for_proptest.rs b/src/bit_string/tests_for_proptest.rs index 1307891..caf567f 100644 --- a/src/bit_string/tests_for_proptest.rs +++ b/src/bit_string/tests_for_proptest.rs @@ -4,7 +4,7 @@ use proptest::collection::vec; use proptest::prelude::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/packed_str/impls_for_slice.rs b/src/packed_str/impls_for_slice.rs index d8d9ba0..25778d9 100644 --- a/src/packed_str/impls_for_slice.rs +++ b/src/packed_str/impls_for_slice.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use super::*; diff --git a/src/packed_string/impls_for_editing/impls_for_drain.rs b/src/packed_string/impls_for_editing/impls_for_drain.rs index cf582d6..e278871 100644 --- a/src/packed_string/impls_for_editing/impls_for_drain.rs +++ b/src/packed_string/impls_for_editing/impls_for_drain.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use super::*; diff --git a/src/packed_string/impls_for_editing/impls_for_replace.rs b/src/packed_string/impls_for_editing/impls_for_replace.rs index 330ee7e..d279a15 100644 --- a/src/packed_string/impls_for_editing/impls_for_replace.rs +++ b/src/packed_string/impls_for_editing/impls_for_replace.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use super::*; diff --git a/src/packed_string/impls_for_editing/impls_for_slice.rs b/src/packed_string/impls_for_editing/impls_for_slice.rs index a2ae8d9..e9c1cff 100644 --- a/src/packed_string/impls_for_editing/impls_for_slice.rs +++ b/src/packed_string/impls_for_editing/impls_for_slice.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use super::*; diff --git a/tests/adversarial/tests_for_access.rs b/tests/adversarial/tests_for_access.rs index ca23465..4c85a56 100644 --- a/tests/adversarial/tests_for_access.rs +++ b/tests/adversarial/tests_for_access.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_get_out_of_bounds() { diff --git a/tests/adversarial/tests_for_bitstr.rs b/tests/adversarial/tests_for_bitstr.rs index e382d82..a5111ae 100644 --- a/tests/adversarial/tests_for_bitstr.rs +++ b/tests/adversarial/tests_for_bitstr.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_clone_invariants() { diff --git a/tests/adversarial/tests_for_bugs.rs b/tests/adversarial/tests_for_bugs.rs index 527cae1..4dadb29 100644 --- a/tests/adversarial/tests_for_bugs.rs +++ b/tests/adversarial/tests_for_bugs.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; /// Regression test for B1: `set_chunk` must mask unused high bits after writing. #[test] diff --git a/tests/adversarial/tests_for_count.rs b/tests/adversarial/tests_for_count.rs index 63ecea5..d0a1d4a 100644 --- a/tests/adversarial/tests_for_count.rs +++ b/tests/adversarial/tests_for_count.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_count_ones_zeros() { diff --git a/tests/adversarial/tests_for_fmt.rs b/tests/adversarial/tests_for_fmt.rs index cce1e3f..39ba508 100644 --- a/tests/adversarial/tests_for_fmt.rs +++ b/tests/adversarial/tests_for_fmt.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_display_roundtrip() { diff --git a/tests/adversarial/tests_for_iter.rs b/tests/adversarial/tests_for_iter.rs index 98f1d3b..10d523d 100644 --- a/tests/adversarial/tests_for_iter.rs +++ b/tests/adversarial/tests_for_iter.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_iter_empty() { diff --git a/tests/adversarial/tests_for_leading_trailing.rs b/tests/adversarial/tests_for_leading_trailing.rs index 8439c64..5f713ee 100644 --- a/tests/adversarial/tests_for_leading_trailing.rs +++ b/tests/adversarial/tests_for_leading_trailing.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; // =========================================================================== // A. BitString leading/trailing — exhaustive small-length coverage diff --git a/tests/adversarial/tests_for_matching.rs b/tests/adversarial/tests_for_matching.rs index 0ebc21e..0927f29 100644 --- a/tests/adversarial/tests_for_matching.rs +++ b/tests/adversarial/tests_for_matching.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn alignment_pattern(len: usize) -> String { (0..len) diff --git a/tests/adversarial/tests_for_ord_hash.rs b/tests/adversarial/tests_for_ord_hash.rs index 8235582..83e6f5c 100644 --- a/tests/adversarial/tests_for_ord_hash.rs +++ b/tests/adversarial/tests_for_ord_hash.rs @@ -1,6 +1,6 @@ use super::*; use core::cmp::Ordering; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn alignment_pattern(len: usize) -> String { (0..len) diff --git a/tests/adversarial/tests_for_predicates.rs b/tests/adversarial/tests_for_predicates.rs index 206b20b..6fec058 100644 --- a/tests/adversarial/tests_for_predicates.rs +++ b/tests/adversarial/tests_for_predicates.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_predicates_corner() { diff --git a/tests/adversarial/tests_for_replace.rs b/tests/adversarial/tests_for_replace.rs index 99a60a4..bc7e689 100644 --- a/tests/adversarial/tests_for_replace.rs +++ b/tests/adversarial/tests_for_replace.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_replace_equal_length() { diff --git a/tests/adversarial/tests_for_slice_drain.rs b/tests/adversarial/tests_for_slice_drain.rs index 7ac5205..ec8d59b 100644 --- a/tests/adversarial/tests_for_slice_drain.rs +++ b/tests/adversarial/tests_for_slice_drain.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_slice_interval_clamping() { diff --git a/tests/packed_char.rs b/tests/packed_char.rs index 9ace679..9ae43ce 100644 --- a/tests/packed_char.rs +++ b/tests/packed_char.rs @@ -1,5 +1,5 @@ use bit_string::{BitString, PackedString, packed, traits::PackedChar}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use proptest::prelude::*; #[packed(bits = 1)] From 4347e2077a53c1c3fe5b20495523e8355e8f679e Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 03:01:36 +0000 Subject: [PATCH 75/79] harden release workflow documentation --- .github/workflows/release.yml | 26 ++++++++++++++++++++++++++ README.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd924c9..da80f2d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,10 +45,36 @@ jobs: run: | if [ "${{ inputs.mode || 'real' }}" = "dryrun" ]; then cargo publish --locked --dry-run --manifest-path derive/Cargo.toml + exit 0 + fi + + derive_package="$(cargo metadata --locked --no-deps --format-version 1 --manifest-path derive/Cargo.toml \ + | python3 -c 'import json, sys; package = next(p for p in json.load(sys.stdin)["packages"] if p["manifest_path"].endswith("/derive/Cargo.toml")); print(package["name"], package["version"])')" + read -r derive_name derive_version <<< "$derive_package" + registry_url="https://crates.io/api/v1/crates/${derive_name}/${derive_version}" + + registry_version_available() { + test "$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' "$registry_url")" = 200 + } + + if registry_version_available; then + echo "${derive_name} ${derive_version} is already published; skipping" else cargo publish --locked --manifest-path derive/Cargo.toml fi + for attempt in $(seq 1 30); do + if registry_version_available; then + echo "${derive_name} ${derive_version} is visible on crates.io" + exit 0 + fi + echo "waiting for ${derive_name} ${derive_version} to become visible (attempt ${attempt}/30)" + sleep 5 + done + + echo "${derive_name} ${derive_version} did not become visible on crates.io" >&2 + exit 1 + release: needs: publish-derive uses: jcfangc/ci/.github/workflows/rust-crates-release.yml@v0 diff --git a/README.md b/README.md index 0ace4b9..91aabbc 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,37 @@ assert_eq!((!a).to_string(), "0101"); |------|------|------|--------| | `BitString` | Owned, `Vec` backing | 4×usize | No | | `BitStr<'bs>` | Zero-copy borrowed view | 3×usize | **Yes** | +| `PackedString` | Owned fixed-width code sequence | 4×usize + marker | No | +| `PackedStr<'ps, C, BITS>` | Zero-copy borrowed packed view | 3×usize + marker | **Yes** | Bits are packed little-endian into `u64` words. Unused high bits in the last word are always zero. +## Packed strings + +Use `#[packed(bits = N)]` to define a compact code alphabet. Codes are stored in +exactly `N` bits, where `N` is between 1 and 8. + +```rust +use bit_string::{PackedString, packed}; + +#[packed(bits = 2)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Base { + A = 0, + C = 1, + G = 2, + T = 3, +} + +let sequence = + PackedString::::from_chars([Base::A, Base::C, Base::G]); +assert_eq!(sequence.char_len(), 3); +``` + +`PackedString` owns the packed storage, while `PackedStr` provides a +zero-copy, character-aligned view. Manual `PackedChar` implementations remain +available when an alphabet needs custom code conversion. + ## API by category ### Construction From 075fc02f84e0f45492671f8770fc2af0841e4826 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 03:19:23 +0000 Subject: [PATCH 76/79] correct packed type size documentation --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 91aabbc..0f4b001 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ assert_eq!((!a).to_string(), "0101"); |------|------|------|--------| | `BitString` | Owned, `Vec` backing | 4×usize | No | | `BitStr<'bs>` | Zero-copy borrowed view | 3×usize | **Yes** | -| `PackedString` | Owned fixed-width code sequence | 4×usize + marker | No | -| `PackedStr<'ps, C, BITS>` | Zero-copy borrowed packed view | 3×usize + marker | **Yes** | +| `PackedString` | Owned fixed-width code sequence | 4×usize | No | +| `PackedStr<'ps, C, BITS>` | Zero-copy borrowed packed view | 3×usize | **Yes** | Bits are packed little-endian into `u64` words. Unused high bits in the last word are always zero. From 3b34a41f6469353a55ccd4b541cdcd91fae8928e Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 03:39:29 +0000 Subject: [PATCH 77/79] reuse shared release publisher for derive --- .github/workflows/release.yml | 45 ++++++++++------------------------- 1 file changed, 13 insertions(+), 32 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index da80f2d..0ca3d3f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,44 +36,25 @@ jobs: - uses: dtolnay/rust-toolchain@stable + - name: Checkout CI scripts + env: + CI_REF: v0 + run: | + rm -rf "$RUNNER_TEMP/ci" + git clone --depth 1 --branch "$CI_REF" \ + https://github.com/jcfangc/ci \ + "$RUNNER_TEMP/ci" + test -f "$RUNNER_TEMP/ci/.github/scripts/release/publish.py" + - name: Test run: cargo test --locked - name: Publish derive crate env: CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} - run: | - if [ "${{ inputs.mode || 'real' }}" = "dryrun" ]; then - cargo publish --locked --dry-run --manifest-path derive/Cargo.toml - exit 0 - fi - - derive_package="$(cargo metadata --locked --no-deps --format-version 1 --manifest-path derive/Cargo.toml \ - | python3 -c 'import json, sys; package = next(p for p in json.load(sys.stdin)["packages"] if p["manifest_path"].endswith("/derive/Cargo.toml")); print(package["name"], package["version"])')" - read -r derive_name derive_version <<< "$derive_package" - registry_url="https://crates.io/api/v1/crates/${derive_name}/${derive_version}" - - registry_version_available() { - test "$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' "$registry_url")" = 200 - } - - if registry_version_available; then - echo "${derive_name} ${derive_version} is already published; skipping" - else - cargo publish --locked --manifest-path derive/Cargo.toml - fi - - for attempt in $(seq 1 30); do - if registry_version_available; then - echo "${derive_name} ${derive_version} is visible on crates.io" - exit 0 - fi - echo "waiting for ${derive_name} ${derive_version} to become visible (attempt ${attempt}/30)" - sleep 5 - done - - echo "${derive_name} ${derive_version} did not become visible on crates.io" >&2 - exit 1 + PACKAGE_MANIFEST: derive/Cargo.toml + RELEASE_MODE: ${{ inputs.mode || 'real' }} + run: python3 "$RUNNER_TEMP/ci/.github/scripts/release/publish.py" release: needs: publish-derive From efc04aebef454bdbcc595f115cb767aa722c537c Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 04:18:59 +0000 Subject: [PATCH 78/79] pin CodSpeed workflow environment --- .github/workflows/codspeed.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 5e33898..e83e6d4 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -14,6 +14,8 @@ jobs: benchmarks: uses: jcfangc/ci/.github/workflows/rust-codspeed.yml@v0 with: - rust_toolchain: stable + rust_toolchain: "1.98.1" + cargo_codspeed_version: "5.0.1" + runner: ubuntu-24.04 build_command: RUSTFLAGS="-C target-feature=+avx2" cargo codspeed build -m simulation run_command: cargo codspeed run -m simulation ours_ From 27ba5cba9a7ea50dfd5e73012d224cfe5993c586 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 5 Sep 2026 05:41:16 +0000 Subject: [PATCH 79/79] pin CodSpeed workflow revision --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index e83e6d4..4206d29 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -12,7 +12,7 @@ permissions: jobs: benchmarks: - uses: jcfangc/ci/.github/workflows/rust-codspeed.yml@v0 + uses: jcfangc/ci/.github/workflows/rust-codspeed.yml@ccd19196bcd8e8a42924c0621c321ee2dc2db61a with: rust_toolchain: "1.98.1" cargo_codspeed_version: "5.0.1"