diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 5e33898..4206d29 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -12,8 +12,10 @@ permissions: jobs: benchmarks: - uses: jcfangc/ci/.github/workflows/rust-codspeed.yml@v0 + uses: jcfangc/ci/.github/workflows/rust-codspeed.yml@ccd19196bcd8e8a42924c0621c321ee2dc2db61a with: - rust_toolchain: stable + rust_toolchain: "1.98.1" + cargo_codspeed_version: "5.0.1" + runner: ubuntu-24.04 build_command: RUSTFLAGS="-C target-feature=+avx2" cargo codspeed build -m simulation run_command: cargo codspeed run -m simulation ours_ diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index ef0abe6..e856da8 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: include: - - name: linux-x64-scalar + - name: linux-x64-baseline os: ubuntu-latest rustflags: "" @@ -40,6 +40,8 @@ jobs: steps: - uses: actions/checkout@v5 + with: + fetch-depth: 0 - uses: dtolnay/rust-toolchain@stable @@ -60,3 +62,7 @@ jobs: - name: Test run: cargo test --locked + + - name: Test benchmark tooling + if: matrix.name == 'linux-x64-baseline' + run: python3 -m unittest discover -s scripts/tests -v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 30ad63a..0ca3d3f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,35 @@ concurrency: cancel-in-progress: false jobs: + publish-derive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: dtolnay/rust-toolchain@stable + + - name: Checkout CI scripts + env: + CI_REF: v0 + run: | + rm -rf "$RUNNER_TEMP/ci" + git clone --depth 1 --branch "$CI_REF" \ + https://github.com/jcfangc/ci \ + "$RUNNER_TEMP/ci" + test -f "$RUNNER_TEMP/ci/.github/scripts/release/publish.py" + + - name: Test + run: cargo test --locked + + - name: Publish derive crate + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + PACKAGE_MANIFEST: derive/Cargo.toml + RELEASE_MODE: ${{ inputs.mode || 'real' }} + run: python3 "$RUNNER_TEMP/ci/.github/scripts/release/publish.py" + release: + needs: publish-derive uses: jcfangc/ci/.github/workflows/rust-crates-release.yml@v0 with: mode: ${{ inputs.mode || 'real' }} diff --git a/.gitignore b/.gitignore index b7cd4ac..3743c0e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,10 @@ /.venv /proptest-regressions .cargo/config.toml + +# Python tooling caches +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index fa0dd9a..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,66 +0,0 @@ -# bit-string - -A `no_std` Rust crate providing a compact owned bit string type with editing, matching, and bitwise operations. - -通用约定参见父级：`../CLAUDE.md` - -## Architecture - -``` -src/ - lib.rs # #![no_std], re-exports BitString and errors - bit_string.rs # BitString struct definition (Box<[u64]> + len) - errors.rs # ParseBitStringError, BitStringLenMismatch - funcs_for_share.rs # Shared low-level helpers: bit_at, set_bit, read/write_chunk, - # mask_unused_bits, last_word_mask, word_len, etc. - impls_for_construction.rs # new, try_from, from_bool_iter, from_words, zeros, repeat - impls_for_access.rs # get, len, is_empty, to_string - impls_for_editing.rs # push, pop, insert, remove, set, extend, truncate, slice, etc. - impls_for_matching.rs # bits_equal_at + submodule declarations - impls_for_matching/ - impls_for_matches_at.rs # matches_at, starts_with, ends_with - impls_for_matches_at/ - funcs_for_starts_with_core.rs # SIMD starts_with word equality - funcs_for_ends_with_core.rs # SIMD ends_with (aligned + unaligned) - impls_for_find.rs # contains, find, rfind - impls_for_find/ - funcs_for_contains_core.rs # SIMD find_any_candidate (shift-outer) - funcs_for_find_core.rs # SIMD find_first_word (word-outer) - funcs_for_rfind_core.rs # SIMD find_last_word (word-reverse) - impls_for_strip.rs # strip_prefix, strip_suffix - impls_for_iter.rs # Iterator impl (yields bools) - impls_for_fmt.rs # Display, Debug (outputs "1010…" string) - impls_for_bit_ops/ - impls_for_not.rs # not, not_assign, not_into - impls_for_count_ones.rs # count_ones, count_zeros - impls_for_shl.rs # shl, shl_assign, shl_into (left shift with zero fill) - funcs_for_binary_core.rs # Generic binary-op dispatch: AND, OR, XOR - impls_for_and.rs # and, and_assign, and_into - impls_for_or.rs # or, or_assign, or_into - impls_for_xor.rs # xor, xor_assign, xor_into - impls_for_shr.rs # shr, shr_assign, shr_into (right shift with zero fill) -``` - -### SIMD Backend Dispatch Pattern - -Each bit operation follows the same multi-backend architecture: - -1. **Public API** methods (`shl`, `shl_assign`, `shl_into`) delegate to a `funcs_*_core` module. -2. The **core module** provides `owned()` (allocate new) and `assign()` (in-place) entry points. -3. A **`dispatch`** function uses `#[cfg]` gates to select the best available SIMD backend at compile time: - - `avx2` (x86/x86_64, 256-bit, 4×u64 lanes) - - `sse2` (x86/x86_64, 128-bit, 2×u64 lanes) — gated with `not(target_feature = "avx2")` - - `neon` (aarch64, 128-bit, 2×u64 lanes) - - `scalar` — fallback, supports all targets -4. Each backend lives in its own `mod` annotated with `#[target_feature(enable = "...")]`. -5. **Backend equivalence tests** run the scalar backend as oracle and assert every available SIMD backend produces identical output for a matrix of inputs and amounts. - -### Core Data Structure - -`BitString` stores bits in a `Box<[u64]>` array. The logical bit length (`len: usize`) is tracked separately. The last `u64` word is masked via `last_word_mask` to zero bits beyond `len`. Internal word size: `WORD_BITS = u64::BITS = 64`. - -## Key Dependencies - -- `int-interval` — interval type for editing operations (slice, remove, replace) -- `codspeed-divan-compat` — benchmarking framework -- `bitvec_simd` — dev-dependency for benchmark comparisons diff --git a/Cargo.lock b/Cargo.lock index 038a9fe..2f9184a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,15 +49,26 @@ dependencies = [ [[package]] name = "bit-string" -version = "0.5.0" +version = "0.6.0" dependencies = [ + "bit-string-derive", "bitvec_simd", "codspeed-divan-compat", - "int-interval", + "int-intervals", "proptest", "witnessed", ] +[[package]] +name = "bit-string-derive" +version = "0.6.0" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "bit-vec" version = "0.8.0" @@ -318,10 +329,10 @@ dependencies = [ ] [[package]] -name = "int-interval" -version = "0.9.6" +name = "int-intervals" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9801bebb6a8355e9a7bcc42ce9cc910b045bd5217cb750b8b9e0528eb1842e37" +checksum = "bbae229a6c2c8562cc2bee9365152c5787dd6ec7f42404fbd3563f2b437c4094" [[package]] name = "itertools" diff --git a/Cargo.toml b/Cargo.toml index 87207e8..ad4560d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bit-string" -version = "0.5.0" +version.workspace = true edition = "2024" description = "A compact owned bit string type with editing, matching, and bitwise operations." readme = "README.md" @@ -8,11 +8,20 @@ repository = "https://github.com/jcfangc/bit-string" license = "MIT OR Apache-2.0" keywords = ["bitstring", "bits", "bitset", "sequence"] categories = ["data-structures", "no-std"] -exclude = ["/benches", "/src/**/tests_for_*", "/tests", ".github/"] +exclude = ["/benches", "/scripts", "/src/**/tests_for_*", "/tests", ".github/"] +[workspace] +members = ["derive"] + +[workspace.package] +version = "0.6.0" + +[workspace.dependencies] +bit-string-derive = { version = "0.6", path = "derive" } [dependencies] -int-interval = "0.9.6" +bit-string-derive.workspace = true +int-intervals = "0.1" witnessed = "0.8.0" [lib] diff --git a/README.md b/README.md index ebf2e78..0f4b001 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/gh/jcfangc/bit-string) [![Coverage](https://codecov.io/gh/jcfangc/bit-string/branch/main/graph/badge.svg)](https://codecov.io/gh/jcfangc/bit-string) -A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero-copy view, with construction, editing, matching, comparison, and bitwise operations — accelerated by compile-time-selected SIMD backends (AVX2, SSSE3, NEON). +A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero-copy view, with construction, editing, matching, comparison, and bitwise operations — accelerated by compile-time-selected SIMD backends (AVX2, SSE2, SSE4.1, SSSE3, NEON). ## Quick start @@ -14,9 +14,9 @@ A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero- use bit_string::BitString; // Parse from a binary string -let bits = BitString::try_from("1010_0011").unwrap(); +let bits = BitString::try_from("10100011").unwrap(); assert_eq!(bits.to_string(), "10100011"); -assert_eq!(bits.len(), 8); +assert_eq!(bits.bit_len(), 8); assert_eq!(bits.count_ones(), 4); // Build programmatically @@ -38,11 +38,39 @@ assert_eq!((!a).to_string(), "0101"); | Type | Role | Size | `Copy` | |------|------|------|--------| -| `BitString` | Owned, `Box<[u64]>` backing | 4×usize | No | +| `BitString` | Owned, `Vec` backing | 4×usize | No | | `BitStr<'bs>` | Zero-copy borrowed view | 3×usize | **Yes** | +| `PackedString` | Owned fixed-width code sequence | 4×usize | No | +| `PackedStr<'ps, C, BITS>` | Zero-copy borrowed packed view | 3×usize | **Yes** | Bits are packed little-endian into `u64` words. Unused high bits in the last word are always zero. +## Packed strings + +Use `#[packed(bits = N)]` to define a compact code alphabet. Codes are stored in +exactly `N` bits, where `N` is between 1 and 8. + +```rust +use bit_string::{PackedString, packed}; + +#[packed(bits = 2)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Base { + A = 0, + C = 1, + G = 2, + T = 3, +} + +let sequence = + PackedString::::from_chars([Base::A, Base::C, Base::G]); +assert_eq!(sequence.char_len(), 3); +``` + +`PackedString` owns the packed storage, while `PackedStr` provides a +zero-copy, character-aligned view. Manual `PackedChar` implementations remain +available when an alphabet needs custom code conversion. + ## API by category ### Construction @@ -52,7 +80,7 @@ use bit_string::BitString; // From a binary string literal let a = BitString::try_from("0101").unwrap(); // "0101" -let b = BitString::try_from("0101_1110").unwrap(); // "01011110" (underscores ignored) +let b = BitString::try_from("01011110").unwrap(); // "01011110" // Pre-allocated let z = BitString::zeros(100); // 100 zero bits @@ -68,9 +96,9 @@ let v: BitString = (0..8).map(|i| i % 2 == 0).collect(); ```rust use bit_string::BitString; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; -let bits = BitString::try_from("1100_1010").unwrap(); +let bits = BitString::try_from("11001010").unwrap(); // Full view let view = bits.as_bit_str(); // &BitStr = "11001010" @@ -78,7 +106,7 @@ assert_eq!(view.bit_len(), 8); // Sub-slice — zero-copy, O(1) let sub = view.slice(UsizeCO::try_new(2, 6).unwrap()); // "0010" -assert!(sub.starts_with(&BitString::try_from("00").unwrap().as_bit_str())); +assert!(sub.starts_with_str(BitString::try_from("00").unwrap().as_bit_str())); // Convert back to owned let owned = sub.to_bit_string(); // BitString = "0010" @@ -103,7 +131,7 @@ assert_eq!(bits.trailing_ones(), 2); // ends with "11" // Bit-level matching let pattern = BitString::try_from("10").unwrap(); -assert!(bits.starts_with(pattern.as_bit_str())); +assert!(bits.starts_with_str(pattern.as_bit_str())); ``` ### Editing @@ -123,31 +151,29 @@ bits.truncate(4); // "1001" bits.split_off(2); // → BitString "01", bits = "10" // Range operations -use int_interval::UsizeCO; +use int_intervals::UsizeCO; let interval = UsizeCO::try_new(1, 3).unwrap(); -bits.replace_interval(interval, &BitString::ones(2)); // "111" -bits.remove(UsizeCO::try_new(0, 2).unwrap()); // "1" +let replaced = bits.replace_interval(interval, &BitString::ones(2)); // "111" +let drained = replaced.drain_interval(UsizeCO::try_new(0, 2).unwrap()); // "1" ``` ### Matching & searching ```rust -let haystack = BitString::try_from("0010_1100").unwrap(); +let haystack = BitString::try_from("00101100").unwrap(); let needle = BitString::try_from("01").unwrap(); -let np = needle.as_bit_str(); - // Fixed-end checks -assert!(haystack.ends_with(np)); -assert!(!haystack.starts_with(np)); +assert!(!haystack.ends_with_string(&needle)); +assert!(!haystack.starts_with_string(&needle)); // Substring search -assert!(haystack.contains(np)); -assert_eq!(haystack.find(np), Some(1)); // "01" starts at index 1 -assert_eq!(haystack.rfind(np), Some(5)); // last "01" at index 5 +assert!(haystack.contains_string(&needle)); +assert_eq!(haystack.find_string(&needle), Some(1)); // "01" starts at index 1 +assert_eq!(haystack.rfind_string(&needle), Some(3)); // last "01" starts at index 3 // Strip let s = BitString::try_from("00010100").unwrap(); -let stripped = s.strip_prefix(&BitString::try_from("00").unwrap().as_bit_str()); +let stripped = s.strip_prefix_str(BitString::try_from("00").unwrap().as_bit_str()); assert_eq!(stripped.unwrap().to_string(), "010100"); ``` @@ -157,19 +183,19 @@ assert_eq!(stripped.unwrap().to_string(), "010100"); let x = BitString::try_from("1010").unwrap(); let y = BitString::try_from("0110").unwrap(); -// Consuming (`_into`) — reuses allocation -let z = x.and_into(&y).unwrap(); // "0010" -let z = y.or_into(&x).unwrap(); // "1110" -let z = y.xor_into(&x).unwrap(); // "1100" +// Non-mutating operations — return a new BitString +let z = x.and(&y).unwrap(); // "0010" +let z = y.or(&x).unwrap(); // "1110" +let z = y.xor(&x).unwrap(); // "1100" // In-place (`_assign`) let mut w = x.clone(); -w.and_assign(&y); // w = "0010" +w.and_assign(&y).unwrap(); // w = "0010" // Shift let s = BitString::try_from("1001").unwrap(); -assert_eq!(s.shl(2).to_string(), "0100"); // left shift, zero-fill -assert_eq!(s.shr(1).to_string(), "0100"); // right shift, zero-fill +assert_eq!(s.shl(2).to_string(), "0010"); // left shift, zero-fill +assert_eq!(s.shr(1).to_string(), "0010"); // right shift, zero-fill // Not assert_eq!((!s).to_string(), "0110"); @@ -205,10 +231,13 @@ The crate selects the fastest SIMD backend enabled for the compilation target: | Backend | Target | Width | |---------|--------|-------| | AVX2 | x86 / x86_64 | 256-bit (4×u64) | -| SSSE3 | x86 / x86_64 | 128-bit (2×u64) | +| SSE2 / SSE4.1 / SSSE3 | x86 / x86_64 | 128-bit (2×u64) | | NEON | aarch64 | 128-bit (2×u64) | | Scalar | all targets | fallback | +The selected backend is operation-dependent and is determined by the +compile-time target features. + There is no runtime CPU detection. For maximum local performance, compile for the host CPU by copying the example config: ```bash diff --git a/benches/bit_ops_leading.rs b/benches/bit_ops_leading.rs index 31b3706..d50ac20 100644 --- a/benches/bit_ops_leading.rs +++ b/benches/bit_ops_leading.rs @@ -1,7 +1,7 @@ use bit_string::BitString; use bitvec_simd::BitVec; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/benches/bit_ops_trailing.rs b/benches/bit_ops_trailing.rs index ec64823..d6c7758 100644 --- a/benches/bit_ops_trailing.rs +++ b/benches/bit_ops_trailing.rs @@ -1,6 +1,6 @@ use bit_string::BitString; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/benches/editing_replace_interval.rs b/benches/editing_replace_interval.rs index e642304..5fed38a 100644 --- a/benches/editing_replace_interval.rs +++ b/benches/editing_replace_interval.rs @@ -1,6 +1,6 @@ use bit_string::BitString; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/benches/editing_slice.rs b/benches/editing_slice.rs index 296518f..774f7d8 100644 --- a/benches/editing_slice.rs +++ b/benches/editing_slice.rs @@ -1,6 +1,6 @@ use bit_string::BitString; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/benches/matching_matches_at.rs b/benches/matching_matches_at.rs index 6bb4cd9..8f6efc6 100644 --- a/benches/matching_matches_at.rs +++ b/benches/matching_matches_at.rs @@ -1,6 +1,6 @@ use bit_string::BitString; use divan::{Bencher, black_box}; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn main() { divan::main(); diff --git a/derive/Cargo.toml b/derive/Cargo.toml new file mode 100644 index 0000000..1005bcc --- /dev/null +++ b/derive/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "bit-string-derive" +version.workspace = true +edition = "2024" +description = "Procedural macros for bit-string" +repository = "https://github.com/jcfangc/bit-string" +license = "MIT OR Apache-2.0" + +[lib] +proc-macro = true + +[dependencies] +proc-macro-crate = "3.3.0" +proc-macro2 = "1.0" +quote = "1.0" +syn = "2.0" diff --git a/derive/src/funcs_for_expand.rs b/derive/src/funcs_for_expand.rs new file mode 100644 index 0000000..6afed9b --- /dev/null +++ b/derive/src/funcs_for_expand.rs @@ -0,0 +1,147 @@ +use std::collections::BTreeMap; + +use proc_macro_crate::{FoundCrate, crate_name}; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::quote; +use syn::{Data, DeriveInput, Error, Expr, Fields, Lit, LitInt, Result}; + +pub(super) fn expand(input: DeriveInput) -> Result { + require_repr_u8(&input)?; + let (bits, bits_span) = parse_bits(&input)?; + if !(1..=8).contains(&bits) { + return Err(Error::new( + bits_span, + "packed bit width must be between 1 and 8", + )); + } + + let Data::Enum(data) = &input.data else { + return Err(Error::new_spanned( + &input.ident, + "packed can only be applied to enums", + )); + }; + + let limit = 1u16 << bits; + let mut codes = BTreeMap::new(); + let mut variants = Vec::with_capacity(data.variants.len()); + + for variant in &data.variants { + if !matches!(variant.fields, Fields::Unit) { + return Err(Error::new_spanned( + &variant.fields, + "packed variants must not contain fields", + )); + } + + let Some((_, Expr::Lit(expression))) = &variant.discriminant else { + return Err(Error::new_spanned( + &variant.ident, + "packed variants require an explicit integer discriminant", + )); + }; + let Lit::Int(discriminant) = &expression.lit else { + return Err(Error::new_spanned( + &expression.lit, + "packed discriminants must be integer literals", + )); + }; + let code = discriminant.base10_parse::()?; + if code >= limit { + return Err(Error::new_spanned( + discriminant, + format!("discriminant {code} does not fit in {bits} bits"), + )); + } + if let Some(previous) = codes.insert(code, discriminant.span()) { + let mut error = Error::new_spanned(discriminant, "duplicate packed discriminant"); + error.combine(Error::new(previous, "first used here")); + return Err(error); + } + + variants.push((&variant.ident, code)); + } + + let crate_path = bit_string_path(); + let name = &input.ident; + let (impl_generics, type_generics, where_clause) = input.generics.split_for_impl(); + let bits = LitInt::new(&bits.to_string(), bits_span); + let variant_names = variants.iter().map(|(variant, _)| variant); + let discriminants = variants + .iter() + .map(|(_, code)| LitInt::new(&code.to_string(), Span::call_site())); + + Ok(quote! { + impl #impl_generics #crate_path::traits::PackedChar<#bits> + for #name #type_generics #where_clause + { + #[inline] + fn code(self) -> u8 { + self as u8 + } + + #[inline] + fn from_code(code: u8) -> ::core::option::Option { + match code { + #(#discriminants => ::core::option::Option::Some(Self::#variant_names),)* + _ => ::core::option::Option::None, + } + } + } + }) +} + +fn require_repr_u8(input: &DeriveInput) -> Result<()> { + let mut found = false; + for attribute in &input.attrs { + if attribute.path().is_ident("repr") { + attribute.parse_nested_meta(|meta| { + found |= meta.path.is_ident("u8"); + Ok(()) + })?; + } + } + if found { + Ok(()) + } else { + Err(Error::new_spanned( + &input.ident, + "packed requires #[repr(u8)]", + )) + } +} + +fn parse_bits(input: &DeriveInput) -> Result<(u8, Span)> { + let mut bits = None; + for attribute in &input.attrs { + if !attribute.path().is_ident("packed") { + continue; + } + attribute.parse_nested_meta(|meta| { + if !meta.path.is_ident("bits") { + return Err(meta.error("expected `bits = N`")); + } + if bits.is_some() { + return Err(meta.error("duplicate packed bit width")); + } + let literal: LitInt = meta.value()?.parse()?; + bits = Some((literal.base10_parse::()?, literal.span())); + Ok(()) + })?; + } + bits.ok_or_else(|| Error::new_spanned(&input.ident, "packed requires #[packed(bits = N)]")) +} + +fn bit_string_path() -> TokenStream2 { + match crate_name("bit-string") { + Ok(FoundCrate::Itself) => quote!(crate), + Ok(FoundCrate::Name(name)) => { + let name = syn::Ident::new(&name, Span::call_site()); + quote!(::#name) + } + Err(_) => quote!(::bit_string), + } +} + +#[cfg(test)] +mod tests_for_expand; diff --git a/derive/src/funcs_for_expand/tests_for_expand.rs b/derive/src/funcs_for_expand/tests_for_expand.rs new file mode 100644 index 0000000..b8e08be --- /dev/null +++ b/derive/src/funcs_for_expand/tests_for_expand.rs @@ -0,0 +1,115 @@ +use super::*; +use quote::quote; +use syn::parse2; + +fn error(input: TokenStream2) -> String { + expand(parse2(input).unwrap()).unwrap_err().to_string() +} + +#[test] +fn accepts_strict_enum_encoding() { + let result = expand( + parse2(quote! { + #[repr(u8)] + #[packed(bits = 2)] + enum Letter { A = 0, B = 1, C = 2, D = 3 } + }) + .unwrap(), + ); + assert!(result.is_ok()); +} + +#[test] +fn requires_explicit_discriminants() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 1)] + enum Letter { A = 0, B } + }) + .contains("explicit integer discriminant") + ); +} + +#[test] +fn rejects_codes_outside_the_width() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 2)] + enum Letter { A = 4 } + }) + .contains("does not fit in 2 bits") + ); +} + +#[test] +fn rejects_widths_above_u8_capacity() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 9)] + enum Letter { A = 0 } + }) + .contains("must be between 1 and 8") + ); +} + +#[test] +fn rejects_zero_width() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 0)] + enum Letter { A = 0 } + }) + .contains("must be between 1 and 8") + ); +} + +#[test] +fn rejects_variants_with_fields() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 1)] + enum Letter { A(u8) = 0 } + }) + .contains("must not contain fields") + ); +} + +#[test] +fn rejects_duplicate_codes() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 1)] + enum Letter { A = 0, B = 0 } + }) + .contains("duplicate packed discriminant") + ); +} + +#[test] +fn rejects_non_literal_discriminants() { + assert!( + error(quote! { + #[repr(u8)] + #[packed(bits = 2)] + enum Letter { A = 1 + 1 } + }) + .contains("explicit integer discriminant") + ); +} + +#[test] +fn requires_repr_u8() { + assert!( + error(quote! { + #[packed(bits = 1)] + enum Letter { A = 0 } + }) + .contains("requires #[repr(u8)]") + ); +} diff --git a/derive/src/lib.rs b/derive/src/lib.rs new file mode 100644 index 0000000..242b1bc --- /dev/null +++ b/derive/src/lib.rs @@ -0,0 +1,35 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{Attribute, DeriveInput, parse_macro_input, parse_quote}; + +mod funcs_for_expand; + +use funcs_for_expand::expand; + +/// Marks a fieldless enum as a packed alphabet and supplies its implementation. +/// +/// This is the preferred spelling for new code: +/// `#[packed(bits = 2)] enum Symbol { ... }`. +#[proc_macro_attribute] +pub fn packed(args: TokenStream, input: TokenStream) -> TokenStream { + let mut item = parse_macro_input!(input as DeriveInput); + let bits: proc_macro2::TokenStream = args.into(); + + let packed: Attribute = parse_quote!(#[packed(#bits)]); + item.attrs.push(packed); + if !item + .attrs + .iter() + .any(|attribute| attribute.path().is_ident("repr")) + { + item.attrs.push(parse_quote!(#[repr(u8)])); + } + + let implementation = match expand(item.clone()) { + Ok(tokens) => tokens, + Err(error) => error.into_compile_error(), + }; + item.attrs + .retain(|attribute| !attribute.path().is_ident("packed")); + quote!(#item #implementation).into() +} diff --git a/pyproject.toml b/pyproject.toml index 69e1a2e..9aa5d11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [project] -name = "bit-string" +name = "bit-string-dev-tools" version = "0.1.0" requires-python = ">=3.11" dependencies = [] + +[tool.uv] +package = false diff --git a/scripts/bench_config.py b/scripts/bench_config.py new file mode 100644 index 0000000..509c677 --- /dev/null +++ b/scripts/bench_config.py @@ -0,0 +1,4 @@ +"""Shared constants for reproducible benchmark runs.""" + +BASELINE_SHA = "c15fa1c254ea5b2bbabfe5f008d41f91e8297b89" +BENCH_RUSTFLAGS = "-C target-cpu=native" diff --git a/scripts/compare_bench.py b/scripts/compare_bench.py new file mode 100755 index 0000000..8cbdd75 --- /dev/null +++ b/scripts/compare_bench.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Compare repeated Divan reports using the median of per-run medians.""" + +from __future__ import annotations + +import argparse +import statistics +import sys +from pathlib import Path + +from divan_parser import parse_report + + +def aggregate(paths: list[Path], expected_runs: int) -> dict[str, float]: + if len(paths) != expected_runs: + raise ValueError(f"expected {expected_runs} reports, got {len(paths)}") + reports = [parse_report(path) for path in paths] + expected_names = set(reports[0]) + for path, report in zip(paths[1:], reports[1:], strict=True): + if set(report) != expected_names: + missing = sorted(expected_names - set(report)) + extra = sorted(set(report) - expected_names) + raise ValueError(f"{path}: inconsistent benchmark set; missing={missing}, extra={extra}") + return {name: statistics.median(report[name].median_ns for report in reports) for name in expected_names} + + +def fmt_ns(value: float) -> str: + if value >= 1_000_000_000: + return f"{value / 1_000_000_000:.3f} s" + if value >= 1_000_000: + return f"{value / 1_000_000:.3f} ms" + if value >= 1_000: + return f"{value / 1_000:.3f} µs" + if value < 1: + return f"{value * 1_000:.3f} ps" + return f"{value:.3f} ns" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("baseline", type=Path, nargs="+", help="baseline Divan text reports") + parser.add_argument("--current", type=Path, nargs="+", required=True, help="current Divan text reports") + parser.add_argument("--expected-runs", type=int, default=5) + parser.add_argument("--warning-threshold", type=float, default=5.0) + args = parser.parse_args() + try: + baseline = aggregate(args.baseline, args.expected_runs) + current = aggregate(args.current, args.expected_runs) + except ValueError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 2 + only_baseline = sorted(baseline.keys() - current.keys()) + only_current = sorted(current.keys() - baseline.keys()) + if only_baseline or only_current: + for name in only_baseline: + print(f"ERROR: missing in current: {name}", file=sys.stderr) + for name in only_current: + print(f"ERROR: missing in baseline: {name}", file=sys.stderr) + return 2 + improved = unchanged = regressed = warnings = 0 + print(f"{'benchmark':<50} {'baseline':>12} {'current':>12} {'delta':>10} status") + print("-" * 96) + for name in sorted(baseline): + before, after = baseline[name], current[name] + delta = (after - before) / before * 100 + improved += delta < 0 + unchanged += delta == 0 + regressed += delta > 0 + status = "WARNING" if delta > args.warning_threshold else "OK" + if status == "WARNING": + warnings += 1 + print(f"WARNING: {name} regressed by {delta:+.2f}%", file=sys.stderr) + print(f"{name:<50.50} {fmt_ns(before):>12} {fmt_ns(after):>12} {delta:>+9.2f}% {status}") + print("\nSUMMARY") + print(f"matched benchmarks : {len(baseline)}") + print(f"improved : {improved}") + print(f"unchanged : {unchanged}") + print(f"regressed : {regressed}") + print(f"warnings : {warnings}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/divan_parser.py b/scripts/divan_parser.py new file mode 100644 index 0000000..2915991 --- /dev/null +++ b/scripts/divan_parser.py @@ -0,0 +1,54 @@ +"""Strict parser for Divan's text table format.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +TIME = re.compile(r"(?P[0-9]+(?:\.[0-9]+)?)\s*(?Pps|ns|µs|us|ms|s)\b") +ROW = re.compile(r"^[│\s]*[├╰]─\s*(?P.+)$") +COUNTS = re.compile(r"│\s*(?P[0-9]+)\s*│\s*(?P[0-9]+)\s*$") +UNITS_TO_NS = {"ps": 1e-3, "ns": 1.0, "µs": 1_000.0, "us": 1_000.0, "ms": 1_000_000.0, "s": 1_000_000_000.0} + + +@dataclass(frozen=True) +class DivanResult: + fastest_ns: float + slowest_ns: float + median_ns: float + mean_ns: float + samples: int + iters: int + + +def _time_ns(match: re.Match[str]) -> float: + return float(match.group("value")) * UNITS_TO_NS[match.group("unit")] + + +def parse_report(path: Path) -> dict[str, DivanResult]: + result: dict[str, DivanResult] = {} + pending: str | None = None + for raw in path.read_text(encoding="utf-8").splitlines(): + row = ROW.match(raw) + if row: + body = row.group("body") + first_time = TIME.search(body) + end = first_time.start() if first_time else body.find("Timer precision:") + pending = body[: end if end >= 0 else None].rstrip() + if pending is None: + continue + times = list(TIME.finditer(raw)) + if len(times) < 4: + continue + counts = COUNTS.search(raw) + if counts is None: + raise ValueError(f"{path}: benchmark row for {pending!r} has no samples/iters") + if pending in result: + raise ValueError(f"{path}: duplicate benchmark name {pending!r}") + values = [_time_ns(match) for match in times[:4]] + result[pending] = DivanResult(*values, int(counts.group("samples")), int(counts.group("iters"))) + pending = None + if not result: + raise ValueError(f"{path}: no Divan benchmark rows found") + return result diff --git a/scripts/run_bench_comparison.py b/scripts/run_bench_comparison.py new file mode 100755 index 0000000..3c65635 --- /dev/null +++ b/scripts/run_bench_comparison.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Run interleaved Divan benchmarks for a fixed baseline and this worktree.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from bench_config import BASELINE_SHA, BENCH_RUSTFLAGS + +ORDER = ("baseline", "current", "current", "baseline", "baseline", "current", "current", "baseline", "baseline", "current") + + +def run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None, output: Path | None = None) -> None: + completed = subprocess.run(command, cwd=cwd, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + if output is not None: + output.write_text(completed.stdout, encoding="utf-8") + if completed.returncode: + sys.stdout.write(completed.stdout) + raise SystemExit(f"command failed ({completed.returncode}): {' '.join(command)}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--runs", type=int, default=5, help="runs per revision; default is 5") + parser.add_argument("--warning-threshold", type=float, default=5.0) + parser.add_argument("--baseline", default=BASELINE_SHA) + args = parser.parse_args() + if args.runs != 5: + raise SystemExit("this runner currently fixes --runs at 5 to preserve the documented interleaving") + + root = Path(__file__).resolve().parents[1] + output_dir = root / "target" / "bench-comparison" + baseline_dir = output_dir / "baseline-worktree" + output_dir.mkdir(parents=True, exist_ok=True) + if not baseline_dir.exists(): + run(["git", "worktree", "add", "--detach", str(baseline_dir), args.baseline], cwd=root) + else: + run(["git", "-C", str(baseline_dir), "checkout", "--detach", args.baseline], cwd=root) + + env = os.environ.copy() + env["RUSTFLAGS"] = BENCH_RUSTFLAGS + command = ["cargo", "bench", "--", "--color", "never"] + counts = {"baseline": 0, "current": 0} + for revision in ORDER: + counts[revision] += 1 + report = output_dir / f"{revision}-{counts[revision]}.txt" + print(f"[{revision} {counts[revision]}/5] {' '.join(command)}") + run(command, cwd=baseline_dir if revision == "baseline" else root, env=env, output=report) + + compare = [sys.executable, str(root / "scripts" / "compare_bench.py")] + compare += [str(output_dir / f"baseline-{i}.txt") for i in range(1, 6)] + compare += ["--current"] + [str(output_dir / f"current-{i}.txt") for i in range(1, 6)] + compare += ["--expected-runs", str(args.runs)] + compare += ["--warning-threshold", str(args.warning_threshold)] + return subprocess.run(compare, cwd=root, check=False).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_bench.py b/scripts/tests/test_bench.py new file mode 100644 index 0000000..7b091c9 --- /dev/null +++ b/scripts/tests/test_bench.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import contextlib +import io +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import compare_bench +from divan_parser import parse_report + +ROW = "├─ bench 500 ps │ 1 ns │ 750 ps │ 800 ps │ 100 │ 1000\n" + + +class DivanParserTests(unittest.TestCase): + def write(self, directory: str, name: str, content: str = ROW) -> Path: + path = Path(directory) / name + path.write_text(content) + return path + + def test_picoseconds_are_converted_to_ns(self) -> None: + with tempfile.TemporaryDirectory() as directory: + result = parse_report(self.write(directory, "report.txt"))["bench"] + self.assertEqual(result.median_ns, 0.75) + self.assertEqual(result.samples, 100) + self.assertEqual(result.iters, 1000) + + def test_duplicate_benchmark_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory, self.assertRaisesRegex(ValueError, "duplicate"): + parse_report(self.write(directory, "report.txt", ROW + ROW)) + + def test_expected_run_count_is_enforced(self) -> None: + with tempfile.TemporaryDirectory() as directory: + paths = [self.write(directory, f"report-{index}.txt") for index in range(4)] + with self.assertRaisesRegex(ValueError, "expected 5 reports"): + compare_bench.aggregate(paths, 5) + + def test_missing_benchmark_in_one_run_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + paths = [self.write(directory, f"report-{index}.txt") for index in range(5)] + paths[-1].write_text(ROW.replace("bench", "different")) + with self.assertRaisesRegex(ValueError, "inconsistent benchmark set"): + compare_bench.aggregate(paths, 5) + + def test_missing_between_revisions_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + baseline = [self.write(directory, f"baseline-{index}.txt") for index in range(5)] + current = [self.write(directory, f"current-{index}.txt", ROW.replace("bench", "other")) for index in range(5)] + argv = [ + "compare_bench.py", + *(str(path) for path in baseline), + "--current", + *(str(path) for path in current), + ] + with ( + mock.patch.object(sys, "argv", argv), + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertNotEqual(compare_bench.main(), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bit_str.rs b/src/bit_str.rs index ee725f1..5119c87 100644 --- a/src/bit_str.rs +++ b/src/bit_str.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_access/tests_for_get.rs b/src/bit_str/impls_for_access/tests_for_get.rs index fcb165a..2ab1496 100644 --- a/src/bit_str/impls_for_access/tests_for_get.rs +++ b/src/bit_str/impls_for_access/tests_for_get.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitStr; use crate::BitString; diff --git a/src/bit_str/impls_for_access/tests_for_get_chunk.rs b/src/bit_str/impls_for_access/tests_for_get_chunk.rs index c0360b8..628a0b2 100644 --- a/src/bit_str/impls_for_access/tests_for_get_chunk.rs +++ b/src/bit_str/impls_for_access/tests_for_get_chunk.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitStr; use crate::BitString; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_ones.rs b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_ones.rs index 98a86eb..c28c043 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_ones.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_ones.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_zeros.rs b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_zeros.rs index f3dd482..b60756f 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_zeros.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_leading_zeros.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs index 21eea89..892c381 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs index ae8d5a1..0cddb2f 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_iter/tests_for_iter.rs b/src/bit_str/impls_for_iter/tests_for_iter.rs index 1154ebf..0f85c64 100644 --- a/src/bit_str/impls_for_iter/tests_for_iter.rs +++ b/src/bit_str/impls_for_iter/tests_for_iter.rs @@ -1,5 +1,5 @@ use alloc::vec::Vec; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_str/impls_for_slice.rs b/src/bit_str/impls_for_slice.rs index 9b89e99..25af156 100644 --- a/src/bit_str/impls_for_slice.rs +++ b/src/bit_str/impls_for_slice.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use super::*; diff --git a/src/bit_str/impls_for_slice/tests_for_slice.rs b/src/bit_str/impls_for_slice/tests_for_slice.rs index b6f3b27..526a734 100644 --- a/src/bit_str/impls_for_slice/tests_for_slice.rs +++ b/src/bit_str/impls_for_slice/tests_for_slice.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_string.rs b/src/bit_string.rs index 474e7e6..621c25a 100644 --- a/src/bit_string.rs +++ b/src/bit_string.rs @@ -23,7 +23,7 @@ impl BitString { &self.words } - /// Returns a zero-copy [`BitStr`] view of the entire bit string. + /// Returns a zero-copy [`crate::BitStr`] view of the entire bit string. #[inline] pub fn as_bit_str(&self) -> crate::BitStr<'_> { crate::BitStr { diff --git a/src/bit_string/impls_for_bit_arith.rs b/src/bit_string/impls_for_bit_arith.rs index 5e26ab7..b207815 100644 --- a/src/bit_string/impls_for_bit_arith.rs +++ b/src/bit_string/impls_for_bit_arith.rs @@ -5,6 +5,7 @@ mod impls_for_not; mod impls_for_or; mod impls_for_shl; mod impls_for_shr; +mod impls_for_trailing_zeros; mod impls_for_xor; use crate::bit_string::errors::BitStringLenMismatch; diff --git a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs index 4a17438..845e095 100644 --- a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs +++ b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs @@ -119,132 +119,4 @@ impl BitString { self.words() .leading_value_bits::(0, bit_len) } - - /// Returns the number of consecutive `false` bits from the end. - #[inline] - pub fn trailing_zeros(&self) -> usize { - let bit_len = self.bit_len; - if bit_len == 0 { - return 0; - } - let words_ptr = self.words.as_ptr(); - - if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { - return count; - } - - // ── Last partial word ──────────────────────────────────── - let end_rem = bit_len % WORD_BITS; - if end_rem != 0 { - let last_wi = (bit_len - 1) / WORD_BITS; - // SAFETY: `last_wi` is a valid index — `bit_len > 0` and - // `words` always has ≥ (bit_len + 63) / 64 elements. - let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); - if last != 0 { - let shifted = last << (WORD_BITS - end_rem); - return (shifted.leading_zeros() as usize).min(bit_len); - } - } - - // ── Rightmost full word ────────────────────────────────── - if bit_len > WORD_BITS { - let last_full = if end_rem != 0 { - (bit_len - 1) / WORD_BITS - 1 - } else { - (bit_len - 1) / WORD_BITS - }; - // SAFETY: `last_full < (bit_len + 63) / 64`, so it is a valid - // index into `words` (backing storage covers all bits). - let w = unsafe { *words_ptr.add(last_full) }; - if w != 0 { - let tail = if end_rem != 0 { end_rem } else { 0 }; - return (tail + w.leading_zeros() as usize).min(bit_len); - } - } - - self.words() - .trailing_value_bits::(0, bit_len) - } - - /// Returns the number of consecutive `true` bits from the end. - #[inline] - pub fn trailing_ones(&self) -> usize { - let bit_len = self.bit_len; - if bit_len == 0 { - return 0; - } - let words_ptr = self.words.as_ptr(); - - if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { - return count; - } - - let end_rem = bit_len % WORD_BITS; - if end_rem != 0 { - let last_wi = (bit_len - 1) / WORD_BITS; - // SAFETY: `last_wi` is a valid index per the same invariant - // as trailing_zeros (BitString backing storage covers all bits). - let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); - if last != ((1u64 << end_rem).wrapping_sub(1)) { - let shifted = (!last) << (WORD_BITS - end_rem); - return (shifted.leading_zeros() as usize).min(bit_len); - } - } - - if bit_len > WORD_BITS { - let last_full = if end_rem != 0 { - (bit_len - 1) / WORD_BITS - 1 - } else { - (bit_len - 1) / WORD_BITS - }; - // SAFETY: `last_full` is a valid index into `words` per the - // BitString backing-storage invariant. - let w = unsafe { *words_ptr.add(last_full) }; - if w != u64::MAX { - let tail = if end_rem != 0 { end_rem } else { 0 }; - return (tail + (!w).leading_zeros() as usize).min(bit_len); - } - } - - self.words() - .trailing_value_bits::(0, bit_len) - } -} - -/// Handles owned strings that fit in at most two words without entering the -/// generic reverse scanner. -/// -/// # Safety -/// -/// `words_ptr` must point to the backing storage for `bit_len` bits. -#[inline] -unsafe fn trailing_small(words_ptr: *const u64, bit_len: usize) -> Option { - if bit_len > WORD_BITS * 2 { - return None; - } - - let last_wi = (bit_len - 1) / WORD_BITS; - let used = bit_len - last_wi * WORD_BITS; - let mask = if used == WORD_BITS { - u64::MAX - } else { - (1u64 << used) - 1 - }; - // SAFETY: the caller guarantees backing storage for every covered word. - let last = unsafe { *words_ptr.add(last_wi) }; - let mismatch = (last ^ FILL) & mask; - if mismatch != 0 { - return Some((mismatch << (WORD_BITS - used)).leading_zeros() as usize); - } - if last_wi == 0 { - return Some(bit_len); - } - - // SAFETY: last_wi == 1, so the first backing word also exists. - let first_mismatch = unsafe { *words_ptr } ^ FILL; - if first_mismatch == 0 { - Some(bit_len) - } else { - Some(used + first_mismatch.leading_zeros() as usize) - } } diff --git a/src/bit_string/impls_for_bit_arith/impls_for_trailing_zeros.rs b/src/bit_string/impls_for_bit_arith/impls_for_trailing_zeros.rs new file mode 100644 index 0000000..fa9d4c1 --- /dev/null +++ b/src/bit_string/impls_for_bit_arith/impls_for_trailing_zeros.rs @@ -0,0 +1,134 @@ +use crate::traits::WordsScan; +use crate::{FILL_ONES, FILL_ZEROS, WORD_BITS}; + +use super::BitString; + +impl BitString { + /// Returns the number of consecutive `false` bits from the end. + #[inline] + pub fn trailing_zeros(&self) -> usize { + let bit_len = self.bit_len; + if bit_len == 0 { + return 0; + } + let words_ptr = self.words.as_ptr(); + + if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { + return count; + } + + // ── Last partial word ──────────────────────────────────── + let end_rem = bit_len % WORD_BITS; + if end_rem != 0 { + let last_wi = (bit_len - 1) / WORD_BITS; + // SAFETY: `last_wi` is a valid index — `bit_len > 0` and + // `words` always has ≥ (bit_len + 63) / 64 elements. + let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); + if last != 0 { + let shifted = last << (WORD_BITS - end_rem); + return (shifted.leading_zeros() as usize).min(bit_len); + } + } + + // ── Rightmost full word ────────────────────────────────── + if bit_len > WORD_BITS { + let last_full = if end_rem != 0 { + (bit_len - 1) / WORD_BITS - 1 + } else { + (bit_len - 1) / WORD_BITS + }; + // SAFETY: `last_full < (bit_len + 63) / 64`, so it is a valid + // index into `words` (backing storage covers all bits). + let w = unsafe { *words_ptr.add(last_full) }; + if w != 0 { + let tail = if end_rem != 0 { end_rem } else { 0 }; + return (tail + w.leading_zeros() as usize).min(bit_len); + } + } + + self.words() + .trailing_value_bits::(0, bit_len) + } + + /// Returns the number of consecutive `true` bits from the end. + #[inline] + pub fn trailing_ones(&self) -> usize { + let bit_len = self.bit_len; + if bit_len == 0 { + return 0; + } + let words_ptr = self.words.as_ptr(); + + if let Some(count) = unsafe { trailing_small::(words_ptr, bit_len) } { + return count; + } + + let end_rem = bit_len % WORD_BITS; + if end_rem != 0 { + let last_wi = (bit_len - 1) / WORD_BITS; + // SAFETY: `last_wi` is a valid index per the same invariant + // as trailing_zeros (BitString backing storage covers all bits). + let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); + if last != ((1u64 << end_rem).wrapping_sub(1)) { + let shifted = (!last) << (WORD_BITS - end_rem); + return (shifted.leading_zeros() as usize).min(bit_len); + } + } + + if bit_len > WORD_BITS { + let last_full = if end_rem != 0 { + (bit_len - 1) / WORD_BITS - 1 + } else { + (bit_len - 1) / WORD_BITS + }; + // SAFETY: `last_full` is a valid index into `words` per the + // BitString backing-storage invariant. + let w = unsafe { *words_ptr.add(last_full) }; + if w != u64::MAX { + let tail = if end_rem != 0 { end_rem } else { 0 }; + return (tail + (!w).leading_zeros() as usize).min(bit_len); + } + } + + self.words() + .trailing_value_bits::(0, bit_len) + } +} + +/// Handles owned strings that fit in at most two words without entering the +/// generic reverse scanner. +/// +/// # Safety +/// +/// `words_ptr` must point to the backing storage for `bit_len` bits. +#[inline] +unsafe fn trailing_small(words_ptr: *const u64, bit_len: usize) -> Option { + if bit_len > WORD_BITS * 2 { + return None; + } + + let last_wi = (bit_len - 1) / WORD_BITS; + let used = bit_len - last_wi * WORD_BITS; + let mask = if used == WORD_BITS { + u64::MAX + } else { + (1u64 << used) - 1 + }; + // SAFETY: the caller guarantees backing storage for every covered word. + let last = unsafe { *words_ptr.add(last_wi) }; + let mismatch = (last ^ FILL) & mask; + if mismatch != 0 { + return Some((mismatch << (WORD_BITS - used)).leading_zeros() as usize); + } + if last_wi == 0 { + return Some(bit_len); + } + + // SAFETY: last_wi == 1, so the first backing word also exists. + let first_mismatch = unsafe { *words_ptr } ^ FILL; + if first_mismatch == 0 { + Some(bit_len) + } else { + Some(used + first_mismatch.leading_zeros() as usize) + } +} diff --git a/src/bit_string/impls_for_construction.rs b/src/bit_string/impls_for_construction.rs index 1b0fc74..6571ad5 100644 --- a/src/bit_string/impls_for_construction.rs +++ b/src/bit_string/impls_for_construction.rs @@ -1,4 +1,6 @@ mod funcs_for_pack_bools_core; +mod impls_for_from_bool_iter; +mod impls_for_from_bool_slice; mod impls_for_from_str; mod impls_for_from_words; mod impls_for_new; diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs index a395135..7969596 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs @@ -70,273 +70,19 @@ unsafe fn dispatch(dst: *mut u64, src: *const u8, bit_len: usize) { } #[allow(unused)] -mod scalar { - use super::scalar; - - /// Scalar backend: accumulate 64 bytes at a time into one u64. - /// - /// # Safety - /// - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[inline] - pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { - while bit_len >= 64 { - // SAFETY: `bit_len >= 64`, so reading 64 bytes from `src` is valid. - // `dst` points to the next output slot. - unsafe { - *dst = scalar::pack_64(src); - } - // SAFETY: - // - `dst` advances by 1 word; the caller ensures the destination - // has enough capacity for all full-word writes. - // - `src` advances by 64 bytes; `bit_len >= 64` ensures read bounds. - unsafe { - dst = dst.add(1); - src = src.add(64); - } - bit_len -= 64; - } - - if bit_len > 0 { - // SAFETY: `bit_len > 0`, so reading `bit_len` bytes from `src` is valid. - unsafe { - *dst = scalar::pack_partial(src, bit_len); - } - } - } - - /// Pack exactly 64 bytes into one u64 (little-endian: byte i → bit i). - /// - /// # Safety - /// - /// `src` must be valid for reads of 64 u8 values. - #[inline] - unsafe fn pack_64(src: *const u8) -> u64 { - let mut word = 0u64; - for i in 0..64 { - // SAFETY: - // - `i < 64`, offset is in bounds. - // - `src` is valid for 64 reads per caller contract. - let byte = unsafe { src.add(i).read() }; - word |= ((byte & 1) as u64) << i; - } - word - } - - /// Pack fewer than 64 bytes into one u64. - /// - /// # Safety - /// - /// `src` must be valid for reads of `len` u8 values (`len < 64`). - #[inline] - unsafe fn pack_partial(src: *const u8, len: usize) -> u64 { - let mut word = 0u64; - for i in 0..len { - // SAFETY: - // - `i < len < 64`. - // - `src` is valid for `len` reads per caller contract. - let byte = unsafe { src.add(i).read() }; - word |= ((byte & 1) as u64) << i; - } - word - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi8, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi8, - }; - - const LANES: usize = 32; - - /// AVX2 backend: 32 bytes → 32 movemask bits, 2 iterations per u64. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { - // SAFETY: this function is only callable when AVX2 is available - // (enforced by the caller / dispatch gating). - let ones = _mm256_set1_epi8(1); - - while bit_len >= 64 { - // SAFETY: - // - `bit_len >= 64`, so two 32-byte reads from `src` are in bounds. - // - `_mm256_loadu_si256` permits unaligned loads. - let lo = unsafe { _mm256_loadu_si256(src.cast::<__m256i>()) }; - // SAFETY: `src + 32` is valid; `bit_len >= 64`. - let hi = unsafe { _mm256_loadu_si256(src.add(LANES).cast::<__m256i>()) }; - - // cmpeq extracts LSB: 0x01 → 0xFF, 0x00 → 0x00 - let lo_eq = _mm256_cmpeq_epi8(lo, ones); - let hi_eq = _mm256_cmpeq_epi8(hi, ones); - - // movemask takes the MSB of each byte → the comparison result. - let lo_bits = _mm256_movemask_epi8(lo_eq) as u32 as u64; - let hi_bits = _mm256_movemask_epi8(hi_eq) as u32 as u64; - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = lo_bits | (hi_bits << 32); - } - - // SAFETY: destination has capacity; source has `bit_len >= 64`. - unsafe { - dst = dst.add(1); - src = src.add(64); - } - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`, delegate tail to scalar. - unsafe { - scalar::words(dst, src, bit_len); - } - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi8, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi8, - }; - - const LANES: usize = 16; - - /// SSE2 backend: 16 bytes → 16 movemask bits, 4 iterations per u64. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { - let ones = _mm_set1_epi8(1); - - while bit_len >= 64 { - // SAFETY: `bit_len >= 64`, so four 16-byte reads from `src` are in bounds. - // `_mm_loadu_si128` permits unaligned loads. - let v0 = unsafe { _mm_loadu_si128(src.cast::<__m128i>()) }; - let v1 = unsafe { _mm_loadu_si128(src.add(LANES).cast::<__m128i>()) }; - let v2 = unsafe { _mm_loadu_si128(src.add(LANES * 2).cast::<__m128i>()) }; - let v3 = unsafe { _mm_loadu_si128(src.add(LANES * 3).cast::<__m128i>()) }; - - let m0 = _mm_movemask_epi8(_mm_cmpeq_epi8(v0, ones)) as u32 as u64; - let m1 = _mm_movemask_epi8(_mm_cmpeq_epi8(v1, ones)) as u32 as u64; - let m2 = _mm_movemask_epi8(_mm_cmpeq_epi8(v2, ones)) as u32 as u64; - let m3 = _mm_movemask_epi8(_mm_cmpeq_epi8(v3, ones)) as u32 as u64; - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); - } - - // SAFETY: destination has capacity; source has `bit_len >= 64`. - unsafe { - dst = dst.add(1); - src = src.add(64); - } - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`, delegate tail to scalar. - unsafe { - scalar::words(dst, src, bit_len); - } - } -} +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::scalar; - - use core::arch::aarch64::{ - vand_u8, vceq_u8, vdup_n_u8, vget_lane_u64, vld1_u8, vpaddl_u8, vpaddl_u16, vpaddl_u32, - }; - - /// Bit-position masks: [1, 2, 4, 8, 16, 32, 64, 128]. - const BIT_MASKS: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; - - /// NEON backend: packs 64 bytes at a time into one u64. - /// - /// Each 8-byte group contributes bits at `group * 8` offset within the - /// current word, then 8 groups are OR'd together. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { - // SAFETY: constant pointer to static mask array. - let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; - let ones = vdup_n_u8(1); - - while bit_len >= 64 { - // Accumulate 8 groups × 8 bytes → one u64. - let mut word = 0u64; - for group in 0..8 { - // SAFETY: `bit_len >= 64` and `group < 8`, so the load - // is within bounds. `vld1_u8` permits unaligned reads. - let bytes = unsafe { vld1_u8(src.add(group * 8)) }; - - // Expand 1 → 0xFF (0 stays 0x00), then mask to position - // each bit before reducing to a single u64 via pairwise adds. - let is_one = vceq_u8(bytes, ones); - let masked = vand_u8(is_one, bit_masks); - let sum16 = vpaddl_u8(masked); - let sum32 = vpaddl_u16(sum16); - let sum64 = vpaddl_u32(sum32); - - let group_bits = vget_lane_u64::<0>(sum64); - word |= group_bits << (group * 8); - } - - // SAFETY: `dst` points to the current output slot; destination - // has capacity for this write per the caller's contract. - unsafe { - *dst = word; - dst = dst.add(1); - src = src.add(64); - } - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`, delegate tail to scalar. - unsafe { - scalar::words(dst, src, bit_len); - } - } -} - -mod impls_for_from_bool_iter; -mod impls_for_from_bool_slice; +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/avx2.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/avx2.rs new file mode 100644 index 0000000..a14044e --- /dev/null +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/avx2.rs @@ -0,0 +1,61 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi8, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi8, +}; + +const LANES: usize = 32; + +/// AVX2 backend: 32 bytes → 32 movemask bits, 2 iterations per u64. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { + // SAFETY: this function is only callable when AVX2 is available + // (enforced by the caller / dispatch gating). + let ones = _mm256_set1_epi8(1); + + while bit_len >= 64 { + // SAFETY: + // - `bit_len >= 64`, so two 32-byte reads from `src` are in bounds. + // - `_mm256_loadu_si256` permits unaligned loads. + let lo = unsafe { _mm256_loadu_si256(src.cast::<__m256i>()) }; + // SAFETY: `src + 32` is valid; `bit_len >= 64`. + let hi = unsafe { _mm256_loadu_si256(src.add(LANES).cast::<__m256i>()) }; + + // cmpeq extracts LSB: 0x01 → 0xFF, 0x00 → 0x00 + let lo_eq = _mm256_cmpeq_epi8(lo, ones); + let hi_eq = _mm256_cmpeq_epi8(hi, ones); + + // movemask takes the MSB of each byte → the comparison result. + let lo_bits = _mm256_movemask_epi8(lo_eq) as u32 as u64; + let hi_bits = _mm256_movemask_epi8(hi_eq) as u32 as u64; + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = lo_bits | (hi_bits << 32); + } + + // SAFETY: destination has capacity; source has `bit_len >= 64`. + unsafe { + dst = dst.add(1); + src = src.add(64); + } + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`, delegate tail to scalar. + unsafe { + scalar::words(dst, src, bit_len); + } +} diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/neon.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/neon.rs new file mode 100644 index 0000000..95c7fce --- /dev/null +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/neon.rs @@ -0,0 +1,60 @@ +use super::scalar; + +use core::arch::aarch64::{ + vand_u8, vceq_u8, vdup_n_u8, vget_lane_u64, vld1_u8, vpaddl_u8, vpaddl_u16, vpaddl_u32, +}; + +/// Bit-position masks: [1, 2, 4, 8, 16, 32, 64, 128]. +const BIT_MASKS: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; + +/// NEON backend: packs 64 bytes at a time into one u64. +/// +/// Each 8-byte group contributes bits at `group * 8` offset within the +/// current word, then 8 groups are OR'd together. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { + // SAFETY: constant pointer to static mask array. + let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; + let ones = vdup_n_u8(1); + + while bit_len >= 64 { + // Accumulate 8 groups × 8 bytes → one u64. + let mut word = 0u64; + for group in 0..8 { + // SAFETY: `bit_len >= 64` and `group < 8`, so the load + // is within bounds. `vld1_u8` permits unaligned reads. + let bytes = unsafe { vld1_u8(src.add(group * 8)) }; + + // Expand 1 → 0xFF (0 stays 0x00), then mask to position + // each bit before reducing to a single u64 via pairwise adds. + let is_one = vceq_u8(bytes, ones); + let masked = vand_u8(is_one, bit_masks); + let sum16 = vpaddl_u8(masked); + let sum32 = vpaddl_u16(sum16); + let sum64 = vpaddl_u32(sum32); + + let group_bits = vget_lane_u64::<0>(sum64); + word |= group_bits << (group * 8); + } + + // SAFETY: `dst` points to the current output slot; destination + // has capacity for this write per the caller's contract. + unsafe { + *dst = word; + dst = dst.add(1); + src = src.add(64); + } + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`, delegate tail to scalar. + unsafe { + scalar::words(dst, src, bit_len); + } +} diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/scalar.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/scalar.rs new file mode 100644 index 0000000..7c0637c --- /dev/null +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/scalar.rs @@ -0,0 +1,70 @@ +use super::scalar; + +/// Scalar backend: accumulate 64 bytes at a time into one u64. +/// +/// # Safety +/// +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[inline] +pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { + while bit_len >= 64 { + // SAFETY: `bit_len >= 64`, so reading 64 bytes from `src` is valid. + // `dst` points to the next output slot. + unsafe { + *dst = scalar::pack_64(src); + } + // SAFETY: + // - `dst` advances by 1 word; the caller ensures the destination + // has enough capacity for all full-word writes. + // - `src` advances by 64 bytes; `bit_len >= 64` ensures read bounds. + unsafe { + dst = dst.add(1); + src = src.add(64); + } + bit_len -= 64; + } + + if bit_len > 0 { + // SAFETY: `bit_len > 0`, so reading `bit_len` bytes from `src` is valid. + unsafe { + *dst = scalar::pack_partial(src, bit_len); + } + } +} + +/// Pack exactly 64 bytes into one u64 (little-endian: byte i → bit i). +/// +/// # Safety +/// +/// `src` must be valid for reads of 64 u8 values. +#[inline] +unsafe fn pack_64(src: *const u8) -> u64 { + let mut word = 0u64; + for i in 0..64 { + // SAFETY: + // - `i < 64`, offset is in bounds. + // - `src` is valid for 64 reads per caller contract. + let byte = unsafe { src.add(i).read() }; + word |= ((byte & 1) as u64) << i; + } + word +} + +/// Pack fewer than 64 bytes into one u64. +/// +/// # Safety +/// +/// `src` must be valid for reads of `len` u8 values (`len < 64`). +#[inline] +unsafe fn pack_partial(src: *const u8, len: usize) -> u64 { + let mut word = 0u64; + for i in 0..len { + // SAFETY: + // - `i < len < 64`. + // - `src` is valid for `len` reads per caller contract. + let byte = unsafe { src.add(i).read() }; + word |= ((byte & 1) as u64) << i; + } + word +} diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/sse2.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/sse2.rs new file mode 100644 index 0000000..4f29401 --- /dev/null +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/sse2.rs @@ -0,0 +1,54 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi8}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi8, +}; + +const LANES: usize = 16; + +/// SSE2 backend: 16 bytes → 16 movemask bits, 4 iterations per u64. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words(mut dst: *mut u64, mut src: *const u8, mut bit_len: usize) { + let ones = _mm_set1_epi8(1); + + while bit_len >= 64 { + // SAFETY: `bit_len >= 64`, so four 16-byte reads from `src` are in bounds. + // `_mm_loadu_si128` permits unaligned loads. + let v0 = unsafe { _mm_loadu_si128(src.cast::<__m128i>()) }; + let v1 = unsafe { _mm_loadu_si128(src.add(LANES).cast::<__m128i>()) }; + let v2 = unsafe { _mm_loadu_si128(src.add(LANES * 2).cast::<__m128i>()) }; + let v3 = unsafe { _mm_loadu_si128(src.add(LANES * 3).cast::<__m128i>()) }; + + let m0 = _mm_movemask_epi8(_mm_cmpeq_epi8(v0, ones)) as u32 as u64; + let m1 = _mm_movemask_epi8(_mm_cmpeq_epi8(v1, ones)) as u32 as u64; + let m2 = _mm_movemask_epi8(_mm_cmpeq_epi8(v2, ones)) as u32 as u64; + let m3 = _mm_movemask_epi8(_mm_cmpeq_epi8(v3, ones)) as u32 as u64; + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); + } + + // SAFETY: destination has capacity; source has `bit_len >= 64`. + unsafe { + dst = dst.add(1); + src = src.add(64); + } + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`, delegate tail to scalar. + unsafe { + scalar::words(dst, src, bit_len); + } +} diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs b/src/bit_string/impls_for_construction/impls_for_from_bool_iter.rs similarity index 90% rename from src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs rename to src/bit_string/impls_for_construction/impls_for_from_bool_iter.rs index 0a60129..bc3e0fa 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_bool_iter.rs @@ -13,7 +13,7 @@ impl BitString { let bit_len = bytes.len(); let src = bytes.as_ptr(); Self { - words: super::bools_core(src, bit_len), + words: super::funcs_for_pack_bools_core::bools_core(src, bit_len), bit_len, } } diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter/tests_for_from_bool_iter.rs b/src/bit_string/impls_for_construction/impls_for_from_bool_iter/tests_for_from_bool_iter.rs similarity index 100% rename from src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_iter/tests_for_from_bool_iter.rs rename to src/bit_string/impls_for_construction/impls_for_from_bool_iter/tests_for_from_bool_iter.rs diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_slice.rs b/src/bit_string/impls_for_construction/impls_for_from_bool_slice.rs similarity index 85% rename from src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_slice.rs rename to src/bit_string/impls_for_construction/impls_for_from_bool_slice.rs index 53aed80..b284912 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core/impls_for_from_bool_slice.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_bool_slice.rs @@ -10,7 +10,7 @@ impl From<&[bool]> for BitString { let src = values.as_ptr() as *const u8; let len = values.len(); Self { - words: super::bools_core(src, len), + words: super::funcs_for_pack_bools_core::bools_core(src, len), bit_len: len, } } @@ -25,7 +25,7 @@ impl From<[bool; N]> for BitString { // - Valid bool values are 0x00 (false) or 0x01 (true). let src = values.as_ptr() as *const u8; Self { - words: super::bools_core(src, N), + words: super::funcs_for_pack_bools_core::bools_core(src, N), bit_len: N, } } diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs index d4dd417..eb713ec 100644 --- a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs @@ -78,348 +78,19 @@ unsafe fn dispatch(dst: *mut u64, src: *const u8, bit_len: usize) -> Option<(usi } } -// --------------------------------------------------------------------------- -// Scalar backend -// --------------------------------------------------------------------------- - #[allow(unused)] -mod scalar { - /// Validate-and-pack: process 64 bytes at a time. - /// - /// # Safety - /// - /// - `src` must be valid for reads of `bit_len` u8 values. - /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. - #[inline] - pub(super) unsafe fn words( - mut dst: *mut u64, - mut src: *const u8, - mut bit_len: usize, - ) -> Option<(usize, u8)> { - let mut global_offset = 0usize; - - while bit_len >= 64 { - let mut word = 0u64; - for i in 0..64 { - // SAFETY: `i < 64` and `bit_len >= 64`. - let byte = unsafe { src.add(i).read() }; - let bit = match byte { - b'0' => 0u64, - b'1' => 1u64, - _ => return Some((global_offset + i, byte)), - }; - word |= bit << i; - } - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = word; - dst = dst.add(1); - src = src.add(64); - } - global_offset += 64; - bit_len -= 64; - } - - if bit_len > 0 { - let mut word = 0u64; - for i in 0..bit_len { - // SAFETY: `i < bit_len`. - let byte = unsafe { src.add(i).read() }; - let bit = match byte { - b'0' => 0u64, - b'1' => 1u64, - _ => return Some((global_offset + i, byte)), - }; - word |= bit << i; - } - - // SAFETY: `dst` points to the last output slot. - unsafe { - *dst = word; - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// AVX2 backend -// --------------------------------------------------------------------------- +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, - _mm256_set1_epi8, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, - _mm256_set1_epi8, - }; - - const LANES: usize = 32; - - /// AVX2 backend: validate + pack, 32 bytes / lane. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words( - mut dst: *mut u64, - mut src: *const u8, - mut bit_len: usize, - ) -> Option<(usize, u8)> { - let ones = _mm256_set1_epi8(b'1' as i8); - let zeros = _mm256_set1_epi8(b'0' as i8); - let mut global_offset = 0usize; - - while bit_len >= 64 { - // Load 2 × 32 bytes. - // SAFETY: `bit_len >= 64`, two unaligned 32-byte loads are in bounds. - let lo = unsafe { _mm256_loadu_si256(src.cast::<__m256i>()) }; - let hi = unsafe { _mm256_loadu_si256(src.add(LANES).cast::<__m256i>()) }; - - // Validate: each byte must equal b'0' or b'1'. - // cmpeq(v, b'1') → 0xFF for '1', 0x00 otherwise. - // cmpeq(v, b'0') → 0xFF for '0', 0x00 otherwise. - let lo_ones = _mm256_cmpeq_epi8(lo, ones); - let lo_zeros = _mm256_cmpeq_epi8(lo, zeros); - let hi_ones = _mm256_cmpeq_epi8(hi, ones); - let hi_zeros = _mm256_cmpeq_epi8(hi, zeros); - - let lo_valid = _mm256_or_si256(lo_ones, lo_zeros); - let hi_valid = _mm256_or_si256(hi_ones, hi_zeros); - - let lo_valid_mask = _mm256_movemask_epi8(lo_valid) as u32; - let hi_valid_mask = _mm256_movemask_epi8(hi_valid) as u32; - - if lo_valid_mask != 0xFFFF_FFFF || hi_valid_mask != 0xFFFF_FFFF { - // Fall back to scalar within this 64-byte chunk for exact - // error position. - // SAFETY: src points at this chunk, dst points at current slot. - let (i, b) = - unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); - return Some((global_offset + i, b)); - } - - // Pack: movemask extracts the MSB of cmpeq result. - // b'1' → 0xFF (MSB=1), b'0' → 0x00 (MSB=0). - let lo_bits = _mm256_movemask_epi8(lo_ones) as u32 as u64; - let hi_bits = _mm256_movemask_epi8(hi_ones) as u32 as u64; - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = lo_bits | (hi_bits << 32); - dst = dst.add(1); - src = src.add(64); - } - global_offset += 64; - bit_len -= 64; - } - - // Delegate tail to scalar. - // SAFETY: `bit_len < 64`. - unsafe { - if let Some((i, b)) = scalar::words(dst, src, bit_len) { - return Some((global_offset + i, b)); - } - } - None - } -} - -// --------------------------------------------------------------------------- -// SSE2 backend -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, - }; - - const LANES: usize = 16; - - /// SSE2 backend: validate + pack, 16 bytes / lane. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words( - mut dst: *mut u64, - mut src: *const u8, - mut bit_len: usize, - ) -> Option<(usize, u8)> { - let ones = _mm_set1_epi8(b'1' as i8); - let zeros = _mm_set1_epi8(b'0' as i8); - let mut global_offset = 0usize; - - while bit_len >= 64 { - // Load 4 × 16 bytes. - // SAFETY: `bit_len >= 64`, four unaligned 16-byte loads are in bounds. - let v0 = unsafe { _mm_loadu_si128(src.cast::<__m128i>()) }; - let v1 = unsafe { _mm_loadu_si128(src.add(LANES).cast::<__m128i>()) }; - let v2 = unsafe { _mm_loadu_si128(src.add(LANES * 2).cast::<__m128i>()) }; - let v3 = unsafe { _mm_loadu_si128(src.add(LANES * 3).cast::<__m128i>()) }; - - let ones0 = _mm_cmpeq_epi8(v0, ones); - let zeros0 = _mm_cmpeq_epi8(v0, zeros); - let ones1 = _mm_cmpeq_epi8(v1, ones); - let zeros1 = _mm_cmpeq_epi8(v1, zeros); - let ones2 = _mm_cmpeq_epi8(v2, ones); - let zeros2 = _mm_cmpeq_epi8(v2, zeros); - let ones3 = _mm_cmpeq_epi8(v3, ones); - let zeros3 = _mm_cmpeq_epi8(v3, zeros); - - let valid0 = _mm_or_si128(ones0, zeros0); - let valid1 = _mm_or_si128(ones1, zeros1); - let valid2 = _mm_or_si128(ones2, zeros2); - let valid3 = _mm_or_si128(ones3, zeros3); - - let mask0 = _mm_movemask_epi8(valid0) as u16; - let mask1 = _mm_movemask_epi8(valid1) as u16; - let mask2 = _mm_movemask_epi8(valid2) as u16; - let mask3 = _mm_movemask_epi8(valid3) as u16; - - if mask0 != 0xFFFF || mask1 != 0xFFFF || mask2 != 0xFFFF || mask3 != 0xFFFF { - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - let (i, b) = - unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); - return Some((global_offset + i, b)); - } - - let b0 = _mm_movemask_epi8(ones0) as u32 as u64; - let b1 = _mm_movemask_epi8(ones1) as u32 as u64; - let b2 = _mm_movemask_epi8(ones2) as u32 as u64; - let b3 = _mm_movemask_epi8(ones3) as u32 as u64; - - // SAFETY: `dst` points to the next output slot. - unsafe { - *dst = b0 | (b1 << 16) | (b2 << 32) | (b3 << 48); - dst = dst.add(1); - src = src.add(64); - } - global_offset += 64; - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`. - unsafe { - if let Some((i, b)) = scalar::words(dst, src, bit_len) { - return Some((global_offset + i, b)); - } - } - None - } -} - -// --------------------------------------------------------------------------- -// NEON backend (uses XOR approach: xor0 serves both validation and extraction) -// --------------------------------------------------------------------------- +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::scalar; - - use core::arch::aarch64::{ - vand_u8, vceq_u8, vdup_n_u8, veor_u8, vget_lane_u64, vld1_u8, vpaddl_u8, vpaddl_u16, - vpaddl_u32, vreinterpret_u64_u8, - }; - - /// Bit-position masks for vpaddl reduction. - const BIT_MASKS: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; - - /// NEON backend: validate + pack, 64 bytes at a time. - /// - /// Uses `(v ^ 0x30) & (v ^ 0x31)` for validation, reusing `v ^ 0x30` - /// (which holds the bit value in LSB) for extraction. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words( - mut dst: *mut u64, - mut src: *const u8, - mut bit_len: usize, - ) -> Option<(usize, u8)> { - let zero_byte = vdup_n_u8(b'0'); - let one_byte = vdup_n_u8(b'1'); - // SAFETY: `BIT_MASKS` is a static array of 8 u8 values, so its pointer is - // valid for reads of 8 bytes. - let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; - let mut global_offset = 0usize; - - while bit_len >= 64 { - // Validate all 8 groups, then pack them into one u64. - let mut word = 0u64; - for group in 0..8 { - // SAFETY: `bit_len >= 64` and `group < 8`, so the - // 8-byte read from `src + group*8` is in bounds. - let v = unsafe { vld1_u8(src.add(group * 8)) }; - - // `xor0 = v ^ 0x30`: - // b'0' → 0x00, b'1' → 0x01 (bit value in LSB) - // `xor1 = v ^ 0x31`: - // b'0' → 0x01, b'1' → 0x00 - // `invalid = xor0 & xor1`: - // valid → 0x00, invalid → non-zero - let xor0 = veor_u8(v, zero_byte); - let xor1 = veor_u8(v, one_byte); - let invalid = vand_u8(xor0, xor1); - - // Cheap all-zero test via reinterpret as u64. - let invalid_u64 = vget_lane_u64::<0>(vreinterpret_u64_u8(invalid)); - if invalid_u64 != 0 { - // Fall back to scalar for exact error position within - // this 64-byte chunk. - // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. - let (i, b) = - unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); - return Some((global_offset + i, b)); - } - - // Pack: xor0 holds the bit value (0x00 or 0x01). - // Expand 0x01 → 0xFF via vceq_u8 so that vand with - // bit-position masks correctly captures every lane, - // then horizontal pairwise add collapses to one u64. - let is_one = vceq_u8(xor0, vdup_n_u8(1)); - let masked = vand_u8(is_one, bit_masks); - let sum16 = vpaddl_u8(masked); - let sum32 = vpaddl_u16(sum16); - let sum64 = vpaddl_u32(sum32); - - let group_bits = vget_lane_u64::<0>(sum64); - word |= group_bits << (group * 8); - } - - // SAFETY: `dst` points to the current output slot. - unsafe { - *dst = word; - dst = dst.add(1); - src = src.add(64); - } - global_offset += 64; - bit_len -= 64; - } - - // SAFETY: `bit_len < 64`. - unsafe { - if let Some((i, b)) = scalar::words(dst, src, bit_len) { - return Some((global_offset + i, b)); - } - } - None - } -} - +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/avx2.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/avx2.rs new file mode 100644 index 0000000..fa44613 --- /dev/null +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/avx2.rs @@ -0,0 +1,79 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, + _mm256_set1_epi8, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, + _mm256_set1_epi8, +}; + +const LANES: usize = 32; + +/// AVX2 backend: validate + pack, 32 bytes / lane. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words( + mut dst: *mut u64, + mut src: *const u8, + mut bit_len: usize, +) -> Option<(usize, u8)> { + let ones = _mm256_set1_epi8(b'1' as i8); + let zeros = _mm256_set1_epi8(b'0' as i8); + let mut global_offset = 0usize; + + while bit_len >= 64 { + // Load 2 × 32 bytes. + // SAFETY: `bit_len >= 64`, two unaligned 32-byte loads are in bounds. + let lo = unsafe { _mm256_loadu_si256(src.cast::<__m256i>()) }; + let hi = unsafe { _mm256_loadu_si256(src.add(LANES).cast::<__m256i>()) }; + + // Validate: each byte must equal b'0' or b'1'. + // cmpeq(v, b'1') → 0xFF for '1', 0x00 otherwise. + // cmpeq(v, b'0') → 0xFF for '0', 0x00 otherwise. + let lo_ones = _mm256_cmpeq_epi8(lo, ones); + let lo_zeros = _mm256_cmpeq_epi8(lo, zeros); + let hi_ones = _mm256_cmpeq_epi8(hi, ones); + let hi_zeros = _mm256_cmpeq_epi8(hi, zeros); + + let lo_valid = _mm256_or_si256(lo_ones, lo_zeros); + let hi_valid = _mm256_or_si256(hi_ones, hi_zeros); + + let lo_valid_mask = _mm256_movemask_epi8(lo_valid) as u32; + let hi_valid_mask = _mm256_movemask_epi8(hi_valid) as u32; + + if lo_valid_mask != 0xFFFF_FFFF || hi_valid_mask != 0xFFFF_FFFF { + // Fall back to scalar within this 64-byte chunk for exact + // error position. + // SAFETY: src points at this chunk, dst points at current slot. + let (i, b) = unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); + return Some((global_offset + i, b)); + } + + // Pack: movemask extracts the MSB of cmpeq result. + // b'1' → 0xFF (MSB=1), b'0' → 0x00 (MSB=0). + let lo_bits = _mm256_movemask_epi8(lo_ones) as u32 as u64; + let hi_bits = _mm256_movemask_epi8(hi_ones) as u32 as u64; + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = lo_bits | (hi_bits << 32); + dst = dst.add(1); + src = src.add(64); + } + global_offset += 64; + bit_len -= 64; + } + + // Delegate tail to scalar. + // SAFETY: `bit_len < 64`. + unsafe { + if let Some((i, b)) = scalar::words(dst, src, bit_len) { + return Some((global_offset + i, b)); + } + } + None +} diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/neon.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/neon.rs new file mode 100644 index 0000000..04e1575 --- /dev/null +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/neon.rs @@ -0,0 +1,88 @@ +use super::scalar; + +use core::arch::aarch64::{ + vand_u8, vceq_u8, vdup_n_u8, veor_u8, vget_lane_u64, vld1_u8, vpaddl_u8, vpaddl_u16, + vpaddl_u32, vreinterpret_u64_u8, +}; + +/// Bit-position masks for vpaddl reduction. +const BIT_MASKS: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128]; + +/// NEON backend: validate + pack, 64 bytes at a time. +/// +/// Uses `(v ^ 0x30) & (v ^ 0x31)` for validation, reusing `v ^ 0x30` +/// (which holds the bit value in LSB) for extraction. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words( + mut dst: *mut u64, + mut src: *const u8, + mut bit_len: usize, +) -> Option<(usize, u8)> { + let zero_byte = vdup_n_u8(b'0'); + let one_byte = vdup_n_u8(b'1'); + // SAFETY: `BIT_MASKS` is a static array of 8 u8 values, so its pointer is + // valid for reads of 8 bytes. + let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; + let mut global_offset = 0usize; + + while bit_len >= 64 { + // Validate all 8 groups, then pack them into one u64. + let mut word = 0u64; + for group in 0..8 { + // SAFETY: `bit_len >= 64` and `group < 8`, so the + // 8-byte read from `src + group*8` is in bounds. + let v = unsafe { vld1_u8(src.add(group * 8)) }; + + // `xor0 = v ^ 0x30`: + // b'0' → 0x00, b'1' → 0x01 (bit value in LSB) + // `xor1 = v ^ 0x31`: + // b'0' → 0x01, b'1' → 0x00 + // `invalid = xor0 & xor1`: + // valid → 0x00, invalid → non-zero + let xor0 = veor_u8(v, zero_byte); + let xor1 = veor_u8(v, one_byte); + let invalid = vand_u8(xor0, xor1); + + // Cheap all-zero test via reinterpret as u64. + let invalid_u64 = vget_lane_u64::<0>(vreinterpret_u64_u8(invalid)); + if invalid_u64 != 0 { + // Fall back to scalar for exact error position within + // this 64-byte chunk. + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + let (i, b) = + unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); + return Some((global_offset + i, b)); + } + + // Pack: xor0 holds the bit value (0x00 or 0x01). + // Expand 0x01 → 0xFF via vceq_u8 so that vand with + // bit-position masks correctly captures every lane, + // then horizontal pairwise add collapses to one u64. + let is_one = vceq_u8(xor0, vdup_n_u8(1)); + let masked = vand_u8(is_one, bit_masks); + let sum16 = vpaddl_u8(masked); + let sum32 = vpaddl_u16(sum16); + let sum64 = vpaddl_u32(sum32); + + let group_bits = vget_lane_u64::<0>(sum64); + word |= group_bits << (group * 8); + } + + // SAFETY: `dst` points to the current output slot. + unsafe { + *dst = word; + dst = dst.add(1); + src = src.add(64); + } + global_offset += 64; + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`. + unsafe { + if let Some((i, b)) = scalar::words(dst, src, bit_len) { + return Some((global_offset + i, b)); + } + } + None +} diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/scalar.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/scalar.rs new file mode 100644 index 0000000..eb9d2b2 --- /dev/null +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/scalar.rs @@ -0,0 +1,58 @@ +/// Validate-and-pack: process 64 bytes at a time. +/// +/// # Safety +/// +/// - `src` must be valid for reads of `bit_len` u8 values. +/// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. +#[inline] +pub(super) unsafe fn words( + mut dst: *mut u64, + mut src: *const u8, + mut bit_len: usize, +) -> Option<(usize, u8)> { + let mut global_offset = 0usize; + + while bit_len >= 64 { + let mut word = 0u64; + for i in 0..64 { + // SAFETY: `i < 64` and `bit_len >= 64`. + let byte = unsafe { src.add(i).read() }; + let bit = match byte { + b'0' => 0u64, + b'1' => 1u64, + _ => return Some((global_offset + i, byte)), + }; + word |= bit << i; + } + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = word; + dst = dst.add(1); + src = src.add(64); + } + global_offset += 64; + bit_len -= 64; + } + + if bit_len > 0 { + let mut word = 0u64; + for i in 0..bit_len { + // SAFETY: `i < bit_len`. + let byte = unsafe { src.add(i).read() }; + let bit = match byte { + b'0' => 0u64, + b'1' => 1u64, + _ => return Some((global_offset + i, byte)), + }; + word |= bit << i; + } + + // SAFETY: `dst` points to the last output slot. + unsafe { + *dst = word; + } + } + + None +} diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/sse2.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/sse2.rs new file mode 100644 index 0000000..d947c44 --- /dev/null +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core/sse2.rs @@ -0,0 +1,81 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8, +}; + +const LANES: usize = 16; + +/// SSE2 backend: validate + pack, 16 bytes / lane. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words( + mut dst: *mut u64, + mut src: *const u8, + mut bit_len: usize, +) -> Option<(usize, u8)> { + let ones = _mm_set1_epi8(b'1' as i8); + let zeros = _mm_set1_epi8(b'0' as i8); + let mut global_offset = 0usize; + + while bit_len >= 64 { + // Load 4 × 16 bytes. + // SAFETY: `bit_len >= 64`, four unaligned 16-byte loads are in bounds. + let v0 = unsafe { _mm_loadu_si128(src.cast::<__m128i>()) }; + let v1 = unsafe { _mm_loadu_si128(src.add(LANES).cast::<__m128i>()) }; + let v2 = unsafe { _mm_loadu_si128(src.add(LANES * 2).cast::<__m128i>()) }; + let v3 = unsafe { _mm_loadu_si128(src.add(LANES * 3).cast::<__m128i>()) }; + + let ones0 = _mm_cmpeq_epi8(v0, ones); + let zeros0 = _mm_cmpeq_epi8(v0, zeros); + let ones1 = _mm_cmpeq_epi8(v1, ones); + let zeros1 = _mm_cmpeq_epi8(v1, zeros); + let ones2 = _mm_cmpeq_epi8(v2, ones); + let zeros2 = _mm_cmpeq_epi8(v2, zeros); + let ones3 = _mm_cmpeq_epi8(v3, ones); + let zeros3 = _mm_cmpeq_epi8(v3, zeros); + + let valid0 = _mm_or_si128(ones0, zeros0); + let valid1 = _mm_or_si128(ones1, zeros1); + let valid2 = _mm_or_si128(ones2, zeros2); + let valid3 = _mm_or_si128(ones3, zeros3); + + let mask0 = _mm_movemask_epi8(valid0) as u16; + let mask1 = _mm_movemask_epi8(valid1) as u16; + let mask2 = _mm_movemask_epi8(valid2) as u16; + let mask3 = _mm_movemask_epi8(valid3) as u16; + + if mask0 != 0xFFFF || mask1 != 0xFFFF || mask2 != 0xFFFF || mask3 != 0xFFFF { + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + let (i, b) = unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); + return Some((global_offset + i, b)); + } + + let b0 = _mm_movemask_epi8(ones0) as u32 as u64; + let b1 = _mm_movemask_epi8(ones1) as u32 as u64; + let b2 = _mm_movemask_epi8(ones2) as u32 as u64; + let b3 = _mm_movemask_epi8(ones3) as u32 as u64; + + // SAFETY: `dst` points to the next output slot. + unsafe { + *dst = b0 | (b1 << 16) | (b2 << 32) | (b3 << 48); + dst = dst.add(1); + src = src.add(64); + } + global_offset += 64; + bit_len -= 64; + } + + // SAFETY: `bit_len < 64`. + unsafe { + if let Some((i, b)) = scalar::words(dst, src, bit_len) { + return Some((global_offset + i, b)); + } + } + None +} diff --git a/src/bit_string/impls_for_editing/impls_for_drain.rs b/src/bit_string/impls_for_editing/impls_for_drain.rs index a8c7cf9..0c3a179 100644 --- a/src/bit_string/impls_for_editing/impls_for_drain.rs +++ b/src/bit_string/impls_for_editing/impls_for_drain.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::funcs_for_bits::*; use crate::traits::*; diff --git a/src/bit_string/impls_for_editing/impls_for_drain/tests_for_drain_interval.rs b/src/bit_string/impls_for_editing/impls_for_drain/tests_for_drain_interval.rs index 39b7272..cb88077 100644 --- a/src/bit_string/impls_for_editing/impls_for_drain/tests_for_drain_interval.rs +++ b/src/bit_string/impls_for_editing/impls_for_drain/tests_for_drain_interval.rs @@ -1,5 +1,5 @@ use alloc::string::ToString; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_string/impls_for_editing/impls_for_replace.rs b/src/bit_string/impls_for_editing/impls_for_replace.rs index cc5e184..809aba5 100644 --- a/src/bit_string/impls_for_editing/impls_for_replace.rs +++ b/src/bit_string/impls_for_editing/impls_for_replace.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use witnessed::{WitnessExt, Witnessed}; use crate::funcs_for_bits::*; diff --git a/src/bit_string/impls_for_editing/impls_for_replace/tests_for_replace_interval.rs b/src/bit_string/impls_for_editing/impls_for_replace/tests_for_replace_interval.rs index b0a138b..f77cf4c 100644 --- a/src/bit_string/impls_for_editing/impls_for_replace/tests_for_replace_interval.rs +++ b/src/bit_string/impls_for_editing/impls_for_replace/tests_for_replace_interval.rs @@ -1,7 +1,7 @@ use crate::BitString; use alloc::string::ToString; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[inline] fn iv(start: usize, end_excl: usize) -> UsizeCO { diff --git a/src/bit_string/impls_for_editing/impls_for_set.rs b/src/bit_string/impls_for_editing/impls_for_set.rs index 883c5fc..5c4df00 100644 --- a/src/bit_string/impls_for_editing/impls_for_set.rs +++ b/src/bit_string/impls_for_editing/impls_for_set.rs @@ -1,4 +1,3 @@ -use crate::funcs_for_bits::*; use crate::traits::*; use super::*; @@ -17,27 +16,22 @@ impl BitString { Some(old) } - /// Writes `len` bits of `value` starting at `bit_start`, OR-ing them - /// with the existing bits. Bits beyond `self.len()` are ignored. + /// Overwrites `len` bits starting at `bit_start` with the low bits of + /// `value`. Bits beyond `self.len()` are ignored. /// /// Only the low `len` bits of `value` are used; higher bits are /// masked out. #[inline] pub fn set_chunk(&mut self, bit_start: usize, value: u64, len: usize) { - let value = value & low_mask(len); - let word = bit_start / WORD_BITS; - let shift = bit_start % WORD_BITS; - - if let Some(w) = self.words.get_mut(word) { - *w |= value << shift; - } - - if shift != 0 { - if let Some(w) = self.words.get_mut(word + 1) { - *w |= value >> (WORD_BITS - shift); - } + let len = len + .min(WORD_BITS) + .min(self.bit_len.saturating_sub(bit_start)); + if len == 0 { + return; } + self.words.clear_bits_at(bit_start, len); + self.words.write_word_at::(bit_start, value, len); self.words.mask_unused_bits(self.bit_len); } } diff --git a/src/bit_string/impls_for_editing/impls_for_set/tests_for_set_chunk.rs b/src/bit_string/impls_for_editing/impls_for_set/tests_for_set_chunk.rs index b9bc9fd..e86d535 100644 --- a/src/bit_string/impls_for_editing/impls_for_set/tests_for_set_chunk.rs +++ b/src/bit_string/impls_for_editing/impls_for_set/tests_for_set_chunk.rs @@ -20,13 +20,25 @@ fn sets_unaligned_chunk() { } #[test] -fn set_chunk_ors_with_existing() { - let mut bits = BitString::zeros(128); - bits.set_chunk(0, 0b11, 2); - bits.set_chunk(1, 0b10, 2); +fn set_chunk_overwrites_existing_bits() { + let mut bits = BitString::ones(4); + bits.set_chunk(1, 0b01, 2); assert_eq!(bits.get(0), Some(true)); assert_eq!(bits.get(1), Some(true)); - assert_eq!(bits.get(2), Some(true)); + assert_eq!(bits.get(2), Some(false)); + assert_eq!(bits.get(3), Some(true)); +} + +#[test] +fn overwrite_crosses_a_word_boundary() { + let mut bits = BitString::ones(130); + bits.set_chunk(62, 0b001, 3); + + assert_eq!(bits.get(61), Some(true)); + assert_eq!(bits.get(62), Some(true)); + assert_eq!(bits.get(63), Some(false)); + assert_eq!(bits.get(64), Some(false)); + assert_eq!(bits.get(65), Some(true)); } #[test] diff --git a/src/bit_string/impls_for_editing/impls_for_slice.rs b/src/bit_string/impls_for_editing/impls_for_slice.rs index d474f1a..8016d06 100644 --- a/src/bit_string/impls_for_editing/impls_for_slice.rs +++ b/src/bit_string/impls_for_editing/impls_for_slice.rs @@ -1,4 +1,4 @@ -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::funcs_for_bits::*; use crate::traits::*; diff --git a/src/bit_string/impls_for_editing/impls_for_slice/tests_for_slice.rs b/src/bit_string/impls_for_editing/impls_for_slice/tests_for_slice.rs index 56a6e2f..5753cc0 100644 --- a/src/bit_string/impls_for_editing/impls_for_slice/tests_for_slice.rs +++ b/src/bit_string/impls_for_editing/impls_for_slice/tests_for_slice.rs @@ -1,5 +1,5 @@ use alloc::string::ToString; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/bit_string/impls_for_editing/tests_for_invariants.rs b/src/bit_string/impls_for_editing/tests_for_invariants.rs index c6002d8..e5965c4 100644 --- a/src/bit_string/impls_for_editing/tests_for_invariants.rs +++ b/src/bit_string/impls_for_editing/tests_for_invariants.rs @@ -5,7 +5,7 @@ use alloc::string::ToString; use crate::BitString; use crate::word_len; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; /// Verify that `self.as_words().len() == word_len(self.bit_len())`. fn assert_invariants(bits: &BitString) { diff --git a/src/bit_string/tests_for_proptest.rs b/src/bit_string/tests_for_proptest.rs index 1307891..caf567f 100644 --- a/src/bit_string/tests_for_proptest.rs +++ b/src/bit_string/tests_for_proptest.rs @@ -4,7 +4,7 @@ use proptest::collection::vec; use proptest::prelude::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; use crate::BitString; diff --git a/src/consts_for_bits.rs b/src/consts_for_bits.rs index 48af95b..eedf4e3 100644 --- a/src/consts_for_bits.rs +++ b/src/consts_for_bits.rs @@ -1,8 +1,9 @@ /// Number of bits in a `u64` — the word size for `BitString` backing storage. pub(crate) const WORD_BITS: usize = u64::BITS as usize; -/// Below this many full words, scalar loops beat SIMD dispatch overhead. -/// Must match each backend's `LANES`. +/// Below this many full words, scalar loops beat the selected SIMD dispatch +/// overhead. The value is selected per target-feature configuration; it is +/// not a universal requirement for every accelerator in the crate. #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), target_feature = "avx2" diff --git a/src/funcs_for_packed.rs b/src/funcs_for_packed.rs new file mode 100644 index 0000000..886fe03 --- /dev/null +++ b/src/funcs_for_packed.rs @@ -0,0 +1,17 @@ +#[inline] +pub(crate) const fn assert_valid_width() { + assert!( + BITS > 0 && BITS <= 8, + "packed character width must be between 1 and 8" + ); +} + +#[inline] +pub(crate) const fn code_mask() -> u8 { + assert_valid_width::(); + match BITS { + 1..=7 => (1u8 << BITS) - 1, + 8 => u8::MAX, + _ => unreachable!(), + } +} diff --git a/src/lib.rs b/src/lib.rs index e2864ca..b29060c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,16 +4,23 @@ extern crate alloc; mod consts_for_bits; mod funcs_for_bits; +mod funcs_for_packed; -pub(crate) mod traits; +pub mod traits; pub(crate) use consts_for_bits::*; pub(crate) use funcs_for_bits::*; +pub(crate) use funcs_for_packed::*; mod bit_str; mod bit_string; +mod packed_str; +mod packed_string; pub use bit_str::BitStr; pub use bit_string::BitString; +pub use bit_string_derive::packed; +pub use packed_str::PackedStr; +pub use packed_string::PackedString; pub use bit_str::errors::*; pub use bit_string::errors::*; diff --git a/src/packed_str.rs b/src/packed_str.rs new file mode 100644 index 0000000..507a3af --- /dev/null +++ b/src/packed_str.rs @@ -0,0 +1,23 @@ +//! A zero-copy borrowed view into a [`PackedString`](crate::PackedString). + +use core::marker::PhantomData; + +use crate::{BitStr, PackedString, traits::PackedChar}; + +/// A character-aligned borrowed view into a [`PackedString`]. +#[derive(Clone, Copy)] +pub struct PackedStr<'ps, C, const BITS: u8> +where + C: PackedChar, +{ + bits: BitStr<'ps>, + marker: PhantomData C>, +} + +mod impls_for_access; +mod impls_for_conversion; +mod impls_for_eq; +mod impls_for_iter; +mod impls_for_matching; +mod impls_for_ord; +mod impls_for_slice; diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs new file mode 100644 index 0000000..613e80a --- /dev/null +++ b/src/packed_str/impls_for_access.rs @@ -0,0 +1,37 @@ +use super::*; + +impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ + pub fn char_len(&self) -> usize { + self.bits.bit_len() / usize::from(BITS) + } + + pub fn is_empty(&self) -> bool { + self.bits.bit_len() == 0 + } + + pub fn get(&self, index: usize) -> Option { + if index >= self.char_len() { + return None; + } + Some( + C::from_code( + (self.bits.get_chunk(index * usize::from(BITS)) + & u64::from(crate::code_mask::())) as u8, + ) + .expect("PackedChar rejected a code in a PackedStr invariant"), + ) + } + + pub fn first(&self) -> Option { + self.get(0) + } + + pub fn last(&self) -> Option { + self.char_len() + .checked_sub(1) + .and_then(|index| self.get(index)) + } +} diff --git a/src/packed_str/impls_for_conversion.rs b/src/packed_str/impls_for_conversion.rs new file mode 100644 index 0000000..5e5fc38 --- /dev/null +++ b/src/packed_str/impls_for_conversion.rs @@ -0,0 +1,31 @@ +use super::*; + +impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ + pub fn to_packed_string(&self) -> PackedString { + PackedString::from_valid_bits(self.bits.to_bit_string()) + } + + #[inline] + pub(crate) fn from_valid_bit_str(bits: BitStr<'ps>) -> Self { + let width = usize::from(BITS); + crate::assert_valid_width::(); + debug_assert_eq!(bits.start() % width, 0); + debug_assert_eq!(bits.bit_len() % width, 0); + Self { + bits, + marker: core::marker::PhantomData, + } + } +} + +impl PackedString +where + C: PackedChar, +{ + pub fn as_packed_str(&self) -> PackedStr<'_, C, BITS> { + PackedStr::from_valid_bit_str(self.bits().as_bit_str()) + } +} diff --git a/src/packed_str/impls_for_eq.rs b/src/packed_str/impls_for_eq.rs new file mode 100644 index 0000000..12ded5f --- /dev/null +++ b/src/packed_str/impls_for_eq.rs @@ -0,0 +1,12 @@ +use super::*; + +impl PartialEq for PackedStr<'_, C, BITS> +where + C: PackedChar, +{ + fn eq(&self, other: &Self) -> bool { + self.bits == other.bits + } +} + +impl Eq for PackedStr<'_, C, BITS> where C: PackedChar {} diff --git a/src/packed_str/impls_for_iter.rs b/src/packed_str/impls_for_iter.rs new file mode 100644 index 0000000..8eb20d6 --- /dev/null +++ b/src/packed_str/impls_for_iter.rs @@ -0,0 +1,77 @@ +use core::iter::FusedIterator; + +use super::*; + +impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ + pub fn iter(&self) -> Iter<'ps, C, BITS> { + Iter { + view: *self, + front: 0, + back: self.char_len(), + } + } +} + +#[allow(dead_code)] +pub struct Iter<'ps, C, const BITS: u8> +where + C: PackedChar, +{ + view: PackedStr<'ps, C, BITS>, + front: usize, + back: usize, +} + +impl Iterator for Iter<'_, C, BITS> +where + C: PackedChar, +{ + type Item = C; + + fn next(&mut self) -> Option { + if self.front == self.back { + None + } else { + let c = self.view.get(self.front); + self.front += 1; + c + } + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.back - self.front; + (len, Some(len)) + } +} + +impl DoubleEndedIterator for Iter<'_, C, BITS> +where + C: PackedChar, +{ + fn next_back(&mut self) -> Option { + if self.front == self.back { + None + } else { + self.back -= 1; + self.view.get(self.back) + } + } +} + +impl ExactSizeIterator for Iter<'_, C, BITS> where C: PackedChar {} +impl FusedIterator for Iter<'_, C, BITS> where C: PackedChar {} + +impl<'ps, C, const BITS: u8> IntoIterator for &'ps PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ + type Item = C; + type IntoIter = Iter<'ps, C, BITS>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} diff --git a/src/packed_str/impls_for_matching.rs b/src/packed_str/impls_for_matching.rs new file mode 100644 index 0000000..496c735 --- /dev/null +++ b/src/packed_str/impls_for_matching.rs @@ -0,0 +1,52 @@ +use super::*; + +impl PackedStr<'_, C, BITS> +where + C: PackedChar, +{ + pub fn matches_at(&self, index: usize, pattern: Self) -> bool { + let Some(bit_index) = index.checked_mul(usize::from(BITS)) else { + return false; + }; + if index > self.char_len() { + return false; + } + self.bits.matches_at_str(bit_index, pattern.bits) + } + + pub fn starts_with(&self, prefix: Self) -> bool { + self.bits.starts_with_str(prefix.bits) + } + + pub fn ends_with(&self, suffix: Self) -> bool { + self.bits.ends_with_str(suffix.bits) + } + + pub fn contains(&self, needle: Self) -> bool { + (0..=self.char_len().saturating_sub(needle.char_len())) + .any(|index| self.matches_at(index, needle)) + } + + pub fn find(&self, needle: Self) -> Option { + (0..=self.char_len().saturating_sub(needle.char_len())) + .find(|&index| self.matches_at(index, needle)) + } + + pub fn rfind(&self, needle: Self) -> Option { + (0..=self.char_len().saturating_sub(needle.char_len())) + .rev() + .find(|&index| self.matches_at(index, needle)) + } + + pub fn strip_prefix(&self, prefix: Self) -> Option { + self.bits + .strip_prefix_str(prefix.bits) + .map(Self::from_valid_bit_str) + } + + pub fn strip_suffix(&self, suffix: Self) -> Option { + self.bits + .strip_suffix_str(suffix.bits) + .map(Self::from_valid_bit_str) + } +} diff --git a/src/packed_str/impls_for_ord.rs b/src/packed_str/impls_for_ord.rs new file mode 100644 index 0000000..b14809a --- /dev/null +++ b/src/packed_str/impls_for_ord.rs @@ -0,0 +1,41 @@ +use core::cmp::Ordering; + +use super::*; + +impl PackedStr<'_, C, BITS> +where + C: PackedChar, +{ + pub(crate) fn cmp_codes(&self, other: &Self) -> Ordering { + let mask = u64::from(crate::code_mask::()); + let shared_len = self.char_len().min(other.char_len()); + for index in 0..shared_len { + let start = index * usize::from(BITS); + let lhs = self.bits.get_chunk(start) & mask; + let rhs = other.bits.get_chunk(start) & mask; + match lhs.cmp(&rhs) { + Ordering::Equal => {} + ordering => return ordering, + } + } + self.char_len().cmp(&other.char_len()) + } +} + +impl PartialOrd for PackedStr<'_, C, BITS> +where + C: PackedChar, +{ + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp_codes(other)) + } +} + +impl Ord for PackedStr<'_, C, BITS> +where + C: PackedChar, +{ + fn cmp(&self, other: &Self) -> Ordering { + self.cmp_codes(other) + } +} diff --git a/src/packed_str/impls_for_slice.rs b/src/packed_str/impls_for_slice.rs new file mode 100644 index 0000000..25778d9 --- /dev/null +++ b/src/packed_str/impls_for_slice.rs @@ -0,0 +1,36 @@ +use int_intervals::UsizeCO; + +use super::*; + +impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> +where + C: PackedChar, +{ + pub fn slice(&self, interval: UsizeCO) -> Self { + let bits = usize::from(BITS); + let start = interval.start().min(self.char_len()).saturating_mul(bits); + let end = interval + .end_excl() + .min(self.char_len()) + .saturating_mul(bits); + if end <= start { + return Self::from_valid_bit_str(self.bits.slice_from(start)); + } + let interval = UsizeCO::checked_from_start_len(start, end - start).unwrap(); + Self::from_valid_bit_str(self.bits.slice(interval)) + } + + pub fn slice_from(&self, start: usize) -> Self { + if start >= self.char_len() { + return Self::from_valid_bit_str(self.bits.slice_from(self.bits.bit_len())); + } + self.slice(UsizeCO::checked_from_start_len(start, self.char_len() - start).unwrap()) + } + + pub fn slice_until(&self, end: usize) -> Self { + if end == 0 { + return Self::from_valid_bit_str(self.bits.slice_until(0)); + } + self.slice(UsizeCO::checked_from_start_len(0, end).unwrap()) + } +} diff --git a/src/packed_string.rs b/src/packed_string.rs new file mode 100644 index 0000000..9e4b78e --- /dev/null +++ b/src/packed_string.rs @@ -0,0 +1,36 @@ +//! A packed sequence whose character type defines its own bit encoding. + +use core::marker::PhantomData; + +use crate::{BitString, traits::PackedChar}; + +/// An owned sequence of [`PackedChar`] values stored in exactly `BITS` bits +/// per character. +/// +/// The character type is the alphabet and its enum discriminant is the stored +/// value, so no alphabet allocation or lookup table is required. +#[derive(Clone)] +pub struct PackedString +where + C: PackedChar, +{ + bits: BitString, + marker: PhantomData C>, +} + +mod funcs_for_code; +mod impls_for_access; +mod impls_for_construction; +mod impls_for_editing; +mod impls_for_eq; +mod impls_for_fmt; +mod impls_for_hash; +mod impls_for_iter; +mod impls_for_matching; +mod impls_for_ord; + +use crate::{assert_valid_width, code_mask}; +use funcs_for_code::write_code; + +#[cfg(test)] +mod tests_for_support; diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs new file mode 100644 index 0000000..04d47bb --- /dev/null +++ b/src/packed_string/funcs_for_code.rs @@ -0,0 +1,7 @@ +use super::*; + +#[inline] +pub(super) fn write_code(bits: &mut BitString, position: usize, code: u8) { + let start = position * usize::from(BITS); + bits.set_chunk(start, u64::from(code), usize::from(BITS)); +} diff --git a/src/packed_string/impls_for_access.rs b/src/packed_string/impls_for_access.rs new file mode 100644 index 0000000..3ee54d3 --- /dev/null +++ b/src/packed_string/impls_for_access.rs @@ -0,0 +1,57 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + /// Number of packed characters, not number of bits. + #[inline] + pub fn char_len(&self) -> usize { + self.bits.bit_len() / usize::from(BITS) + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.bits.is_empty() + } + + #[inline] + pub const fn bits_per_char(&self) -> usize { + BITS as usize + } + + #[inline] + pub fn bits(&self) -> &BitString { + &self.bits + } + + #[inline] + pub fn get(&self, index: usize) -> Option { + if index >= self.char_len() { + return None; + } + Some( + C::from_code(self.code_at(index)) + .expect("PackedChar rejected a code it previously produced"), + ) + } + + #[inline] + pub fn first(&self) -> Option { + self.get(0) + } + + #[inline] + pub fn last(&self) -> Option { + self.char_len().checked_sub(1).and_then(|i| self.get(i)) + } + + #[inline] + fn code_at(&self, index: usize) -> u8 { + let start = index * usize::from(BITS); + (self.bits.get_chunk(start) & u64::from(code_mask::())) as u8 + } +} + +#[cfg(test)] +mod tests_for_access; diff --git a/src/packed_string/impls_for_access/tests_for_access.rs b/src/packed_string/impls_for_access/tests_for_access.rs new file mode 100644 index 0000000..be4b598 --- /dev/null +++ b/src/packed_string/impls_for_access/tests_for_access.rs @@ -0,0 +1,17 @@ +use crate::packed_string::tests_for_support::{Letter, LetterString, Only, OnlyString}; + +#[test] +fn get_first_and_last_decode_values() { + let value = LetterString::from_chars([Letter::A, Letter::B, Letter::C]); + assert_eq!(value.first(), Some(Letter::A)); + assert_eq!(value.last(), Some(Letter::C)); + assert_eq!(value.get(3), None); +} + +#[test] +fn one_value_type_still_has_one_bit_per_character() { + let value = OnlyString::from_chars([Only::Value; 100]); + assert_eq!(value.char_len(), 100); + assert_eq!(value.bits().bit_len(), 100); + assert_eq!(value.get(99), Some(Only::Value)); +} diff --git a/src/packed_string/impls_for_construction.rs b/src/packed_string/impls_for_construction.rs new file mode 100644 index 0000000..df209f8 --- /dev/null +++ b/src/packed_string/impls_for_construction.rs @@ -0,0 +1,50 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + #[inline] + pub fn new() -> Self { + assert_valid_width::(); + Self { + bits: BitString::new(), + marker: PhantomData, + } + } + + /// Packs enum values directly, without an alphabet lookup. + pub fn from_chars(chars: I) -> Self + where + I: IntoIterator, + { + let mut result = Self::new(); + result.extend(chars); + result + } +} + +impl Default for PackedString +where + C: PackedChar, +{ + fn default() -> Self { + Self::new() + } +} + +impl FromIterator for PackedString +where + C: PackedChar, +{ + fn from_iter>(iter: I) -> Self { + Self::from_chars(iter) + } +} + +#[cfg(test)] +mod tests_for_construction; + +mod impls_for_from_bits; +mod impls_for_from_slice; +mod impls_for_repeat; diff --git a/src/packed_string/impls_for_construction/impls_for_from_bits.rs b/src/packed_string/impls_for_construction/impls_for_from_bits.rs new file mode 100644 index 0000000..8b030d3 --- /dev/null +++ b/src/packed_string/impls_for_construction/impls_for_from_bits.rs @@ -0,0 +1,36 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + /// Validates and adopts an already packed bit payload. + pub fn from_bits(bits: BitString) -> Option { + assert_valid_width::(); + let bits_per_char = usize::from(BITS); + if bits.bit_len() % bits_per_char != 0 { + return None; + } + let code_mask = u64::from(code_mask::()); + for start in (0..bits.bit_len()).step_by(bits_per_char) { + C::from_code((bits.get_chunk(start) & code_mask) as u8)?; + } + Some(Self::from_valid_bits(bits)) + } + + /// Adopts a bit payload whose packed-character invariant is already known. + #[inline] + pub(crate) fn from_valid_bits(bits: BitString) -> Self { + let width = usize::from(BITS); + assert_valid_width::(); + debug_assert_eq!(bits.bit_len() % width, 0); + Self { + bits, + marker: PhantomData, + } + } + + pub fn into_bits(self) -> BitString { + self.bits + } +} diff --git a/src/packed_string/impls_for_construction/impls_for_from_slice.rs b/src/packed_string/impls_for_construction/impls_for_from_slice.rs new file mode 100644 index 0000000..ceaa0ae --- /dev/null +++ b/src/packed_string/impls_for_construction/impls_for_from_slice.rs @@ -0,0 +1,19 @@ +use super::*; + +impl From<&[C]> for PackedString +where + C: PackedChar, +{ + fn from(characters: &[C]) -> Self { + Self::from_chars(characters.iter().copied()) + } +} + +impl From<[C; N]> for PackedString +where + C: PackedChar, +{ + fn from(characters: [C; N]) -> Self { + Self::from_chars(characters) + } +} diff --git a/src/packed_string/impls_for_construction/impls_for_repeat.rs b/src/packed_string/impls_for_construction/impls_for_repeat.rs new file mode 100644 index 0000000..4ba0b06 --- /dev/null +++ b/src/packed_string/impls_for_construction/impls_for_repeat.rs @@ -0,0 +1,12 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn repeat(character: C, char_len: usize) -> Self { + let mut result = Self::new(); + result.extend(core::iter::repeat_n(character, char_len)); + result + } +} diff --git a/src/packed_string/impls_for_construction/tests_for_construction.rs b/src/packed_string/impls_for_construction/tests_for_construction.rs new file mode 100644 index 0000000..cbe7a9d --- /dev/null +++ b/src/packed_string/impls_for_construction/tests_for_construction.rs @@ -0,0 +1,16 @@ +use crate::packed_string::tests_for_support::{Letter, LetterString}; + +#[test] +fn enum_discriminants_are_stored_directly() { + let value = LetterString::from_chars([Letter::A, Letter::B, Letter::C, Letter::D]); + assert_eq!(value.bits_per_char(), 2); + assert_eq!(value.bits().bit_len(), 8); + assert_eq!(value.bits().get_chunk(0), 0b11_10_01_00); +} + +#[test] +fn collect_constructs_a_packed_string() { + let value: LetterString = [Letter::A, Letter::C].into_iter().collect(); + assert_eq!(value.char_len(), 2); + assert_eq!(value.get(0), Some(Letter::A)); +} diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs new file mode 100644 index 0000000..000f4f1 --- /dev/null +++ b/src/packed_string/impls_for_editing.rs @@ -0,0 +1,73 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + /// Appends the character's code directly to the payload. + pub fn push(&mut self, character: C) { + let code = character.code(); + self.bits + .bit_len() + .checked_add(usize::from(BITS)) + .expect("packed string length overflow"); + for shift in 0..BITS { + self.bits.push((code >> shift) & 1 != 0); + } + } + + pub fn pop(&mut self) -> Option { + let character = self.last()?; + self.bits + .truncate((self.char_len() - 1) * usize::from(BITS)); + Some(character) + } + + pub fn set(&mut self, index: usize, character: C) -> Option { + let previous = self.get(index)?; + write_code::(&mut self.bits, index, character.code()); + Some(previous) + } + + pub fn truncate(&mut self, new_len: usize) { + if new_len >= self.char_len() { + return; + } + self.bits.truncate(new_len * usize::from(BITS)); + } + + pub fn clear(&mut self) { + self.bits.clear(); + } +} + +impl Extend for PackedString +where + C: PackedChar, +{ + fn extend>(&mut self, iter: I) { + for character in iter { + self.push(character); + } + } +} + +impl<'a, C, const BITS: u8> Extend<&'a C> for PackedString +where + C: PackedChar, +{ + fn extend>(&mut self, iter: I) { + self.extend(iter.into_iter().copied()); + } +} + +#[cfg(test)] +mod tests_for_editing; + +mod impls_for_concat; +mod impls_for_drain; +mod impls_for_insert_remove; +mod impls_for_replace; +mod impls_for_retain; +mod impls_for_reverse; +mod impls_for_slice; diff --git a/src/packed_string/impls_for_editing/impls_for_concat.rs b/src/packed_string/impls_for_editing/impls_for_concat.rs new file mode 100644 index 0000000..b65df9a --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_concat.rs @@ -0,0 +1,20 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn push_packed_string(&mut self, other: &Self) { + self.bits.push_bit_string(&other.bits); + } + + pub fn insert_packed_string(&mut self, index: usize, other: &Self) { + let start = index.min(self.char_len()) * usize::from(BITS); + self.bits.insert_bit_string(start, &other.bits); + } + + pub fn split_off(&mut self, at: usize) -> Self { + let at = at.min(self.char_len()) * usize::from(BITS); + Self::from_valid_bits(self.bits.split_off(at)) + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_drain.rs b/src/packed_string/impls_for_editing/impls_for_drain.rs new file mode 100644 index 0000000..e278871 --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_drain.rs @@ -0,0 +1,30 @@ +use int_intervals::UsizeCO; + +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn drain_interval(&self, interval: UsizeCO) -> Self { + let bits = usize::from(BITS); + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + if start == end { + return self.clone(); + } + let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; + Self::from_valid_bits(self.bits.drain_interval(interval)) + } + + pub fn drain_interval_assign(&mut self, interval: UsizeCO) { + let bits = usize::from(BITS); + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + if start == end { + return; + } + let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; + self.bits.drain_interval_assign(interval); + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs new file mode 100644 index 0000000..03f4f02 --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs @@ -0,0 +1,32 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn insert(&mut self, index: usize, character: C) { + let index = index.min(self.char_len()); + self.bits + .bit_len() + .checked_add(usize::from(BITS)) + .expect("packed string length overflow"); + let start = index * usize::from(BITS); + for shift in (0..BITS).rev() { + self.bits + .insert(start, (character.code() >> shift) & 1 != 0); + } + } + + pub fn remove(&mut self, index: usize) -> C { + let character = self.get(index).expect("packed string index out of bounds"); + let start = index * usize::from(BITS); + let mut code = 0; + for shift in 0..BITS { + if self.bits.remove(start) { + code |= 1 << shift; + } + } + debug_assert_eq!(code, character.code()); + character + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_replace.rs b/src/packed_string/impls_for_editing/impls_for_replace.rs new file mode 100644 index 0000000..d279a15 --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_replace.rs @@ -0,0 +1,58 @@ +use int_intervals::UsizeCO; + +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn replace_interval(&self, interval: UsizeCO, replacement: &Self) -> Self { + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + self.replace_range(start, end, replacement) + } + + pub fn replace_interval_assign(&mut self, interval: UsizeCO, replacement: &Self) { + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + self.replace_range_assign(start, end, replacement); + } + + pub fn replace(&self, start: usize, replacement: &Self) -> Self { + let start = start.min(self.char_len()); + let end = start + .saturating_add(replacement.char_len()) + .min(self.char_len()); + self.replace_range(start, end, replacement) + } + + pub fn replace_assign(&mut self, start: usize, replacement: &Self) { + let start = start.min(self.char_len()); + let end = start + .saturating_add(replacement.char_len()) + .min(self.char_len()); + self.replace_range_assign(start, end, replacement); + } + + fn replace_range(&self, start: usize, end: usize, replacement: &Self) -> Self { + let bits = usize::from(BITS); + let bits = if end > start { + let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; + self.bits.replace_interval(interval, &replacement.bits) + } else { + self.bits.replace(start * bits, &replacement.bits) + }; + Self::from_valid_bits(bits) + } + + fn replace_range_assign(&mut self, start: usize, end: usize, replacement: &Self) { + let bits = usize::from(BITS); + if end > start { + let interval = unsafe { UsizeCO::new_unchecked(start * bits, end * bits) }; + self.bits + .replace_interval_assign(interval, &replacement.bits); + } else { + self.bits.replace_assign(start * bits, &replacement.bits); + } + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_retain.rs b/src/packed_string/impls_for_editing/impls_for_retain.rs new file mode 100644 index 0000000..fee913f --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_retain.rs @@ -0,0 +1,14 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn retain(&mut self, mut predicate: F) + where + F: FnMut(C) -> bool, + { + let retained = self.iter().filter(|&character| predicate(character)); + *self = Self::from_chars(retained); + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_reverse.rs b/src/packed_string/impls_for_editing/impls_for_reverse.rs new file mode 100644 index 0000000..2a6743d --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_reverse.rs @@ -0,0 +1,14 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn reverse(&self) -> Self { + Self::from_chars(self.iter().rev()) + } + + pub fn reverse_assign(&mut self) { + *self = self.reverse(); + } +} diff --git a/src/packed_string/impls_for_editing/impls_for_slice.rs b/src/packed_string/impls_for_editing/impls_for_slice.rs new file mode 100644 index 0000000..e9c1cff --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_slice.rs @@ -0,0 +1,38 @@ +use int_intervals::UsizeCO; + +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn slice(&self, interval: UsizeCO) -> Self { + let start = interval.start().min(self.char_len()); + let end = interval.end_excl().min(self.char_len()).max(start); + if start == end { + return Self::new(); + } + let bits = usize::from(BITS); + Self::from_valid_bits( + self.bits.slice( + UsizeCO::checked_from_start_len(start * bits, (end - start) * bits).unwrap(), + ), + ) + } + + pub fn slice_from(&self, start: usize) -> Self { + if start >= self.char_len() { + return Self::new(); + } + self.slice( + UsizeCO::checked_from_start_len(start, self.char_len().saturating_sub(start)).unwrap(), + ) + } + + pub fn slice_until(&self, end: usize) -> Self { + if end == 0 { + return Self::new(); + } + self.slice(UsizeCO::checked_from_start_len(0, end).unwrap()) + } +} diff --git a/src/packed_string/impls_for_editing/tests_for_editing.rs b/src/packed_string/impls_for_editing/tests_for_editing.rs new file mode 100644 index 0000000..06fad30 --- /dev/null +++ b/src/packed_string/impls_for_editing/tests_for_editing.rs @@ -0,0 +1,16 @@ +use crate::packed_string::tests_for_support::{Letter, LetterString}; + +#[test] +fn push_set_pop_and_truncate_preserve_values() { + let mut value = LetterString::from_chars([Letter::A, Letter::B, Letter::C]); + value.push(Letter::D); + assert_eq!(value.set(1, Letter::C), Some(Letter::B)); + assert_eq!(value.pop(), Some(Letter::D)); + value.truncate(2); + assert_eq!( + value.iter().collect::>(), + [Letter::A, Letter::C] + ); + value.clear(); + assert!(value.is_empty()); +} diff --git a/src/packed_string/impls_for_eq.rs b/src/packed_string/impls_for_eq.rs new file mode 100644 index 0000000..f13d84f --- /dev/null +++ b/src/packed_string/impls_for_eq.rs @@ -0,0 +1,15 @@ +use super::*; + +impl PartialEq for PackedString +where + C: PackedChar, +{ + fn eq(&self, other: &Self) -> bool { + self.bits == other.bits + } +} + +impl Eq for PackedString where C: PackedChar {} + +#[cfg(test)] +mod tests_for_eq; diff --git a/src/packed_string/impls_for_eq/tests_for_eq.rs b/src/packed_string/impls_for_eq/tests_for_eq.rs new file mode 100644 index 0000000..461176e --- /dev/null +++ b/src/packed_string/impls_for_eq/tests_for_eq.rs @@ -0,0 +1,10 @@ +use crate::packed_string::tests_for_support::{Letter, LetterString}; + +#[test] +fn equality_uses_length_and_packed_values() { + let left = LetterString::from_chars([Letter::A, Letter::B]); + let same = LetterString::from_chars([Letter::A, Letter::B]); + let different = LetterString::from_chars([Letter::A, Letter::C]); + assert_eq!(left, same); + assert_ne!(left, different); +} diff --git a/src/packed_string/impls_for_fmt.rs b/src/packed_string/impls_for_fmt.rs new file mode 100644 index 0000000..7b80e89 --- /dev/null +++ b/src/packed_string/impls_for_fmt.rs @@ -0,0 +1,27 @@ +use core::fmt; + +use super::*; + +impl fmt::Debug for PackedString +where + C: PackedChar + fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl fmt::Display for PackedString +where + C: PackedChar + fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for character in self.iter() { + fmt::Display::fmt(&character, f)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests_for_fmt; diff --git a/src/packed_string/impls_for_fmt/tests_for_fmt.rs b/src/packed_string/impls_for_fmt/tests_for_fmt.rs new file mode 100644 index 0000000..a9626f8 --- /dev/null +++ b/src/packed_string/impls_for_fmt/tests_for_fmt.rs @@ -0,0 +1,14 @@ +use crate::packed_string::tests_for_support::{Letter, LetterString}; +use alloc::string::ToString; + +#[test] +fn display_uses_the_character_display_implementation() { + let value = LetterString::from_chars([Letter::A, Letter::B, Letter::C, Letter::D]); + assert_eq!(value.to_string(), "ABCD"); +} + +#[test] +fn debug_lists_the_enum_values() { + let value = LetterString::from_chars([Letter::A, Letter::B]); + assert_eq!(alloc::format!("{value:?}"), "[A, B]"); +} diff --git a/src/packed_string/impls_for_hash.rs b/src/packed_string/impls_for_hash.rs new file mode 100644 index 0000000..315776a --- /dev/null +++ b/src/packed_string/impls_for_hash.rs @@ -0,0 +1,12 @@ +use core::hash::{Hash, Hasher}; + +use super::*; + +impl Hash for PackedString +where + C: PackedChar, +{ + fn hash(&self, state: &mut H) { + self.bits.hash(state) + } +} diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs new file mode 100644 index 0000000..48273e3 --- /dev/null +++ b/src/packed_string/impls_for_iter.rs @@ -0,0 +1,84 @@ +use core::iter::FusedIterator; + +use super::*; + +impl PackedString +where + C: PackedChar, +{ + #[inline] + pub fn iter(&self) -> Iter<'_, C, BITS> { + Iter { + string: self, + front: 0, + back: self.char_len(), + } + } + + /// Collects the decoded characters into a vector. + pub fn to_vec(&self) -> alloc::vec::Vec { + self.iter().collect() + } +} + +#[derive(Clone)] +pub struct Iter<'a, C, const BITS: u8> +where + C: PackedChar, +{ + string: &'a PackedString, + front: usize, + back: usize, +} + +impl Iterator for Iter<'_, C, BITS> +where + C: PackedChar, +{ + type Item = C; + + fn next(&mut self) -> Option { + if self.front == self.back { + return None; + } + let character = self.string.get(self.front); + self.front += 1; + character + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.back - self.front; + (len, Some(len)) + } +} + +impl DoubleEndedIterator for Iter<'_, C, BITS> +where + C: PackedChar, +{ + fn next_back(&mut self) -> Option { + if self.front == self.back { + return None; + } + self.back -= 1; + self.string.get(self.back) + } +} + +impl ExactSizeIterator for Iter<'_, C, BITS> where C: PackedChar {} +impl FusedIterator for Iter<'_, C, BITS> where C: PackedChar {} + +impl<'a, C, const BITS: u8> IntoIterator for &'a PackedString +where + C: PackedChar, +{ + type Item = C; + type IntoIter = Iter<'a, C, BITS>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +#[cfg(test)] +mod tests_for_iter; diff --git a/src/packed_string/impls_for_iter/tests_for_iter.rs b/src/packed_string/impls_for_iter/tests_for_iter.rs new file mode 100644 index 0000000..22f755b --- /dev/null +++ b/src/packed_string/impls_for_iter/tests_for_iter.rs @@ -0,0 +1,12 @@ +use crate::packed_string::tests_for_support::{Letter, LetterString}; + +#[test] +fn iter_is_double_ended_and_exact_size() { + let value = LetterString::from_chars([Letter::A, Letter::B, Letter::C]); + let mut iter = value.iter(); + assert_eq!(iter.len(), 3); + assert_eq!(iter.next(), Some(Letter::A)); + assert_eq!(iter.next_back(), Some(Letter::C)); + assert_eq!(iter.next(), Some(Letter::B)); + assert_eq!(iter.next(), None); +} diff --git a/src/packed_string/impls_for_matching.rs b/src/packed_string/impls_for_matching.rs new file mode 100644 index 0000000..a6fa37a --- /dev/null +++ b/src/packed_string/impls_for_matching.rs @@ -0,0 +1,52 @@ +use super::*; + +impl PackedString +where + C: PackedChar, +{ + pub fn matches_at(&self, index: usize, pattern: &Self) -> bool { + let Some(bit_index) = index.checked_mul(usize::from(BITS)) else { + return false; + }; + if index > self.char_len() { + return false; + } + self.bits.matches_at_string(bit_index, &pattern.bits) + } + + pub fn starts_with(&self, prefix: &Self) -> bool { + self.bits.starts_with_string(&prefix.bits) + } + + pub fn ends_with(&self, suffix: &Self) -> bool { + self.bits.ends_with_string(&suffix.bits) + } + + pub fn contains(&self, needle: &Self) -> bool { + (0..=self.char_len().saturating_sub(needle.char_len())) + .any(|index| self.matches_at(index, needle)) + } + + pub fn find(&self, needle: &Self) -> Option { + (0..=self.char_len().saturating_sub(needle.char_len())) + .find(|&index| self.matches_at(index, needle)) + } + + pub fn rfind(&self, needle: &Self) -> Option { + (0..=self.char_len().saturating_sub(needle.char_len())) + .rev() + .find(|&index| self.matches_at(index, needle)) + } + + pub fn strip_prefix(&self, prefix: &Self) -> Option { + self.bits + .strip_prefix_string(&prefix.bits) + .map(Self::from_valid_bits) + } + + pub fn strip_suffix(&self, suffix: &Self) -> Option { + self.bits + .strip_suffix_string(&suffix.bits) + .map(Self::from_valid_bits) + } +} diff --git a/src/packed_string/impls_for_ord.rs b/src/packed_string/impls_for_ord.rs new file mode 100644 index 0000000..8e4b596 --- /dev/null +++ b/src/packed_string/impls_for_ord.rs @@ -0,0 +1,31 @@ +use core::cmp::Ordering; + +use super::*; + +impl PackedString +where + C: PackedChar, +{ + /// Compares characters lexicographically by their packed code values. + pub fn cmp_string(&self, other: &Self) -> Ordering { + self.as_packed_str().cmp_codes(&other.as_packed_str()) + } +} + +impl PartialOrd for PackedString +where + C: PackedChar, +{ + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp_string(other)) + } +} + +impl Ord for PackedString +where + C: PackedChar, +{ + fn cmp(&self, other: &Self) -> Ordering { + self.cmp_string(other) + } +} diff --git a/src/packed_string/tests_for_support.rs b/src/packed_string/tests_for_support.rs new file mode 100644 index 0000000..855145f --- /dev/null +++ b/src/packed_string/tests_for_support.rs @@ -0,0 +1,32 @@ +use core::fmt; + +use crate::packed; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[packed(bits = 2)] +pub(super) enum Letter { + A = 0b00, + B = 0b01, + C = 0b10, + D = 0b11, +} + +impl fmt::Display for Letter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::A => "A", + Self::B => "B", + Self::C => "C", + Self::D => "D", + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[packed(bits = 1)] +pub(super) enum Only { + Value = 0, +} + +pub(super) type LetterString = super::PackedString; +pub(super) type OnlyString = super::PackedString; diff --git a/src/traits.rs b/src/traits.rs index 8723483..c682bf3 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,3 +1,4 @@ +mod packed_char; pub(crate) mod word_ord; pub(crate) mod words_arith; pub(crate) mod words_edit; @@ -6,6 +7,7 @@ pub(crate) mod words_find; pub(crate) mod words_ord; pub(crate) mod words_scan; +pub use packed_char::PackedChar; pub(crate) use word_ord::*; pub(crate) use words_arith::*; pub(crate) use words_edit::*; diff --git a/src/traits/packed_char.rs b/src/traits/packed_char.rs new file mode 100644 index 0000000..e238f3c --- /dev/null +++ b/src/traits/packed_char.rs @@ -0,0 +1,21 @@ +/// A character with a fixed-width bit representation. +/// +/// This trait is intended for fieldless `#[repr(u8)]` enums. Use the +/// `#[packed(bits = N)]` attribute for that common case; manual +/// implementations must satisfy all of the following invariants: +/// +/// - `1 <= BITS <= 8`; +/// - [`code`](Self::code) fits in `BITS` bits; +/// - `from_code(code(value)) == Some(value)`; +/// - each valid code identifies at most one value. +/// +/// [`PackedString`](crate::PackedString) relies on these invariants. The packed +/// attribute validates them at compile time and generates both directions from +/// the same discriminants. No `transmute` is used. +pub trait PackedChar: Copy + Eq { + /// Returns the character's packed value. + fn code(self) -> u8; + + /// Decodes a packed value, rejecting unused bit patterns. + fn from_code(code: u8) -> Option; +} diff --git a/src/traits/word_ord.rs b/src/traits/word_ord.rs index c18bbad..2d92b41 100644 --- a/src/traits/word_ord.rs +++ b/src/traits/word_ord.rs @@ -2,7 +2,7 @@ use core::cmp::Ordering; /// Lexicographic comparison of two `u64` values (LSB-first bit order). /// -/// This is the word-level primitive used by [`BitsOrd`](super::BitsOrd) +/// This is the word-level primitive used by [`WordsOrd`](super::WordsOrd) /// to resolve ordering once the first differing word has been found. pub(crate) trait WordOrd { fn bitwise_cmp(self, other: Self) -> Ordering; diff --git a/src/traits/words_arith/funcs_for_binary_core.rs b/src/traits/words_arith/funcs_for_binary_core.rs index 6b5a546..6c42bbe 100644 --- a/src/traits/words_arith/funcs_for_binary_core.rs +++ b/src/traits/words_arith/funcs_for_binary_core.rs @@ -98,277 +98,18 @@ unsafe fn dispatch(dst: *mut u64, lhs: *const u64, rhs: *const u64 } #[allow(unused)] -mod scalar { - use super::*; - - #[inline] - fn apply(lhs: u64, rhs: u64) -> u64 { - match OP { - OP_AND => lhs & rhs, - OP_OR => lhs | rhs, - OP_XOR => lhs ^ rhs, - _ => unreachable!("unsupported binary bit operation"), - } - } - - /// Scalar backend for `dst[i] = lhs[i] OP rhs[i]`. - /// - /// Supports `dst == lhs`. - /// - /// # Safety - /// - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. - /// - `dst` must not overlap `rhs`. - #[inline] - pub(super) unsafe fn words( - dst: *mut u64, - lhs: *const u64, - rhs: *const u64, - len: usize, - ) { - for i in 0..len { - // SAFETY: - // - `i < len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == lhs` is safe because both operands are read before writing `dst[i]`. - unsafe { - let lhs_word = lhs.add(i).read(); - let rhs_word = rhs.add(i).read(); - dst.add(i).write(apply::(lhs_word, rhs_word)); - } - } - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_and_si128, _mm_loadu_si128, _mm_or_si128, _mm_storeu_si128, _mm_xor_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_and_si128, _mm_loadu_si128, _mm_or_si128, _mm_storeu_si128, _mm_xor_si128, - }; - - const LANES: usize = 2; - - #[inline] - fn apply(lhs: __m128i, rhs: __m128i) -> __m128i { - match OP { - OP_AND => unsafe { _mm_and_si128(lhs, rhs) }, - OP_OR => unsafe { _mm_or_si128(lhs, rhs) }, - OP_XOR => unsafe { _mm_xor_si128(lhs, rhs) }, - _ => unreachable!("unsupported binary bit operation"), - } - } - - /// SSE2 backend for `dst[i] = lhs[i] OP rhs[i]`. - /// - /// Supports `dst == lhs`. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. - /// - `dst` must not overlap `rhs`. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words( - dst: *mut u64, - lhs: *const u64, - rhs: *const u64, - len: usize, - ) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - Unaligned load/store intrinsics permit unaligned access. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == lhs` is safe because loads happen before the store. - unsafe { - let lhs_vec = _mm_loadu_si128(lhs.add(offset).cast::<__m128i>()); - let rhs_vec = _mm_loadu_si128(rhs.add(offset).cast::<__m128i>()); - let out_vec = apply::(lhs_vec, rhs_vec); - - _mm_storeu_si128(dst.add(offset).cast::<__m128i>(), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); - } - } -} +mod sse2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_and_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_storeu_si256, - _mm256_xor_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_and_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_storeu_si256, - _mm256_xor_si256, - }; - - const LANES: usize = 4; - - #[inline] - fn apply(lhs: __m256i, rhs: __m256i) -> __m256i { - match OP { - OP_AND => unsafe { _mm256_and_si256(lhs, rhs) }, - OP_OR => unsafe { _mm256_or_si256(lhs, rhs) }, - OP_XOR => unsafe { _mm256_xor_si256(lhs, rhs) }, - _ => unreachable!("unsupported binary bit operation"), - } - } - - /// AVX2 backend for `dst[i] = lhs[i] OP rhs[i]`. - /// - /// Supports `dst == lhs`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. - /// - `dst` must not overlap `rhs`. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words( - dst: *mut u64, - lhs: *const u64, - rhs: *const u64, - len: usize, - ) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm256_loadu_si256` and `_mm256_storeu_si256` permit unaligned access. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == lhs` is safe because both loads happen before the store. - unsafe { - let lhs_vec = _mm256_loadu_si256(lhs.add(offset).cast::<__m256i>()); - let rhs_vec = _mm256_loadu_si256(rhs.add(offset).cast::<__m256i>()); - let out_vec = apply::(lhs_vec, rhs_vec); - - _mm256_storeu_si256(dst.add(offset).cast::<__m256i>(), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); - } - } -} +mod avx2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::*; - - use core::arch::aarch64::{uint64x2_t, vandq_u64, veorq_u64, vld1q_u64, vorrq_u64, vst1q_u64}; - - const LANES: usize = 2; - - #[inline] - fn apply(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { - // SAFETY: - // - This helper is only called from `words`, which has - // `#[target_feature(enable = "neon")]`. - // - The dispatch path only reaches `words` when NEON is enabled. - unsafe { - match OP { - OP_AND => vandq_u64(lhs, rhs), - OP_OR => vorrq_u64(lhs, rhs), - OP_XOR => veorq_u64(lhs, rhs), - _ => unreachable!("unsupported binary bit operation"), - } - } - } - - /// NEON backend for `dst[i] = lhs[i] OP rhs[i]`. - /// - /// Supports `dst == lhs`. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. - /// - `dst` must not overlap `rhs`. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words( - dst: *mut u64, - lhs: *const u64, - rhs: *const u64, - len: usize, - ) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - Each NEON vector reads/writes 2 u64 values. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == lhs` is safe because both loads happen before the store. - unsafe { - let lhs_vec = vld1q_u64(lhs.add(offset)); - let rhs_vec = vld1q_u64(rhs.add(offset)); - let out_vec = apply::(lhs_vec, rhs_vec); - - vst1q_u64(dst.add(offset), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); - } - } -} - +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_arith/funcs_for_binary_core/avx2.rs b/src/traits/words_arith/funcs_for_binary_core/avx2.rs new file mode 100644 index 0000000..c8ee3a0 --- /dev/null +++ b/src/traits/words_arith/funcs_for_binary_core/avx2.rs @@ -0,0 +1,73 @@ +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_and_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_storeu_si256, + _mm256_xor_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_and_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_storeu_si256, + _mm256_xor_si256, +}; + +const LANES: usize = 4; + +#[inline] +fn apply(lhs: __m256i, rhs: __m256i) -> __m256i { + match OP { + OP_AND => unsafe { _mm256_and_si256(lhs, rhs) }, + OP_OR => unsafe { _mm256_or_si256(lhs, rhs) }, + OP_XOR => unsafe { _mm256_xor_si256(lhs, rhs) }, + _ => unreachable!("unsupported binary bit operation"), + } +} + +/// AVX2 backend for `dst[i] = lhs[i] OP rhs[i]`. +/// +/// Supports `dst == lhs`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. +/// - `dst` must not overlap `rhs`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words( + dst: *mut u64, + lhs: *const u64, + rhs: *const u64, + len: usize, +) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm256_loadu_si256` and `_mm256_storeu_si256` permit unaligned access. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == lhs` is safe because both loads happen before the store. + unsafe { + let lhs_vec = _mm256_loadu_si256(lhs.add(offset).cast::<__m256i>()); + let rhs_vec = _mm256_loadu_si256(rhs.add(offset).cast::<__m256i>()); + let out_vec = apply::(lhs_vec, rhs_vec); + + _mm256_storeu_si256(dst.add(offset).cast::<__m256i>(), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_binary_core/neon.rs b/src/traits/words_arith/funcs_for_binary_core/neon.rs new file mode 100644 index 0000000..69d12d0 --- /dev/null +++ b/src/traits/words_arith/funcs_for_binary_core/neon.rs @@ -0,0 +1,69 @@ +use super::*; + +use core::arch::aarch64::{uint64x2_t, vandq_u64, veorq_u64, vld1q_u64, vorrq_u64, vst1q_u64}; + +const LANES: usize = 2; + +#[inline] +fn apply(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { + // SAFETY: + // - This helper is only called from `words`, which has + // `#[target_feature(enable = "neon")]`. + // - The dispatch path only reaches `words` when NEON is enabled. + unsafe { + match OP { + OP_AND => vandq_u64(lhs, rhs), + OP_OR => vorrq_u64(lhs, rhs), + OP_XOR => veorq_u64(lhs, rhs), + _ => unreachable!("unsupported binary bit operation"), + } + } +} + +/// NEON backend for `dst[i] = lhs[i] OP rhs[i]`. +/// +/// Supports `dst == lhs`. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. +/// - `dst` must not overlap `rhs`. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words( + dst: *mut u64, + lhs: *const u64, + rhs: *const u64, + len: usize, +) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - Each NEON vector reads/writes 2 u64 values. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == lhs` is safe because both loads happen before the store. + unsafe { + let lhs_vec = vld1q_u64(lhs.add(offset)); + let rhs_vec = vld1q_u64(rhs.add(offset)); + let out_vec = apply::(lhs_vec, rhs_vec); + + vst1q_u64(dst.add(offset), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_binary_core/scalar.rs b/src/traits/words_arith/funcs_for_binary_core/scalar.rs new file mode 100644 index 0000000..cd47d09 --- /dev/null +++ b/src/traits/words_arith/funcs_for_binary_core/scalar.rs @@ -0,0 +1,41 @@ +use super::*; + +#[inline] +fn apply(lhs: u64, rhs: u64) -> u64 { + match OP { + OP_AND => lhs & rhs, + OP_OR => lhs | rhs, + OP_XOR => lhs ^ rhs, + _ => unreachable!("unsupported binary bit operation"), + } +} + +/// Scalar backend for `dst[i] = lhs[i] OP rhs[i]`. +/// +/// Supports `dst == lhs`. +/// +/// # Safety +/// +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. +/// - `dst` must not overlap `rhs`. +#[inline] +pub(super) unsafe fn words( + dst: *mut u64, + lhs: *const u64, + rhs: *const u64, + len: usize, +) { + for i in 0..len { + // SAFETY: + // - `i < len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == lhs` is safe because both operands are read before writing `dst[i]`. + unsafe { + let lhs_word = lhs.add(i).read(); + let rhs_word = rhs.add(i).read(); + dst.add(i).write(apply::(lhs_word, rhs_word)); + } + } +} diff --git a/src/traits/words_arith/funcs_for_binary_core/sse2.rs b/src/traits/words_arith/funcs_for_binary_core/sse2.rs new file mode 100644 index 0000000..999c626 --- /dev/null +++ b/src/traits/words_arith/funcs_for_binary_core/sse2.rs @@ -0,0 +1,71 @@ +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_and_si128, _mm_loadu_si128, _mm_or_si128, _mm_storeu_si128, _mm_xor_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_and_si128, _mm_loadu_si128, _mm_or_si128, _mm_storeu_si128, _mm_xor_si128, +}; + +const LANES: usize = 2; + +#[inline] +fn apply(lhs: __m128i, rhs: __m128i) -> __m128i { + match OP { + OP_AND => unsafe { _mm_and_si128(lhs, rhs) }, + OP_OR => unsafe { _mm_or_si128(lhs, rhs) }, + OP_XOR => unsafe { _mm_xor_si128(lhs, rhs) }, + _ => unreachable!("unsupported binary bit operation"), + } +} + +/// SSE2 backend for `dst[i] = lhs[i] OP rhs[i]`. +/// +/// Supports `dst == lhs`. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `lhs` and `rhs` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `lhs`, or be exactly equal to `lhs`. +/// - `dst` must not overlap `rhs`. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words( + dst: *mut u64, + lhs: *const u64, + rhs: *const u64, + len: usize, +) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - Unaligned load/store intrinsics permit unaligned access. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == lhs` is safe because loads happen before the store. + unsafe { + let lhs_vec = _mm_loadu_si128(lhs.add(offset).cast::<__m128i>()); + let rhs_vec = _mm_loadu_si128(rhs.add(offset).cast::<__m128i>()); + let out_vec = apply::(lhs_vec, rhs_vec); + + _mm_storeu_si128(dst.add(offset).cast::<__m128i>(), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words::(dst.add(done), lhs.add(done), rhs.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_not_core.rs b/src/traits/words_arith/funcs_for_not_core.rs index d619286..9f84a10 100644 --- a/src/traits/words_arith/funcs_for_not_core.rs +++ b/src/traits/words_arith/funcs_for_not_core.rs @@ -91,216 +91,19 @@ unsafe fn dispatch(dst: *mut u64, src: *const u64, len: usize) { } #[allow(unused)] -mod scalar { - /// Scalar backend for `dst[i] = !src[i]`. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `src` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[inline] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { - for i in 0..len { - // SAFETY: - // - `i < len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == src` is safe because `src[i]` is read before `dst[i]` is written. - unsafe { - let word = src.add(i).read(); - dst.add(i).write(!word); - } - } - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_setzero_si256, _mm256_storeu_si256, - _mm256_xor_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_setzero_si256, _mm256_storeu_si256, - _mm256_xor_si256, - }; - - const LANES: usize = 4; - - /// AVX2 backend for `dst[i] = !src[i]`. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `src` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm256_loadu_si256` and `_mm256_storeu_si256` permit unaligned access. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == src` is safe because load happens before store. - unsafe { - let zero = _mm256_setzero_si256(); - let all_ones = _mm256_cmpeq_epi8(zero, zero); - let src_vec = _mm256_loadu_si256(src.add(offset).cast::<__m256i>()); - let out_vec = _mm256_xor_si256(src_vec, all_ones); - - _mm256_storeu_si256(dst.add(offset).cast::<__m256i>(), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words(dst.add(done), src.add(done), len - done); - } - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_setzero_si128, _mm_storeu_si128, - _mm_xor_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_setzero_si128, _mm_storeu_si128, - _mm_xor_si128, - }; - - const LANES: usize = 2; - - /// SSE2 backend for `dst[i] = !src[i]`. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `src` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm_loadu_si128` and `_mm_storeu_si128` permit unaligned access. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == src` is safe because load happens before store. - unsafe { - let zero = _mm_setzero_si128(); - let all_ones = _mm_cmpeq_epi8(zero, zero); - let src_vec = _mm_loadu_si128(src.add(offset).cast::<__m128i>()); - let out_vec = _mm_xor_si128(src_vec, all_ones); - - _mm_storeu_si128(dst.add(offset).cast::<__m128i>(), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words(dst.add(done), src.add(done), len - done); - } - } -} +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::scalar; - - use core::arch::aarch64::{uint64x2_t, vdupq_n_u64, veorq_u64, vld1q_u64, vst1q_u64}; - - const LANES: usize = 2; - - #[inline] - fn not_vec(src: uint64x2_t) -> uint64x2_t { - // SAFETY: - // - This helper is only called from `words`, which has - // `#[target_feature(enable = "neon")]`. - // - The dispatch path only reaches `words` when NEON is enabled. - unsafe { veorq_u64(src, vdupq_n_u64(u64::MAX)) } - } - - /// NEON backend for `dst[i] = !src[i]`. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `dst` must be valid for writes of `len` initialized `u64` values. - /// - `src` must be valid for reads of `len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { - let chunks = len / LANES; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - Each NEON vector reads/writes 2 u64 values. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - `dst == src` is safe because load happens before store. - unsafe { - let src_vec = vld1q_u64(src.add(offset)); - let out_vec = not_vec(src_vec); - - vst1q_u64(dst.add(offset), out_vec); - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - scalar::words(dst.add(done), src.add(done), len - done); - } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_arith/funcs_for_not_core/avx2.rs b/src/traits/words_arith/funcs_for_not_core/avx2.rs new file mode 100644 index 0000000..9638103 --- /dev/null +++ b/src/traits/words_arith/funcs_for_not_core/avx2.rs @@ -0,0 +1,58 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_setzero_si256, _mm256_storeu_si256, + _mm256_xor_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_cmpeq_epi8, _mm256_loadu_si256, _mm256_setzero_si256, _mm256_storeu_si256, + _mm256_xor_si256, +}; + +const LANES: usize = 4; + +/// AVX2 backend for `dst[i] = !src[i]`. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `src` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm256_loadu_si256` and `_mm256_storeu_si256` permit unaligned access. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == src` is safe because load happens before store. + unsafe { + let zero = _mm256_setzero_si256(); + let all_ones = _mm256_cmpeq_epi8(zero, zero); + let src_vec = _mm256_loadu_si256(src.add(offset).cast::<__m256i>()); + let out_vec = _mm256_xor_si256(src_vec, all_ones); + + _mm256_storeu_si256(dst.add(offset).cast::<__m256i>(), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words(dst.add(done), src.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_not_core/neon.rs b/src/traits/words_arith/funcs_for_not_core/neon.rs new file mode 100644 index 0000000..dddfcef --- /dev/null +++ b/src/traits/words_arith/funcs_for_not_core/neon.rs @@ -0,0 +1,55 @@ +use super::scalar; + +use core::arch::aarch64::{uint64x2_t, vdupq_n_u64, veorq_u64, vld1q_u64, vst1q_u64}; + +const LANES: usize = 2; + +#[inline] +fn not_vec(src: uint64x2_t) -> uint64x2_t { + // SAFETY: + // - This helper is only called from `words`, which has + // `#[target_feature(enable = "neon")]`. + // - The dispatch path only reaches `words` when NEON is enabled. + unsafe { veorq_u64(src, vdupq_n_u64(u64::MAX)) } +} + +/// NEON backend for `dst[i] = !src[i]`. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `src` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - Each NEON vector reads/writes 2 u64 values. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == src` is safe because load happens before store. + unsafe { + let src_vec = vld1q_u64(src.add(offset)); + let out_vec = not_vec(src_vec); + + vst1q_u64(dst.add(offset), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words(dst.add(done), src.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_not_core/scalar.rs b/src/traits/words_arith/funcs_for_not_core/scalar.rs new file mode 100644 index 0000000..dd13c56 --- /dev/null +++ b/src/traits/words_arith/funcs_for_not_core/scalar.rs @@ -0,0 +1,22 @@ +/// Scalar backend for `dst[i] = !src[i]`. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `src` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[inline] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { + for i in 0..len { + // SAFETY: + // - `i < len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == src` is safe because `src[i]` is read before `dst[i]` is written. + unsafe { + let word = src.add(i).read(); + dst.add(i).write(!word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_not_core/sse2.rs b/src/traits/words_arith/funcs_for_not_core/sse2.rs new file mode 100644 index 0000000..82efcdd --- /dev/null +++ b/src/traits/words_arith/funcs_for_not_core/sse2.rs @@ -0,0 +1,56 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_setzero_si128, _mm_storeu_si128, _mm_xor_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi8, _mm_loadu_si128, _mm_setzero_si128, _mm_storeu_si128, _mm_xor_si128, +}; + +const LANES: usize = 2; + +/// SSE2 backend for `dst[i] = !src[i]`. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `dst` must be valid for writes of `len` initialized `u64` values. +/// - `src` must be valid for reads of `len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, len: usize) { + let chunks = len / LANES; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm_loadu_si128` and `_mm_storeu_si128` permit unaligned access. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - `dst == src` is safe because load happens before store. + unsafe { + let zero = _mm_setzero_si128(); + let all_ones = _mm_cmpeq_epi8(zero, zero); + let src_vec = _mm_loadu_si128(src.add(offset).cast::<__m128i>()); + let out_vec = _mm_xor_si128(src_vec, all_ones); + + _mm_storeu_si128(dst.add(offset).cast::<__m128i>(), out_vec); + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + scalar::words(dst.add(done), src.add(done), len - done); + } +} diff --git a/src/traits/words_arith/funcs_for_shl_core.rs b/src/traits/words_arith/funcs_for_shl_core.rs index 42ab3dc..e933b47 100644 --- a/src/traits/words_arith/funcs_for_shl_core.rs +++ b/src/traits/words_arith/funcs_for_shl_core.rs @@ -130,320 +130,19 @@ unsafe fn scalar_word(src: *const u64, word_len: usize, dst_index: usize, amount } #[allow(unused)] -mod scalar { - use super::scalar_word; - - /// Scalar backend for word-level left shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - for dst_index in (0..word_len).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - Descending order makes `dst == src` safe for left shift. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - const LANES: usize = 4; - - /// AVX2 backend for word-level left shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in (0..word_len).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_start = word_shift + usize::from(bit_shift != 0); - let mut end = word_len; - - // Process the vectorizable suffix from high to low. - while end >= bulk_start + LANES { - let dst_start = end - LANES; - let src_start = dst_start - word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES <= word_len`. - // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. - // - Unaligned load/store intrinsics permit unaligned access. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from high to low. - unsafe { - let cur = _mm256_loadu_si256(src.add(src_start).cast::<__m256i>()); - let out = if bit_shift == 0 { - cur - } else { - let prev = _mm256_loadu_si256(src.add(src_start - 1).cast::<__m256i>()); - let left_count: __m128i = _mm_cvtsi64_si128(bit_shift as i64); - let right_count: __m128i = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); - - let left = _mm256_sll_epi64(cur, left_count); - let right = _mm256_srl_epi64(prev, right_count); - - _mm256_or_si256(left, right) - }; - - _mm256_storeu_si256(dst.add(dst_start).cast::<__m256i>(), out); - } - - end = dst_start; - } - - // Scalar tail below the vectorized suffix, still high to low. - for dst_index in (0..end).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - const LANES: usize = 2; - - /// SSE2 backend for word-level left shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in (0..word_len).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_start = word_shift + usize::from(bit_shift != 0); - let mut end = word_len; - - while end >= bulk_start + LANES { - let dst_start = end - LANES; - let src_start = dst_start - word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES <= word_len`. - // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. - // - Unaligned load/store intrinsics permit unaligned access. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from high to low. - unsafe { - let cur = _mm_loadu_si128(src.add(src_start).cast::<__m128i>()); - let out = if bit_shift == 0 { - cur - } else { - let prev = _mm_loadu_si128(src.add(src_start - 1).cast::<__m128i>()); - let left_count = _mm_cvtsi64_si128(bit_shift as i64); - let right_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); - - let left = _mm_sll_epi64(cur, left_count); - let right = _mm_srl_epi64(prev, right_count); - - _mm_or_si128(left, right) - }; - - _mm_storeu_si128(dst.add(dst_start).cast::<__m128i>(), out); - } - - end = dst_start; - } - - for dst_index in (0..end).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - use core::arch::aarch64::{ - int64x2_t, uint64x2_t, vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64, - }; - - const LANES: usize = 2; - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn shl_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vshlq_u64(src, vdupq_n_s64(amount as i64) as int64x2_t) } - } - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn shr_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vshlq_u64(src, vdupq_n_s64(-(amount as i64)) as int64x2_t) } - } - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn or_vec(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vorrq_u64(lhs, rhs) } - } - - /// NEON backend for word-level left shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in (0..word_len).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_start = word_shift + usize::from(bit_shift != 0); - let mut end = word_len; - - while end >= bulk_start + LANES { - let dst_start = end - LANES; - let src_start = dst_start - word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES <= word_len`. - // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. - // - `vld1q_u64` reads exactly 2 u64 values. - // - `vst1q_u64` writes exactly 2 u64 values. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from high to low. - unsafe { - let cur = vld1q_u64(src.add(src_start)); - let out = if bit_shift == 0 { - cur - } else { - let prev = vld1q_u64(src.add(src_start - 1)); - let left = shl_vec(cur, bit_shift); - let right = shr_vec(prev, WORD_BITS - bit_shift); - - or_vec(left, right) - }; - - vst1q_u64(dst.add(dst_start), out); - } - - end = dst_start; - } - - for dst_index in (0..end).rev() { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_arith/funcs_for_shl_core/avx2.rs b/src/traits/words_arith/funcs_for_shl_core/avx2.rs new file mode 100644 index 0000000..b13adff --- /dev/null +++ b/src/traits/words_arith/funcs_for_shl_core/avx2.rs @@ -0,0 +1,88 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +const LANES: usize = 4; + +/// AVX2 backend for word-level left shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in (0..word_len).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_start = word_shift + usize::from(bit_shift != 0); + let mut end = word_len; + + // Process the vectorizable suffix from high to low. + while end >= bulk_start + LANES { + let dst_start = end - LANES; + let src_start = dst_start - word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES <= word_len`. + // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. + // - Unaligned load/store intrinsics permit unaligned access. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from high to low. + unsafe { + let cur = _mm256_loadu_si256(src.add(src_start).cast::<__m256i>()); + let out = if bit_shift == 0 { + cur + } else { + let prev = _mm256_loadu_si256(src.add(src_start - 1).cast::<__m256i>()); + let left_count: __m128i = _mm_cvtsi64_si128(bit_shift as i64); + let right_count: __m128i = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + + let left = _mm256_sll_epi64(cur, left_count); + let right = _mm256_srl_epi64(prev, right_count); + + _mm256_or_si256(left, right) + }; + + _mm256_storeu_si256(dst.add(dst_start).cast::<__m256i>(), out); + } + + end = dst_start; + } + + // Scalar tail below the vectorized suffix, still high to low. + for dst_index in (0..end).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shl_core/neon.rs b/src/traits/words_arith/funcs_for_shl_core/neon.rs new file mode 100644 index 0000000..f426911 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shl_core/neon.rs @@ -0,0 +1,100 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +use core::arch::aarch64::{ + int64x2_t, uint64x2_t, vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64, +}; + +const LANES: usize = 2; + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn shl_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vshlq_u64(src, vdupq_n_s64(amount as i64) as int64x2_t) } +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn shr_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vshlq_u64(src, vdupq_n_s64(-(amount as i64)) as int64x2_t) } +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn or_vec(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vorrq_u64(lhs, rhs) } +} + +/// NEON backend for word-level left shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in (0..word_len).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_start = word_shift + usize::from(bit_shift != 0); + let mut end = word_len; + + while end >= bulk_start + LANES { + let dst_start = end - LANES; + let src_start = dst_start - word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES <= word_len`. + // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. + // - `vld1q_u64` reads exactly 2 u64 values. + // - `vst1q_u64` writes exactly 2 u64 values. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from high to low. + unsafe { + let cur = vld1q_u64(src.add(src_start)); + let out = if bit_shift == 0 { + cur + } else { + let prev = vld1q_u64(src.add(src_start - 1)); + let left = shl_vec(cur, bit_shift); + let right = shr_vec(prev, WORD_BITS - bit_shift); + + or_vec(left, right) + }; + + vst1q_u64(dst.add(dst_start), out); + } + + end = dst_start; + } + + for dst_index in (0..end).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shl_core/scalar.rs b/src/traits/words_arith/funcs_for_shl_core/scalar.rs new file mode 100644 index 0000000..407d362 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shl_core/scalar.rs @@ -0,0 +1,23 @@ +use super::scalar_word; + +/// Scalar backend for word-level left shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + for dst_index in (0..word_len).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - Descending order makes `dst == src` safe for left shift. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shl_core/sse2.rs b/src/traits/words_arith/funcs_for_shl_core/sse2.rs new file mode 100644 index 0000000..7d74057 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shl_core/sse2.rs @@ -0,0 +1,86 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +const LANES: usize = 2; + +/// SSE2 backend for word-level left shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in (0..word_len).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_start = word_shift + usize::from(bit_shift != 0); + let mut end = word_len; + + while end >= bulk_start + LANES { + let dst_start = end - LANES; + let src_start = dst_start - word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES <= word_len`. + // - If `bit_shift != 0`, `bulk_start` guarantees `src_start > 0`. + // - Unaligned load/store intrinsics permit unaligned access. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from high to low. + unsafe { + let cur = _mm_loadu_si128(src.add(src_start).cast::<__m128i>()); + let out = if bit_shift == 0 { + cur + } else { + let prev = _mm_loadu_si128(src.add(src_start - 1).cast::<__m128i>()); + let left_count = _mm_cvtsi64_si128(bit_shift as i64); + let right_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + + let left = _mm_sll_epi64(cur, left_count); + let right = _mm_srl_epi64(prev, right_count); + + _mm_or_si128(left, right) + }; + + _mm_storeu_si128(dst.add(dst_start).cast::<__m128i>(), out); + } + + end = dst_start; + } + + for dst_index in (0..end).rev() { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shr_core.rs b/src/traits/words_arith/funcs_for_shr_core.rs index 6086500..578fe5c 100644 --- a/src/traits/words_arith/funcs_for_shr_core.rs +++ b/src/traits/words_arith/funcs_for_shr_core.rs @@ -128,339 +128,19 @@ unsafe fn scalar_word(src: *const u64, word_len: usize, dst_index: usize, amount } #[allow(unused)] -mod scalar { - use super::scalar_word; - - /// Scalar backend for word-level right shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - for dst_index in 0..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - // - Ascending order makes `dst == src` safe for right shift. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - const LANES: usize = 4; - - /// AVX2 backend for word-level right shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in 0..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); - let mut start = 0usize; - - // Process the vectorizable prefix from low to high. - while start + LANES <= bulk_end { - let dst_start = start; - let src_start = dst_start + word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len` (since `dst_start + LANES <= bulk_end <= word_len`). - // - `src_start + LANES + 1 <= word_len` (since `bulk_end` guarantees room for the carry word). - // - Unaligned load/store intrinsics permit unaligned access. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from low to high. - unsafe { - let cur = _mm256_loadu_si256(src.add(src_start).cast::<__m256i>()); - let out = if bit_shift == 0 { - cur - } else { - let next = _mm256_loadu_si256(src.add(src_start + 1).cast::<__m256i>()); - let right_count: __m128i = _mm_cvtsi64_si128(bit_shift as i64); - let left_count: __m128i = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); - - let right = _mm256_srl_epi64(cur, right_count); - let left = _mm256_sll_epi64(next, left_count); - - _mm256_or_si256(right, left) - }; - - _mm256_storeu_si256(dst.add(dst_start).cast::<__m256i>(), out); - } - - start = dst_start + LANES; - } - - // Non-zero scalar tail. - for dst_index in start..word_len - word_shift { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - - // Zero-fill suffix: dst_index >= word_len - word_shift. - for dst_index in (word_len - word_shift)..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - const LANES: usize = 2; - - /// SSE2 backend for word-level right shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when SSE2 is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in 0..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); - let mut start = 0usize; - - while start + LANES <= bulk_end { - let dst_start = start; - let src_start = dst_start + word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES + 1 <= word_len` (guaranteed by `bulk_end`). - // - Unaligned load/store intrinsics permit unaligned access. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from low to high. - unsafe { - let cur = _mm_loadu_si128(src.add(src_start).cast::<__m128i>()); - let out = if bit_shift == 0 { - cur - } else { - let next = _mm_loadu_si128(src.add(src_start + 1).cast::<__m128i>()); - let right_count = _mm_cvtsi64_si128(bit_shift as i64); - let left_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); - - let right = _mm_srl_epi64(cur, right_count); - let left = _mm_sll_epi64(next, left_count); - - _mm_or_si128(right, left) - }; - - _mm_storeu_si128(dst.add(dst_start).cast::<__m128i>(), out); - } - - start = dst_start + LANES; - } - - for dst_index in start..word_len - word_shift { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - - for dst_index in (word_len - word_shift)..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - } -} +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::{scalar_word, split_amount}; - use crate::WORD_BITS; - - use core::arch::aarch64::{ - int64x2_t, uint64x2_t, vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64, - }; - - const LANES: usize = 2; - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn shl_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vshlq_u64(src, vdupq_n_s64(amount as i64) as int64x2_t) } - } - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn shr_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vshlq_u64(src, vdupq_n_s64(-(amount as i64)) as int64x2_t) } - } - - #[inline] - #[target_feature(enable = "neon")] - unsafe fn or_vec(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { - // SAFETY: - // - This helper is called only from `words`, which enables NEON. - unsafe { vorrq_u64(lhs, rhs) } - } - - /// NEON backend for word-level right shift. - /// - /// Supports `dst == src`. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `dst` must be valid for writes of `word_len` initialized `u64` values. - /// - `src` must be valid for reads of `word_len` initialized `u64` values. - /// - `dst` must either not overlap `src`, or be exactly equal to `src`. - #[target_feature(enable = "neon")] - pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - let (word_shift, bit_shift) = split_amount(amount); - - if word_shift >= word_len { - for dst_index in 0..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - return; - } - - let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); - let mut start = 0usize; - - while start + LANES <= bulk_end { - let dst_start = start; - let src_start = dst_start + word_shift; - - // SAFETY: - // - `dst_start + LANES <= word_len`. - // - `src_start + LANES + 1 <= word_len` (guaranteed by `bulk_end`). - // - `vld1q_u64` reads exactly 2 u64 values. - // - `vst1q_u64` writes exactly 2 u64 values. - // - `dst == src` is safe because all loads happen before the store, - // and chunks are processed from low to high. - unsafe { - let cur = vld1q_u64(src.add(src_start)); - let out = if bit_shift == 0 { - cur - } else { - let next = vld1q_u64(src.add(src_start + 1)); - let right = shr_vec(cur, bit_shift); - let left = shl_vec(next, WORD_BITS - bit_shift); - - or_vec(right, left) - }; - - vst1q_u64(dst.add(dst_start), out); - } - - start = dst_start + LANES; - } - - for dst_index in start..word_len - word_shift { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity and overlap constraints are guaranteed by the caller. - unsafe { - let word = scalar_word(src, word_len, dst_index, amount); - dst.add(dst_index).write(word); - } - } - - for dst_index in (word_len - word_shift)..word_len { - // SAFETY: - // - `dst_index < word_len`. - // - Pointer validity is guaranteed by the caller. - unsafe { dst.add(dst_index).write(0) }; - } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_arith/funcs_for_shr_core/avx2.rs b/src/traits/words_arith/funcs_for_shr_core/avx2.rs new file mode 100644 index 0000000..07a8364 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shr_core/avx2.rs @@ -0,0 +1,95 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_cvtsi64_si128, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +const LANES: usize = 4; + +/// AVX2 backend for word-level right shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in 0..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); + let mut start = 0usize; + + // Process the vectorizable prefix from low to high. + while start + LANES <= bulk_end { + let dst_start = start; + let src_start = dst_start + word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len` (since `dst_start + LANES <= bulk_end <= word_len`). + // - `src_start + LANES + 1 <= word_len` (since `bulk_end` guarantees room for the carry word). + // - Unaligned load/store intrinsics permit unaligned access. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from low to high. + unsafe { + let cur = _mm256_loadu_si256(src.add(src_start).cast::<__m256i>()); + let out = if bit_shift == 0 { + cur + } else { + let next = _mm256_loadu_si256(src.add(src_start + 1).cast::<__m256i>()); + let right_count: __m128i = _mm_cvtsi64_si128(bit_shift as i64); + let left_count: __m128i = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + + let right = _mm256_srl_epi64(cur, right_count); + let left = _mm256_sll_epi64(next, left_count); + + _mm256_or_si256(right, left) + }; + + _mm256_storeu_si256(dst.add(dst_start).cast::<__m256i>(), out); + } + + start = dst_start + LANES; + } + + // Non-zero scalar tail. + for dst_index in start..word_len - word_shift { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } + + // Zero-fill suffix: dst_index >= word_len - word_shift. + for dst_index in (word_len - word_shift)..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } +} diff --git a/src/traits/words_arith/funcs_for_shr_core/neon.rs b/src/traits/words_arith/funcs_for_shr_core/neon.rs new file mode 100644 index 0000000..ae45eb2 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shr_core/neon.rs @@ -0,0 +1,106 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +use core::arch::aarch64::{ + int64x2_t, uint64x2_t, vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64, +}; + +const LANES: usize = 2; + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn shl_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vshlq_u64(src, vdupq_n_s64(amount as i64) as int64x2_t) } +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn shr_vec(src: uint64x2_t, amount: usize) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vshlq_u64(src, vdupq_n_s64(-(amount as i64)) as int64x2_t) } +} + +#[inline] +#[target_feature(enable = "neon")] +unsafe fn or_vec(lhs: uint64x2_t, rhs: uint64x2_t) -> uint64x2_t { + // SAFETY: + // - This helper is called only from `words`, which enables NEON. + unsafe { vorrq_u64(lhs, rhs) } +} + +/// NEON backend for word-level right shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "neon")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in 0..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); + let mut start = 0usize; + + while start + LANES <= bulk_end { + let dst_start = start; + let src_start = dst_start + word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES + 1 <= word_len` (guaranteed by `bulk_end`). + // - `vld1q_u64` reads exactly 2 u64 values. + // - `vst1q_u64` writes exactly 2 u64 values. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from low to high. + unsafe { + let cur = vld1q_u64(src.add(src_start)); + let out = if bit_shift == 0 { + cur + } else { + let next = vld1q_u64(src.add(src_start + 1)); + let right = shr_vec(cur, bit_shift); + let left = shl_vec(next, WORD_BITS - bit_shift); + + or_vec(right, left) + }; + + vst1q_u64(dst.add(dst_start), out); + } + + start = dst_start + LANES; + } + + for dst_index in start..word_len - word_shift { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } + + for dst_index in (word_len - word_shift)..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } +} diff --git a/src/traits/words_arith/funcs_for_shr_core/scalar.rs b/src/traits/words_arith/funcs_for_shr_core/scalar.rs new file mode 100644 index 0000000..65936c0 --- /dev/null +++ b/src/traits/words_arith/funcs_for_shr_core/scalar.rs @@ -0,0 +1,23 @@ +use super::scalar_word; + +/// Scalar backend for word-level right shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + for dst_index in 0..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + // - Ascending order makes `dst == src` safe for right shift. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } +} diff --git a/src/traits/words_arith/funcs_for_shr_core/sse2.rs b/src/traits/words_arith/funcs_for_shr_core/sse2.rs new file mode 100644 index 0000000..6c2d8bd --- /dev/null +++ b/src/traits/words_arith/funcs_for_shr_core/sse2.rs @@ -0,0 +1,92 @@ +use super::{scalar_word, split_amount}; +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +const LANES: usize = 2; + +/// SSE2 backend for word-level right shift. +/// +/// Supports `dst == src`. +/// +/// # Safety +/// +/// - Caller must only call this when SSE2 is available. +/// - `dst` must be valid for writes of `word_len` initialized `u64` values. +/// - `src` must be valid for reads of `word_len` initialized `u64` values. +/// - `dst` must either not overlap `src`, or be exactly equal to `src`. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { + let (word_shift, bit_shift) = split_amount(amount); + + if word_shift >= word_len { + for dst_index in 0..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } + return; + } + + let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); + let mut start = 0usize; + + while start + LANES <= bulk_end { + let dst_start = start; + let src_start = dst_start + word_shift; + + // SAFETY: + // - `dst_start + LANES <= word_len`. + // - `src_start + LANES + 1 <= word_len` (guaranteed by `bulk_end`). + // - Unaligned load/store intrinsics permit unaligned access. + // - `dst == src` is safe because all loads happen before the store, + // and chunks are processed from low to high. + unsafe { + let cur = _mm_loadu_si128(src.add(src_start).cast::<__m128i>()); + let out = if bit_shift == 0 { + cur + } else { + let next = _mm_loadu_si128(src.add(src_start + 1).cast::<__m128i>()); + let right_count = _mm_cvtsi64_si128(bit_shift as i64); + let left_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + + let right = _mm_srl_epi64(cur, right_count); + let left = _mm_sll_epi64(next, left_count); + + _mm_or_si128(right, left) + }; + + _mm_storeu_si128(dst.add(dst_start).cast::<__m128i>(), out); + } + + start = dst_start + LANES; + } + + for dst_index in start..word_len - word_shift { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity and overlap constraints are guaranteed by the caller. + unsafe { + let word = scalar_word(src, word_len, dst_index, amount); + dst.add(dst_index).write(word); + } + } + + for dst_index in (word_len - word_shift)..word_len { + // SAFETY: + // - `dst_index < word_len`. + // - Pointer validity is guaranteed by the caller. + unsafe { dst.add(dst_index).write(0) }; + } +} diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs index 600af9a..22a2cd1 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs @@ -24,9 +24,7 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi debug_assert!(shift > 0 && shift < WORD_BITS); if count < SMALL_WORDS { - for i in 0..count { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - } + scalar::copy_words_shifted(dst, src, count, shift); return; } @@ -60,164 +58,28 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi #[allow(unused)] { - for i in 0..count { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - } + scalar::copy_words_shifted(dst, src, count, shift); } } // --------------------------------------------------------------------------- -// AVX2 — 4 words per iteration +// Backends // --------------------------------------------------------------------------- #[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, - _mm256_srl_epi64, _mm256_storeu_si256, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn copy_words_shifted( - dst: &mut [u64], - src: &[u64], - len: usize, - shift: usize, - ) { - // SAFETY: `#[target_feature(enable = "avx2")]` guarantees the CPU supports AVX2; the `unsafe fn` contract guarantees pointer validity. - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - // SAFETY: Same as above. - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 4 <= len { - // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm256_loadu_si256` uses unaligned loads so alignment is not required. - let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: Same as above. - let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; - // SAFETY: Pure register operations; no memory access. - let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; - // SAFETY: Same as above. - let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; - // SAFETY: Same as above. - let window = unsafe { _mm256_or_si256(lo, hi) }; - // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm256_storeu_si256` uses unaligned stores. - unsafe { _mm256_storeu_si256(dst.as_mut_ptr().add(i).cast::<__m256i>(), window) }; - i += 4; - } - while i < len { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - i += 1; - } - } -} - -// --------------------------------------------------------------------------- -// SSE2 — 2 words per iteration -// --------------------------------------------------------------------------- +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, - }; - - #[target_feature(enable = "sse2")] - pub(super) unsafe fn copy_words_shifted( - dst: &mut [u64], - src: &[u64], - len: usize, - shift: usize, - ) { - // SAFETY: `#[target_feature(enable = "sse2")]` guarantees the CPU supports SSE2; the `unsafe fn` contract guarantees pointer validity. - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - // SAFETY: Same as above. - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 2 <= len { - // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm_loadu_si128` uses unaligned loads so alignment is not required. - let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: Same as above. - let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; - // SAFETY: Pure register operations; no memory access. - let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; - // SAFETY: Same as above. - let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; - // SAFETY: Same as above. - let window = unsafe { _mm_or_si128(lo, hi) }; - // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm_storeu_si128` uses unaligned stores. - unsafe { _mm_storeu_si128(dst.as_mut_ptr().add(i).cast::<__m128i>(), window) }; - i += 2; - } - while i < len { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - i += 1; - } - } -} +mod avx2; -// --------------------------------------------------------------------------- -// NEON — 2 words per iteration -// --------------------------------------------------------------------------- +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use crate::WORD_BITS; - use core::arch::aarch64::{vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64}; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn copy_words_shifted( - dst: &mut [u64], - src: &[u64], - len: usize, - shift: usize, - ) { - // SAFETY: `#[target_feature(enable = "neon")]` guarantees the CPU supports NEON; the `unsafe fn` contract guarantees pointer validity. - let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; - // SAFETY: Same as above. - let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; - - let mut i = 0; - while i + 2 <= len { - // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee). - let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - // SAFETY: Same as above. - let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; - // SAFETY: Pure register operations; no memory access. - let lo = unsafe { vshlq_u64(w0, neg_shift) }; - // SAFETY: Same as above. - let hi = unsafe { vshlq_u64(w1, pos_shift) }; - // SAFETY: Same as above. - let window = unsafe { vorrq_u64(lo, hi) }; - // SAFETY: `dst` pointer is valid for `len` words (caller guarantee). - unsafe { vst1q_u64(dst.as_mut_ptr().add(i), window) }; - i += 2; - } - while i < len { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - i += 1; - } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs new file mode 100644 index 0000000..aa69a99 --- /dev/null +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs @@ -0,0 +1,40 @@ +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_loadu_si256, _mm256_or_si256, _mm256_sll_epi64, + _mm256_srl_epi64, _mm256_storeu_si256, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn copy_words_shifted(dst: &mut [u64], src: &[u64], len: usize, shift: usize) { + // SAFETY: `#[target_feature(enable = "avx2")]` guarantees the CPU supports AVX2; the `unsafe fn` contract guarantees pointer validity. + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: Same as above. + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 4 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm256_loadu_si256` uses unaligned loads so alignment is not required. + let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: Same as above. + let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; + // SAFETY: Pure register operations; no memory access. + let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + // SAFETY: Same as above. + let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + // SAFETY: Same as above. + let window = unsafe { _mm256_or_si256(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm256_storeu_si256` uses unaligned stores. + unsafe { _mm256_storeu_si256(dst.as_mut_ptr().add(i).cast::<__m256i>(), window) }; + i += 4; + } + while i < len { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + i += 1; + } +} diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/neon.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/neon.rs new file mode 100644 index 0000000..034e35b --- /dev/null +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/neon.rs @@ -0,0 +1,31 @@ +use crate::WORD_BITS; +use core::arch::aarch64::{vdupq_n_s64, vld1q_u64, vorrq_u64, vshlq_u64, vst1q_u64}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn copy_words_shifted(dst: &mut [u64], src: &[u64], len: usize, shift: usize) { + // SAFETY: `#[target_feature(enable = "neon")]` guarantees the CPU supports NEON; the `unsafe fn` contract guarantees pointer validity. + let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + // SAFETY: Same as above. + let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; + + let mut i = 0; + while i + 2 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee). + let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: Same as above. + let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; + // SAFETY: Pure register operations; no memory access. + let lo = unsafe { vshlq_u64(w0, neg_shift) }; + // SAFETY: Same as above. + let hi = unsafe { vshlq_u64(w1, pos_shift) }; + // SAFETY: Same as above. + let window = unsafe { vorrq_u64(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee). + unsafe { vst1q_u64(dst.as_mut_ptr().add(i), window) }; + i += 2; + } + while i < len { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + i += 1; + } +} diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/scalar.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/scalar.rs new file mode 100644 index 0000000..6e8853a --- /dev/null +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/scalar.rs @@ -0,0 +1,9 @@ +use crate::WORD_BITS; + +/// Scalar backend for shifted 64-bit windows. +#[inline] +pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shift: usize) { + for i in 0..count { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + } +} diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/sse2.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/sse2.rs new file mode 100644 index 0000000..bd7e23f --- /dev/null +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/sse2.rs @@ -0,0 +1,40 @@ +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_loadu_si128, _mm_or_si128, _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, + _mm_storeu_si128, +}; + +#[target_feature(enable = "sse2")] +pub(super) unsafe fn copy_words_shifted(dst: &mut [u64], src: &[u64], len: usize, shift: usize) { + // SAFETY: `#[target_feature(enable = "sse2")]` guarantees the CPU supports SSE2; the `unsafe fn` contract guarantees pointer validity. + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: Same as above. + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 2 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm_loadu_si128` uses unaligned loads so alignment is not required. + let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: Same as above. + let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; + // SAFETY: Pure register operations; no memory access. + let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; + // SAFETY: Same as above. + let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; + // SAFETY: Same as above. + let window = unsafe { _mm_or_si128(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm_storeu_si128` uses unaligned stores. + unsafe { _mm_storeu_si128(dst.as_mut_ptr().add(i).cast::<__m128i>(), window) }; + i += 2; + } + while i < len { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + i += 1; + } +} diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs index dcfb3f0..cf04373 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs @@ -12,14 +12,6 @@ fn config() -> ProptestConfig { } } -fn scalar_copy(dst: &mut [u64], src: &[u64], count: usize, shift: usize) { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - dst[i] = (w0 >> shift) | (w1 << (64 - shift)); - } -} - proptest! { #![proptest_config(config())] @@ -34,7 +26,7 @@ proptest! { let mut dst_scalar = vec![0u64; count + extra]; super::copy_words_shifted(&mut dst_simd, &src, count, shift); - scalar_copy(&mut dst_scalar, &src, count, shift); + super::scalar::copy_words_shifted(&mut dst_scalar, &src, count, shift); assert_eq!(&dst_simd[..count], &dst_scalar[..count]); } diff --git a/src/traits/words_eq.rs b/src/traits/words_eq.rs index f654922..c0740c5 100644 --- a/src/traits/words_eq.rs +++ b/src/traits/words_eq.rs @@ -1,22 +1,20 @@ /// Word-level equality comparison on `[u64]` backing storage. /// -/// Accepts a bit-level `offset` and handles word slicing and intra-word -/// shift internally: -/// - `offset % WORD_BITS == 0` — word-aligned, uses [`funcs_for_eq_words_aligned_core`]. -/// - `offset % WORD_BITS != 0` — unaligned shifted-window, uses [`funcs_for_eq_words_unaligned_core`]. +/// The caller pre-trims both slices to the relevant backing words and passes +/// the haystack's intra-word shift separately: +/// - `haystack_shift == 0` — word-aligned, uses [`funcs_for_eq_words_aligned_core`]. +/// - `haystack_shift != 0` — unaligned shifted-window, uses [`funcs_for_eq_words_unaligned_core`]. /// -/// Short inputs fall back to scalar in both backends. +/// This trait does not perform bit-level slicing or derive backing-word +/// offsets; those are handled by the higher-level view implementation. +/// +/// Short inputs fall back to scalar before SIMD dispatch. pub(crate) trait WordsEq { - /// Returns `true` if `other` matches `self` starting at `offset` bits. + /// Compares `full_words` logical haystack words with `needle`. /// - /// `count` is the number of full `u64` words to compare (computed from - /// the needle bit length). The intra-word shift and word slicing are - /// derived from `offset` internally. - /// `self` is pre-trimmed haystack `words[base..]`. - /// `needle` is always word-aligned (pre-trimmed by the caller). - /// `full_words` is `needle_bit_len / WORD_BITS` — how many - /// complete u64 words to compare. - /// `haystack_shift` is `original_offset % WORD_BITS`. + /// `self` is the pre-trimmed haystack backing slice. + /// `needle` is word-aligned and pre-trimmed by the caller. + /// `haystack_shift` is the physical start offset within `self[0]`. /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is /// guaranteed and the aligned backend is used unconditionally. /// When it is `false`, no alignment guarantee is made. diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs index 3c2a9cb..eea0d97 100644 --- a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs @@ -14,12 +14,7 @@ use crate::SMALL_WORDS; #[inline] pub(super) fn eq_words_aligned(src: &[u64], other: &[u64], count: usize) -> bool { if count < SMALL_WORDS { - for i in 0..count { - if src[i] != other[i] { - return false; - } - } - return true; + return scalar::eq_words(src, other, count); } #[cfg(all( @@ -48,117 +43,23 @@ pub(super) fn eq_words_aligned(src: &[u64], other: &[u64], count: usize) -> bool } #[allow(unused)] - { - for i in 0..count { - if src[i] != other[i] { - return false; - } - } - true - } + scalar::eq_words(src, other, count) } #[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 4 <= len { - // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. - // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). - let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: same as above; load from `other`. - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: `_mm256_cmpeq_epi64` and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. - let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; - // SAFETY: same as above - if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { - return false; - } - i += 4; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; +mod avx2; - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 2 <= len { - // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. - // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). - let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: same as above; load from `other`. - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: `_mm_cmpeq_epi64` and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. - let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; - // SAFETY: same as above - if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { - return false; - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod sse41; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 2 <= len { - // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. - // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). - let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - // SAFETY: same as above; load from `other`. - let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - // SAFETY: `vceqq_u64` and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. - let cmp = unsafe { vceqq_u64(a, b) }; - // SAFETY: same as above - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - return false; - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core/avx2.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/avx2.rs new file mode 100644 index 0000000..2cf67e1 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/avx2.rs @@ -0,0 +1,30 @@ +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 4 <= len { + // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: `_mm256_cmpeq_epi64` and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; + // SAFETY: same as above + if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { + return false; + } + i += 4; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core/neon.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/neon.rs new file mode 100644 index 0000000..412ade4 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/neon.rs @@ -0,0 +1,27 @@ +use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + // SAFETY: `vceqq_u64` and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. + let cmp = unsafe { vceqq_u64(a, b) }; + // SAFETY: same as above + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + return false; + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core/scalar.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/scalar.rs new file mode 100644 index 0000000..a7c502c --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/scalar.rs @@ -0,0 +1,8 @@ +pub(super) fn eq_words(src: &[u64], other: &[u64], count: usize) -> bool { + for i in 0..count { + if src[i] != other[i] { + return false; + } + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core/sse41.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/sse41.rs new file mode 100644 index 0000000..5fe6141 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/sse41.rs @@ -0,0 +1,30 @@ +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: `_mm_cmpeq_epi64` and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; + // SAFETY: same as above + if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { + return false; + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs index 2a9fd69..a83f76f 100644 --- a/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs @@ -16,14 +16,7 @@ pub(super) fn eq_words_unaligned(src: &[u64], other: &[u64], count: usize, shift debug_assert!(shift > 0 && shift < WORD_BITS); if count < SMALL_WORDS { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - } - return true; + return scalar::eq_words(src, other, count, shift); } #[cfg(all( @@ -52,213 +45,23 @@ pub(super) fn eq_words_unaligned(src: &[u64], other: &[u64], count: usize, shift } #[allow(unused)] - { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - } - true - } + scalar::eq_words(src, other, count, shift) } #[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, - _mm256_movemask_pd, _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, - _mm256_movemask_pd, _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn eq_words_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> bool { - // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; AVX2 is enabled by `#[target_feature]`. - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - // SAFETY: same as above - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 4 <= len { - // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. - // Pointers `src` and `other` are valid for `len+1` elements (caller guarantees extra word for window shift). - let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: same as above; load from `src[i+1]`. - let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; - // SAFETY: `_mm256_srl_epi64`, `_mm256_sll_epi64`, `_mm256_or_si256`, `_mm256_cmpeq_epi64`, and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. - let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; - // SAFETY: same as above - let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; - // SAFETY: same as above - let window = unsafe { _mm256_or_si256(lo, hi) }; - // SAFETY: AVX2 is enabled; pointer `other` is valid for `len` elements. - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - // SAFETY: `_mm256_cmpeq_epi64` is a pure register operation; AVX2 is enabled by `#[target_feature]`. - let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; - // SAFETY: same as above - if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { - return false; - } - i += 4; - } - while i < len { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - i += 1; - } - true - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use crate::WORD_BITS; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; +mod avx2; - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn eq_words_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> bool { - // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; SSE4.1 is enabled by `#[target_feature]`. - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - // SAFETY: same as above - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 2 <= len { - // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. - // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees). - let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: same as above; load from `src[i+1]`. - let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; - // SAFETY: `_mm_srl_epi64`, `_mm_sll_epi64`, `_mm_or_si128`, `_mm_cmpeq_epi64`, and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. - let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; - // SAFETY: same as above - let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; - // SAFETY: same as above - let window = unsafe { _mm_or_si128(lo, hi) }; - // SAFETY: SSE4.1 is enabled; pointer `other` is valid for `len` elements. - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - // SAFETY: `_mm_cmpeq_epi64` is a pure register operation; SSE4.1 is enabled by `#[target_feature]`. - let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; - // SAFETY: same as above - if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { - return false; - } - i += 2; - } - while i < len { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - i += 1; - } - true - } -} +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod sse41; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use crate::WORD_BITS; - use core::arch::aarch64::{ - vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, - }; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn eq_words_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> bool { - // SAFETY: `shift` is in [1, WORD_BITS); the caller guarantees this - // via the `shift == 0` fast-path in the entry point. - // Both shift vectors fit in i64: - // shift ∈ [1, 63] → -shift ∈ [-63, -1] - // WORD_BITS - shift ∈ [1, 63] - let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; - // SAFETY: same as above - let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; - - // Process 2 lanes (128 bits) per iteration. - let mut i = 0; - while i + 2 <= len { - // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. - // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees extra word for window shift). - // Load [src[i], src[i+1]] and [src[i+1], src[i+2]]. - let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - // SAFETY: same as above; load from `src[i+1]`. - let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; - - // Build the shifted 64-bit window for each lane: - // window[k] = (src[i+k] >> shift) | (src[i+k+1] << (64 - shift)) - // vshlq_u64 with a negative shift amount performs a logical right shift. - // SAFETY: `vshlq_u64`, `vorrq_u64`, `vceqq_u64`, and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. - let lo = unsafe { vshlq_u64(w0, neg_shift) }; - // SAFETY: same as above - let hi = unsafe { vshlq_u64(w1, pos_shift) }; - // SAFETY: same as above - let window = unsafe { vorrq_u64(lo, hi) }; - - // SAFETY: NEON is enabled; pointer `other` is valid for `len` elements. - let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - // SAFETY: `vceqq_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. - let cmp = unsafe { vceqq_u64(window, expected) }; - - // Each lane is all-ones on equality → vgetq_lane_u64 returns u64::MAX. - // SAFETY: `vgetq_lane_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - return false; - } - - i += 2; - } - - // Scalar tail for the last word (when len is odd). - while i < len { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; - } - i += 1; - } - - true - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core/avx2.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/avx2.rs new file mode 100644 index 0000000..fabea42 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/avx2.rs @@ -0,0 +1,57 @@ +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn eq_words_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> bool { + // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; AVX2 is enabled by `#[target_feature]`. + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: same as above + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 4 <= len { + // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. + // Pointers `src` and `other` are valid for `len+1` elements (caller guarantees extra word for window shift). + let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: same as above; load from `src[i+1]`. + let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; + // SAFETY: `_mm256_srl_epi64`, `_mm256_sll_epi64`, `_mm256_or_si256`, `_mm256_cmpeq_epi64`, and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. + let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + // SAFETY: same as above + let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + // SAFETY: same as above + let window = unsafe { _mm256_or_si256(lo, hi) }; + // SAFETY: AVX2 is enabled; pointer `other` is valid for `len` elements. + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: `_mm256_cmpeq_epi64` is a pure register operation; AVX2 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; + // SAFETY: same as above + if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { + return false; + } + i += 4; + } + while i < len { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + i += 1; + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core/neon.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/neon.rs new file mode 100644 index 0000000..390e51c --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/neon.rs @@ -0,0 +1,67 @@ +use crate::WORD_BITS; +use core::arch::aarch64::{ + vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, +}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn eq_words_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> bool { + // SAFETY: `shift` is in [1, WORD_BITS); the caller guarantees this + // via the `shift == 0` fast-path in the entry point. + // Both shift vectors fit in i64: + // shift ∈ [1, 63] → -shift ∈ [-63, -1] + // WORD_BITS - shift ∈ [1, 63] + let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + // SAFETY: same as above + let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; + + // Process 2 lanes (128 bits) per iteration. + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. + // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees extra word for window shift). + // Load [src[i], src[i+1]] and [src[i+1], src[i+2]]. + let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: same as above; load from `src[i+1]`. + let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; + + // Build the shifted 64-bit window for each lane: + // window[k] = (src[i+k] >> shift) | (src[i+k+1] << (64 - shift)) + // vshlq_u64 with a negative shift amount performs a logical right shift. + // SAFETY: `vshlq_u64`, `vorrq_u64`, `vceqq_u64`, and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. + let lo = unsafe { vshlq_u64(w0, neg_shift) }; + // SAFETY: same as above + let hi = unsafe { vshlq_u64(w1, pos_shift) }; + // SAFETY: same as above + let window = unsafe { vorrq_u64(lo, hi) }; + + // SAFETY: NEON is enabled; pointer `other` is valid for `len` elements. + let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + // SAFETY: `vceqq_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. + let cmp = unsafe { vceqq_u64(window, expected) }; + + // Each lane is all-ones on equality → vgetq_lane_u64 returns u64::MAX. + // SAFETY: `vgetq_lane_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + return false; + } + + i += 2; + } + + // Scalar tail for the last word (when len is odd). + while i < len { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + i += 1; + } + + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core/scalar.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/scalar.rs new file mode 100644 index 0000000..eff8f02 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/scalar.rs @@ -0,0 +1,12 @@ +use crate::WORD_BITS; + +pub(super) fn eq_words(src: &[u64], other: &[u64], count: usize, shift: usize) -> bool { + for i in 0..count { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + } + true +} diff --git a/src/traits/words_eq/funcs_for_eq_words_unaligned_core/sse41.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/sse41.rs new file mode 100644 index 0000000..1fa33e7 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/sse41.rs @@ -0,0 +1,57 @@ +use crate::WORD_BITS; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi64x, + _mm_sll_epi64, _mm_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi64x, + _mm_sll_epi64, _mm_srl_epi64, +}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn eq_words_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> bool { + // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; SSE4.1 is enabled by `#[target_feature]`. + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: same as above + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. + // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees). + let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: same as above; load from `src[i+1]`. + let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; + // SAFETY: `_mm_srl_epi64`, `_mm_sll_epi64`, `_mm_or_si128`, `_mm_cmpeq_epi64`, and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. + let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; + // SAFETY: same as above + let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; + // SAFETY: same as above + let window = unsafe { _mm_or_si128(lo, hi) }; + // SAFETY: SSE4.1 is enabled; pointer `other` is valid for `len` elements. + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: `_mm_cmpeq_epi64` is a pure register operation; SSE4.1 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; + // SAFETY: same as above + if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { + return false; + } + i += 2; + } + while i < len { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + i += 1; + } + true +} diff --git a/src/traits/words_find/funcs_for_contains_core.rs b/src/traits/words_find/funcs_for_contains_core.rs index dea6291..b57220d 100644 --- a/src/traits/words_find/funcs_for_contains_core.rs +++ b/src/traits/words_find/funcs_for_contains_core.rs @@ -40,7 +40,7 @@ where let word_limit = (max_word + 1).min(haystack.len()); if haystack.len() < SMALL_WORDS { - return scalar( + return scalar::scalar( haystack, needle_first, needle_mask, @@ -106,7 +106,7 @@ where } #[allow(unused)] - scalar( + scalar::scalar( haystack, needle_first, needle_mask, @@ -116,340 +116,20 @@ where ) } -// --------------------------------------------------------------------------- -// Scalar fallback -// --------------------------------------------------------------------------- - -/// Word-by-word scan: for each shift, check every word pair in -/// `[0, word_limit)` for a matching window. -fn scalar( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - word_limit: usize, - verify: &mut F, -) -> Option -where - F: FnMut(usize) -> bool, -{ - for shift in 0..WORD_BITS { - for i in 0..word_limit { - let pos = i * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - haystack[i] - } else { - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - None -} - -// --------------------------------------------------------------------------- -// SSE2 — 2 haystack words per iteration -// --------------------------------------------------------------------------- +#[allow(unused)] +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - - const LANES: usize = 2; - - /// SSE4.1 backend: loads 2 consecutive words, computes a sliding - /// window for the current shift, and compares against the broadcast - /// needle word. `movemask` extracts match lanes. - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn find_any( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - word_limit: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = unsafe { _mm_set1_epi64x(needle_first as i64) }; - let mask = unsafe { _mm_set1_epi64x(needle_mask as i64) }; - - for shift in 0..WORD_BITS { - let mut i = 0; - while i + LANES <= word_limit { - if i * WORD_BITS + shift > last_start { - break; - } - - let window = if shift == 0 { - unsafe { _mm_loadu_si128(haystack.as_ptr().add(i).cast::<__m128i>()) } - } else { - let src = haystack.as_ptr(); - let w01 = unsafe { _mm_loadu_si128(src.add(i).cast::<__m128i>()) }; - let w12 = unsafe { _mm_loadu_si128(src.add(i + 1).cast::<__m128i>()) }; - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let lo = unsafe { _mm_srl_epi64(w01, count_lo) }; - let hi = unsafe { _mm_sll_epi64(w12, count_hi) }; - unsafe { _mm_or_si128(lo, hi) } - }; - - let masked = unsafe { _mm_and_si128(window, mask) }; - let cmp = unsafe { _mm_cmpeq_epi64(masked, needle) }; - let hits = unsafe { _mm_movemask_epi8(cmp) } as u32; - - if hits & 0xff != 0 { - let pos = i * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if hits & 0xff00 != 0 { - let pos = (i + 1) * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - - i += LANES; - } - - for j in i..word_limit { - let pos = j * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - haystack[j] - } else { - let w0 = haystack[j]; - let w1 = haystack.get(j + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// AVX2 — 4 haystack words per iteration -// --------------------------------------------------------------------------- +mod sse41; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_and_si256, _mm256_cmpeq_epi64, - _mm256_loadu_si256, _mm256_movemask_pd, _mm256_or_si256, _mm256_set1_epi64x, - _mm256_sll_epi64, _mm256_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_and_si256, _mm256_cmpeq_epi64, - _mm256_loadu_si256, _mm256_movemask_pd, _mm256_or_si256, _mm256_set1_epi64x, - _mm256_sll_epi64, _mm256_srl_epi64, - }; - - const LANES: usize = 4; - - /// AVX2 backend: same as SSE2 but with 4-lane (256-bit) vectors. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn find_any( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - word_limit: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = unsafe { _mm256_set1_epi64x(needle_first as i64) }; - let mask = unsafe { _mm256_set1_epi64x(needle_mask as i64) }; - - for shift in 0..WORD_BITS { - let mut i = 0; - while i + LANES <= word_limit { - if i * WORD_BITS + shift > last_start { - break; - } - - let window = if shift == 0 { - unsafe { _mm256_loadu_si256(haystack.as_ptr().add(i).cast::<__m256i>()) } - } else { - let src = haystack.as_ptr(); - let w0 = unsafe { _mm256_loadu_si256(src.add(i).cast::<__m256i>()) }; - let w1 = unsafe { _mm256_loadu_si256(src.add(i + 1).cast::<__m256i>()) }; - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; - let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; - unsafe { _mm256_or_si256(lo, hi) } - }; - - let masked = unsafe { _mm256_and_si256(window, mask) }; - let cmp = unsafe { _mm256_cmpeq_epi64(masked, needle) }; - let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; - - if hits != 0 { - for k in 0..LANES { - if hits & (1 << k) != 0 { - let pos = (i + k) * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - - i += LANES; - } - - for j in i..word_limit { - let pos = j * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - haystack[j] - } else { - let w0 = haystack[j]; - let w1 = haystack.get(j + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// NEON — 2 haystack words per iteration -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::*; - - use core::arch::aarch64::{ - uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, - }; - - const LANES: usize = 2; - - /// NEON backend: 2-lane comparison for aarch64. Unaligned windows - /// fall back to scalar per-position computation. - #[target_feature(enable = "neon")] - pub(super) unsafe fn find_any( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - word_limit: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = unsafe { vdupq_n_u64(needle_first) }; - let mask = unsafe { vdupq_n_u64(needle_mask) }; - - for shift in 0..WORD_BITS { - let mut i = 0; - while i < word_limit { - if i * WORD_BITS + shift > last_start { - break; - } - - if shift == 0 { - let window = unsafe { vld1q_u64(haystack.as_ptr().add(i)) }; - let masked = unsafe { vandq_u64(window, mask) }; - let cmp = unsafe { vceqq_u64(masked, needle) }; - if unsafe { vgetq_lane_u64(cmp, 0) } != 0 { - let pos = i * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if unsafe { vgetq_lane_u64(cmp, 1) } != 0 { - let pos = (i + 1) * WORD_BITS + shift; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } else { - for k in 0..LANES { - let pos = (i + k) * WORD_BITS + shift; - if pos > last_start { - break; - } - let w0 = haystack[i + k]; - let w1 = haystack.get(i + k + 1).copied().unwrap_or(0); - let window = (w0 >> shift) | (w1 << (WORD_BITS - shift)); - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - - i += LANES; - } - - for j in i..word_limit { - let pos = j * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - haystack[j] - } else { - let w0 = haystack[j]; - let w1 = haystack.get(j + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_find/funcs_for_contains_core/avx2.rs b/src/traits/words_find/funcs_for_contains_core/avx2.rs new file mode 100644 index 0000000..c4fa0b6 --- /dev/null +++ b/src/traits/words_find/funcs_for_contains_core/avx2.rs @@ -0,0 +1,89 @@ +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, + _mm256_movemask_pd, _mm256_or_si256, _mm256_set1_epi64x, _mm256_sll_epi64, _mm256_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, + _mm256_movemask_pd, _mm256_or_si256, _mm256_set1_epi64x, _mm256_sll_epi64, _mm256_srl_epi64, +}; + +const LANES: usize = 4; + +/// AVX2 backend: same as SSE4.1 but with 4-lane (256-bit) vectors. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn find_any( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + word_limit: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = unsafe { _mm256_set1_epi64x(needle_first as i64) }; + let mask = unsafe { _mm256_set1_epi64x(needle_mask as i64) }; + + for shift in 0..WORD_BITS { + let mut i = 0; + while i + LANES <= word_limit { + if i * WORD_BITS + shift > last_start { + break; + } + + let window = if shift == 0 { + unsafe { _mm256_loadu_si256(haystack.as_ptr().add(i).cast::<__m256i>()) } + } else { + let src = haystack.as_ptr(); + let w0 = unsafe { _mm256_loadu_si256(src.add(i).cast::<__m256i>()) }; + let w1 = unsafe { _mm256_loadu_si256(src.add(i + 1).cast::<__m256i>()) }; + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + unsafe { _mm256_or_si256(lo, hi) } + }; + + let masked = unsafe { _mm256_and_si256(window, mask) }; + let cmp = unsafe { _mm256_cmpeq_epi64(masked, needle) }; + let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; + + if hits != 0 { + for k in 0..LANES { + if hits & (1 << k) != 0 { + let pos = (i + k) * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + + i += LANES; + } + + for j in i..word_limit { + let pos = j * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + haystack[j] + } else { + let w0 = haystack[j]; + let w1 = haystack.get(j + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_contains_core/neon.rs b/src/traits/words_find/funcs_for_contains_core/neon.rs new file mode 100644 index 0000000..629631a --- /dev/null +++ b/src/traits/words_find/funcs_for_contains_core/neon.rs @@ -0,0 +1,86 @@ +use super::*; + +use core::arch::aarch64::{ + uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, +}; + +const LANES: usize = 2; + +/// NEON backend: 2-lane comparison for aarch64. Unaligned windows +/// fall back to scalar per-position computation. +#[target_feature(enable = "neon")] +pub(super) unsafe fn find_any( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + word_limit: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = unsafe { vdupq_n_u64(needle_first) }; + let mask = unsafe { vdupq_n_u64(needle_mask) }; + + for shift in 0..WORD_BITS { + let mut i = 0; + while i < word_limit { + if i * WORD_BITS + shift > last_start { + break; + } + + if shift == 0 { + let window = unsafe { vld1q_u64(haystack.as_ptr().add(i)) }; + let masked = unsafe { vandq_u64(window, mask) }; + let cmp = unsafe { vceqq_u64(masked, needle) }; + if unsafe { vgetq_lane_u64(cmp, 0) } != 0 { + let pos = i * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if unsafe { vgetq_lane_u64(cmp, 1) } != 0 { + let pos = (i + 1) * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } else { + for k in 0..LANES { + let pos = (i + k) * WORD_BITS + shift; + if pos > last_start { + break; + } + let w0 = haystack[i + k]; + let w1 = haystack.get(i + k + 1).copied().unwrap_or(0); + let window = (w0 >> shift) | (w1 << (WORD_BITS - shift)); + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + + i += LANES; + } + + for j in i..word_limit { + let pos = j * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + haystack[j] + } else { + let w0 = haystack[j]; + let w1 = haystack.get(j + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_contains_core/scalar.rs b/src/traits/words_find/funcs_for_contains_core/scalar.rs new file mode 100644 index 0000000..b9307a8 --- /dev/null +++ b/src/traits/words_find/funcs_for_contains_core/scalar.rs @@ -0,0 +1,35 @@ +use crate::WORD_BITS; + +/// Word-by-word scan: for each shift, check every word pair in +/// `[0, word_limit)` for a matching window. +pub(super) fn scalar( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + word_limit: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + for shift in 0..WORD_BITS { + for i in 0..word_limit { + let pos = i * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + haystack[i] + } else { + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + None +} diff --git a/src/traits/words_find/funcs_for_contains_core/sse41.rs b/src/traits/words_find/funcs_for_contains_core/sse41.rs new file mode 100644 index 0000000..e3fda74 --- /dev/null +++ b/src/traits/words_find/funcs_for_contains_core/sse41.rs @@ -0,0 +1,93 @@ +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, + _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, + _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, +}; + +const LANES: usize = 2; + +/// SSE4.1 backend: loads 2 consecutive words, computes a sliding +/// window for the current shift, and compares against the broadcast +/// needle word. `movemask` extracts match lanes. +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn find_any( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + word_limit: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = unsafe { _mm_set1_epi64x(needle_first as i64) }; + let mask = unsafe { _mm_set1_epi64x(needle_mask as i64) }; + + for shift in 0..WORD_BITS { + let mut i = 0; + while i + LANES <= word_limit { + if i * WORD_BITS + shift > last_start { + break; + } + + let window = if shift == 0 { + unsafe { _mm_loadu_si128(haystack.as_ptr().add(i).cast::<__m128i>()) } + } else { + let src = haystack.as_ptr(); + let w01 = unsafe { _mm_loadu_si128(src.add(i).cast::<__m128i>()) }; + let w12 = unsafe { _mm_loadu_si128(src.add(i + 1).cast::<__m128i>()) }; + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let lo = unsafe { _mm_srl_epi64(w01, count_lo) }; + let hi = unsafe { _mm_sll_epi64(w12, count_hi) }; + unsafe { _mm_or_si128(lo, hi) } + }; + + let masked = unsafe { _mm_and_si128(window, mask) }; + let cmp = unsafe { _mm_cmpeq_epi64(masked, needle) }; + let hits = unsafe { _mm_movemask_epi8(cmp) } as u32; + + if hits & 0xff != 0 { + let pos = i * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if hits & 0xff00 != 0 { + let pos = (i + 1) * WORD_BITS + shift; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + + i += LANES; + } + + for j in i..word_limit { + let pos = j * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + haystack[j] + } else { + let w0 = haystack[j]; + let w1 = haystack.get(j + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs b/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs index edd03b1..50798d3 100644 --- a/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs @@ -1,6 +1,6 @@ //! Verify that `find_any_candidate` produces correct candidate positions //! against brute-force, implicitly validating whichever SIMD backend is -//! active (SSE2, AVX2, or NEON) against the scalar oracle. +//! active (SSE4.1, AVX2, or NEON) against the scalar oracle. use proptest::collection::vec; use proptest::prelude::*; diff --git a/src/traits/words_find/funcs_for_find_core.rs b/src/traits/words_find/funcs_for_find_core.rs index f2d3674..b1e70d4 100644 --- a/src/traits/words_find/funcs_for_find_core.rs +++ b/src/traits/words_find/funcs_for_find_core.rs @@ -26,7 +26,7 @@ where let needle_mask = low_mask(needle_bit_len.min(WORD_BITS)); let last_start = haystack_bit_len - needle_bit_len; if haystack.len() < SMALL_WORDS { - return scalar_find(haystack, needle_first, needle_mask, last_start, verify); + return scalar::scalar_find(haystack, needle_first, needle_mask, last_start, verify); } #[cfg(all( @@ -58,281 +58,23 @@ where } #[allow(unused)] - scalar_find(haystack, needle_first, needle_mask, last_start, verify) + scalar::scalar_find(haystack, needle_first, needle_mask, last_start, verify) } -// --------------------------------------------------------------------------- -// Scalar -// --------------------------------------------------------------------------- - -fn scalar_find( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, -) -> Option -where - F: FnMut(usize) -> bool, -{ - for i in 0..haystack.len() { - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - for shift in 0..WORD_BITS { - let pos = i * WORD_BITS + shift; - if pos > last_start { - break; - } - let window = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_first && verify(pos) { - return Some(pos); - } - } - } - None -} - -// --------------------------------------------------------------------------- -// SSE2 — same loop as scalar but checks 2 consecutive shifts at once -// --------------------------------------------------------------------------- +#[allow(unused)] +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, - _mm_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, - _mm_set1_epi64x, - }; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn find( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = _mm_set1_epi64x(needle_first as i64); - let mask = _mm_set1_epi64x(needle_mask as i64); - - for i in 0..haystack.len() { - let base = i * WORD_BITS; - if base > last_start { - break; - } - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - - let mut s = 0; - while s < WORD_BITS { - if base + s > last_start { - break; - } - // Build two consecutive windows manually, pack into __m128i. - let win0 = if s == 0 { - w0 - } else { - (w0 >> s) | (w1 << (WORD_BITS - s)) - }; - let win1 = if s + 1 >= WORD_BITS { - 0 - } else { - (w0 >> (s + 1)) | (w1 << (WORD_BITS - (s + 1))) - }; - let windows = unsafe { _mm_loadu_si128([win0, win1].as_ptr().cast::<__m128i>()) }; - let m = _mm_and_si128(windows, mask); - let c = unsafe { _mm_cmpeq_epi64(m, needle) }; - let hits = _mm_movemask_epi8(c) as u32; - - if hits & 0xff != 0 { - let pos = base + s; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if hits & 0xff00 != 0 { - let pos = base + s + 1; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - - s += 2; - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// AVX2 — checks 4 consecutive shifts at once -// --------------------------------------------------------------------------- +mod sse41; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, - _mm256_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, - _mm256_set1_epi64x, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn find( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = _mm256_set1_epi64x(needle_first as i64); - let mask = _mm256_set1_epi64x(needle_mask as i64); - - for i in 0..haystack.len() { - let base = i * WORD_BITS; - if base > last_start { - break; - } - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - - let mut s = 0; - while s < WORD_BITS { - if base + s > last_start { - break; - } - let end = WORD_BITS.min(s + 4); - let mut wins = [0u64; 4]; - for k in 0..(end - s) { - let shift = s + k; - wins[k] = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - } - let windows = unsafe { _mm256_loadu_si256(wins.as_ptr().cast::<__m256i>()) }; - let m = _mm256_and_si256(windows, mask); - let c = _mm256_cmpeq_epi64(m, needle); - let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(c)) } as u32; - if hits != 0 { - for k in 0..(end - s) { - if hits & (1 << k) != 0 { - let pos = base + s + k; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - - s += 4; - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// NEON -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::*; - - use core::arch::aarch64::{ - uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, - }; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn find( - haystack: &[u64], - needle_first: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = vdupq_n_u64(needle_first); - let mask = vdupq_n_u64(needle_mask); - - for i in 0..haystack.len() { - let base = i * WORD_BITS; - if base > last_start { - break; - } - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - - let mut s = 0; - while s < WORD_BITS { - if base + s > last_start { - break; - } - let end = WORD_BITS.min(s + 2); - let mut wins = [0u64; 2]; - for k in 0..(end - s) { - let shift = s + k; - wins[k] = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - } - let windows = unsafe { vld1q_u64(wins.as_ptr()) }; - let m = vandq_u64(windows, mask); - let c = vceqq_u64(m, needle); - if vgetq_lane_u64(c, 0) != 0 { - let pos = base + s; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if vgetq_lane_u64(c, 1) != 0 && s + 1 < end { - let pos = base + s + 1; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - - s += 2; - } - } - - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_find/funcs_for_find_core/avx2.rs b/src/traits/words_find/funcs_for_find_core/avx2.rs new file mode 100644 index 0000000..f929da4 --- /dev/null +++ b/src/traits/words_find/funcs_for_find_core/avx2.rs @@ -0,0 +1,75 @@ +// --------------------------------------------------------------------------- +// AVX2 — checks 4 consecutive shifts at once +// --------------------------------------------------------------------------- + +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_set1_epi64x, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_set1_epi64x, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn find( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = _mm256_set1_epi64x(needle_first as i64); + let mask = _mm256_set1_epi64x(needle_mask as i64); + + for i in 0..haystack.len() { + let base = i * WORD_BITS; + if base > last_start { + break; + } + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + + let mut s = 0; + while s < WORD_BITS { + if base + s > last_start { + break; + } + let end = WORD_BITS.min(s + 4); + let mut wins = [0u64; 4]; + for k in 0..(end - s) { + let shift = s + k; + wins[k] = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + } + let windows = unsafe { _mm256_loadu_si256(wins.as_ptr().cast::<__m256i>()) }; + let m = _mm256_and_si256(windows, mask); + let c = _mm256_cmpeq_epi64(m, needle); + let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(c)) } as u32; + if hits != 0 { + for k in 0..(end - s) { + if hits & (1 << k) != 0 { + let pos = base + s + k; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + + s += 4; + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_find_core/neon.rs b/src/traits/words_find/funcs_for_find_core/neon.rs new file mode 100644 index 0000000..5ebbef8 --- /dev/null +++ b/src/traits/words_find/funcs_for_find_core/neon.rs @@ -0,0 +1,69 @@ +// --------------------------------------------------------------------------- +// NEON +// --------------------------------------------------------------------------- + +use super::*; + +use core::arch::aarch64::{ + uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, +}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn find( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = vdupq_n_u64(needle_first); + let mask = vdupq_n_u64(needle_mask); + + for i in 0..haystack.len() { + let base = i * WORD_BITS; + if base > last_start { + break; + } + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + + let mut s = 0; + while s < WORD_BITS { + if base + s > last_start { + break; + } + let end = WORD_BITS.min(s + 2); + let mut wins = [0u64; 2]; + for k in 0..(end - s) { + let shift = s + k; + wins[k] = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + } + let windows = unsafe { vld1q_u64(wins.as_ptr()) }; + let m = vandq_u64(windows, mask); + let c = vceqq_u64(m, needle); + if vgetq_lane_u64(c, 0) != 0 { + let pos = base + s; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if vgetq_lane_u64(c, 1) != 0 && s + 1 < end { + let pos = base + s + 1; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + + s += 2; + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_find_core/scalar.rs b/src/traits/words_find/funcs_for_find_core/scalar.rs new file mode 100644 index 0000000..55d40fe --- /dev/null +++ b/src/traits/words_find/funcs_for_find_core/scalar.rs @@ -0,0 +1,32 @@ +use crate::WORD_BITS; + +pub(super) fn scalar_find( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + for i in 0..haystack.len() { + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + for shift in 0..WORD_BITS { + let pos = i * WORD_BITS + shift; + if pos > last_start { + break; + } + let window = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_first && verify(pos) { + return Some(pos); + } + } + } + None +} diff --git a/src/traits/words_find/funcs_for_find_core/sse41.rs b/src/traits/words_find/funcs_for_find_core/sse41.rs new file mode 100644 index 0000000..8c7fe03 --- /dev/null +++ b/src/traits/words_find/funcs_for_find_core/sse41.rs @@ -0,0 +1,77 @@ +// --------------------------------------------------------------------------- +// SSE4.1 — same loop as scalar but checks 2 consecutive shifts at once +// --------------------------------------------------------------------------- + +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, +}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn find( + haystack: &[u64], + needle_first: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = _mm_set1_epi64x(needle_first as i64); + let mask = _mm_set1_epi64x(needle_mask as i64); + + for i in 0..haystack.len() { + let base = i * WORD_BITS; + if base > last_start { + break; + } + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + + let mut s = 0; + while s < WORD_BITS { + if base + s > last_start { + break; + } + // Build two consecutive windows manually, pack into __m128i. + let win0 = if s == 0 { + w0 + } else { + (w0 >> s) | (w1 << (WORD_BITS - s)) + }; + let win1 = if s + 1 >= WORD_BITS { + 0 + } else { + (w0 >> (s + 1)) | (w1 << (WORD_BITS - (s + 1))) + }; + let windows = unsafe { _mm_loadu_si128([win0, win1].as_ptr().cast::<__m128i>()) }; + let m = _mm_and_si128(windows, mask); + let c = unsafe { _mm_cmpeq_epi64(m, needle) }; + let hits = _mm_movemask_epi8(c) as u32; + + if hits & 0xff != 0 { + let pos = base + s; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if hits & 0xff00 != 0 { + let pos = base + s + 1; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + + s += 2; + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_rfind_core.rs b/src/traits/words_find/funcs_for_rfind_core.rs index 68ecbe1..8abb87a 100644 --- a/src/traits/words_find/funcs_for_rfind_core.rs +++ b/src/traits/words_find/funcs_for_rfind_core.rs @@ -28,7 +28,7 @@ where let last_start = haystack_bit_len - needle_bit_len; if haystack.len() < SMALL_WORDS { - return scalar_rfind(haystack, needle_key, needle_mask, last_start, verify); + return scalar::scalar_rfind(haystack, needle_key, needle_mask, last_start, verify); } #[cfg(all( @@ -60,287 +60,23 @@ where } #[allow(unused)] - scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) + scalar::scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) } -// --------------------------------------------------------------------------- -// Scalar -// --------------------------------------------------------------------------- - -fn scalar_rfind( - haystack: &[u64], - needle_key: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, -) -> Option -where - F: FnMut(usize) -> bool, -{ - let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); - for i in (0..=start_word).rev() { - let base = i * WORD_BITS; - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - // Note: the SIMD backends compute max_shift differently — - // `WORD_BITS.min(last_start - base + 1)` — to process - // shifts in SIMD-sized chunks (2 or 4), relying on - // `pos <= last_start` to skip out-of-range positions. - let max_shift = (last_start - base).min(WORD_BITS - 1); - for shift in (0..=max_shift).rev() { - let pos = base + shift; - let window = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (window & needle_mask) == needle_key && verify(pos) { - return Some(pos); - } - } - } - None -} - -// --------------------------------------------------------------------------- -// SSE2 — 2 consecutive shifts at once, reverse -// --------------------------------------------------------------------------- +#[allow(unused)] +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, - _mm_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, - _mm_set1_epi64x, - }; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn rfind( - haystack: &[u64], - needle_key: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = _mm_set1_epi64x(needle_key as i64); - let mask = _mm_set1_epi64x(needle_mask as i64); - - let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); - for i in (0..=start_word).rev() { - let base = i * WORD_BITS; - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - let max_shift = WORD_BITS.min(last_start - base + 1); - - // Round up to a multiple of 2 so the SIMD loop - // processes shifts in 2-lane pairs. Out-of-range - // positions are guarded by `pos <= last_start`. - let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); - while s > 0 { - s -= 2; - - let win0 = if s == 0 { - w0 - } else { - (w0 >> s) | (w1 << (WORD_BITS - s)) - }; - let win1 = if s + 1 >= WORD_BITS { - 0 - } else { - (w0 >> (s + 1)) | (w1 << (WORD_BITS - (s + 1))) - }; - let windows = unsafe { _mm_loadu_si128([win0, win1].as_ptr().cast::<__m128i>()) }; - let m = _mm_and_si128(windows, mask); - let c = unsafe { _mm_cmpeq_epi64(m, needle) }; - let hits = _mm_movemask_epi8(c) as u32; - - // Check higher shift (lane 1) first for rightmost. - if hits & 0xff00 != 0 { - let pos = base + s + 1; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if hits & 0xff != 0 { - let pos = base + s; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// AVX2 — 4 consecutive shifts at once, reverse -// --------------------------------------------------------------------------- +mod sse41; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::*; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, - _mm256_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, - _mm256_set1_epi64x, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn rfind( - haystack: &[u64], - needle_key: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = _mm256_set1_epi64x(needle_key as i64); - let mask = _mm256_set1_epi64x(needle_mask as i64); - - let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); - for i in (0..=start_word).rev() { - let base = i * WORD_BITS; - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - let max_shift = WORD_BITS.min(last_start - base + 1); - - // Round up to a multiple of 4 so the SIMD loop - // processes shifts in 4-lane groups. Out-of-range - // positions are guarded by `pos <= last_start`. - let mut s = max_shift.next_multiple_of(4).min(WORD_BITS); - while s > 0 { - s -= 4; - - let end = WORD_BITS.min(s + 4); - let mut wins = [0u64; 4]; - for k in 0..(end - s) { - let shift = s + k; - wins[k] = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - } - - let windows = unsafe { _mm256_loadu_si256(wins.as_ptr().cast::<__m256i>()) }; - let m = _mm256_and_si256(windows, mask); - let c = unsafe { _mm256_cmpeq_epi64(m, needle) }; - let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(c)) } as u32; - - // Check higher shifts first for rightmost. - if hits != 0 { - for k in (0..(end - s)).rev() { - if hits & (1 << k) != 0 { - let pos = base + s + k; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - } - } - - None - } -} - -// --------------------------------------------------------------------------- -// NEON — 2 consecutive shifts at once, reverse -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::*; - - use core::arch::aarch64::{ - uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, - }; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn rfind( - haystack: &[u64], - needle_key: u64, - needle_mask: u64, - last_start: usize, - verify: &mut F, - ) -> Option - where - F: FnMut(usize) -> bool, - { - let needle = vdupq_n_u64(needle_key); - let mask = vdupq_n_u64(needle_mask); - - let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); - for i in (0..=start_word).rev() { - let base = i * WORD_BITS; - let w0 = haystack[i]; - let w1 = haystack.get(i + 1).copied().unwrap_or(0); - let max_shift = WORD_BITS.min(last_start - base + 1); - - // Round up to a multiple of 2 so the SIMD loop - // processes shifts in 2-lane pairs. Out-of-range - // positions are guarded by `pos <= last_start`. - let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); - while s > 0 { - s -= 2; - - let end = WORD_BITS.min(s + 2); - let mut wins = [0u64; 2]; - for k in 0..(end - s) { - let shift = s + k; - wins[k] = if shift == 0 { - w0 - } else { - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - } - let windows = unsafe { vld1q_u64(wins.as_ptr()) }; - let m = vandq_u64(windows, mask); - let c = vceqq_u64(m, needle); - - // Check higher shift (lane 1) first for rightmost. - if vgetq_lane_u64(c, 1) != 0 { - let pos = base + s + 1; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - if vgetq_lane_u64(c, 0) != 0 { - let pos = base + s; - if pos <= last_start && verify(pos) { - return Some(pos); - } - } - } - } - - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_find/funcs_for_rfind_core/avx2.rs b/src/traits/words_find/funcs_for_rfind_core/avx2.rs new file mode 100644 index 0000000..d7c894b --- /dev/null +++ b/src/traits/words_find/funcs_for_rfind_core/avx2.rs @@ -0,0 +1,77 @@ +// --------------------------------------------------------------------------- +// AVX2 — 4 consecutive shifts at once, reverse +// --------------------------------------------------------------------------- + +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_set1_epi64x, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_and_si256, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_set1_epi64x, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn rfind( + haystack: &[u64], + needle_key: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = _mm256_set1_epi64x(needle_key as i64); + let mask = _mm256_set1_epi64x(needle_mask as i64); + + let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); + for i in (0..=start_word).rev() { + let base = i * WORD_BITS; + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + let max_shift = WORD_BITS.min(last_start - base + 1); + + // Round up to a multiple of 4 so the SIMD loop + // processes shifts in 4-lane groups. Out-of-range + // positions are guarded by `pos <= last_start`. + let mut s = max_shift.next_multiple_of(4).min(WORD_BITS); + while s > 0 { + s -= 4; + + let end = WORD_BITS.min(s + 4); + let mut wins = [0u64; 4]; + for k in 0..(end - s) { + let shift = s + k; + wins[k] = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + } + + let windows = unsafe { _mm256_loadu_si256(wins.as_ptr().cast::<__m256i>()) }; + let m = _mm256_and_si256(windows, mask); + let c = _mm256_cmpeq_epi64(m, needle); + let hits = unsafe { _mm256_movemask_pd(core::mem::transmute(c)) } as u32; + + // Check higher shifts first for rightmost. + if hits != 0 { + for k in (0..(end - s)).rev() { + if hits & (1 << k) != 0 { + let pos = base + s + k; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_rfind_core/neon.rs b/src/traits/words_find/funcs_for_rfind_core/neon.rs new file mode 100644 index 0000000..9c6991a --- /dev/null +++ b/src/traits/words_find/funcs_for_rfind_core/neon.rs @@ -0,0 +1,70 @@ +// --------------------------------------------------------------------------- +// NEON — 2 consecutive shifts at once, reverse +// --------------------------------------------------------------------------- + +use super::*; + +use core::arch::aarch64::{ + uint64x2_t, vandq_u64, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64, +}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn rfind( + haystack: &[u64], + needle_key: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = vdupq_n_u64(needle_key); + let mask = vdupq_n_u64(needle_mask); + + let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); + for i in (0..=start_word).rev() { + let base = i * WORD_BITS; + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + let max_shift = WORD_BITS.min(last_start - base + 1); + + // Round up to a multiple of 2 so the SIMD loop + // processes shifts in 2-lane pairs. Out-of-range + // positions are guarded by `pos <= last_start`. + let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); + while s > 0 { + s -= 2; + + let end = WORD_BITS.min(s + 2); + let mut wins = [0u64; 2]; + for k in 0..(end - s) { + let shift = s + k; + wins[k] = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + } + let windows = unsafe { vld1q_u64(wins.as_ptr()) }; + let m = vandq_u64(windows, mask); + let c = vceqq_u64(m, needle); + + // Check higher shift (lane 1) first for rightmost. + if vgetq_lane_u64(c, 1) != 0 { + let pos = base + s + 1; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if vgetq_lane_u64(c, 0) != 0 { + let pos = base + s; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_rfind_core/scalar.rs b/src/traits/words_find/funcs_for_rfind_core/scalar.rs new file mode 100644 index 0000000..0b60ee2 --- /dev/null +++ b/src/traits/words_find/funcs_for_rfind_core/scalar.rs @@ -0,0 +1,36 @@ +use crate::WORD_BITS; + +pub(super) fn scalar_rfind( + haystack: &[u64], + needle_key: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); + for i in (0..=start_word).rev() { + let base = i * WORD_BITS; + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + // Note: the SIMD backends compute max_shift differently — + // `WORD_BITS.min(last_start - base + 1)` — to process + // shifts in SIMD-sized chunks (2 or 4), relying on + // `pos <= last_start` to skip out-of-range positions. + let max_shift = (last_start - base).min(WORD_BITS - 1); + for shift in (0..=max_shift).rev() { + let pos = base + shift; + let window = if shift == 0 { + w0 + } else { + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (window & needle_mask) == needle_key && verify(pos) { + return Some(pos); + } + } + } + None +} diff --git a/src/traits/words_find/funcs_for_rfind_core/sse41.rs b/src/traits/words_find/funcs_for_rfind_core/sse41.rs new file mode 100644 index 0000000..ff09522 --- /dev/null +++ b/src/traits/words_find/funcs_for_rfind_core/sse41.rs @@ -0,0 +1,76 @@ +// --------------------------------------------------------------------------- +// SSE4.1 — 2 consecutive shifts at once, reverse +// --------------------------------------------------------------------------- + +use super::*; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_and_si128, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, +}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn rfind( + haystack: &[u64], + needle_key: u64, + needle_mask: u64, + last_start: usize, + verify: &mut F, +) -> Option +where + F: FnMut(usize) -> bool, +{ + let needle = _mm_set1_epi64x(needle_key as i64); + let mask = _mm_set1_epi64x(needle_mask as i64); + + let start_word = (last_start / WORD_BITS).min(haystack.len().saturating_sub(1)); + for i in (0..=start_word).rev() { + let base = i * WORD_BITS; + let w0 = haystack[i]; + let w1 = haystack.get(i + 1).copied().unwrap_or(0); + let max_shift = WORD_BITS.min(last_start - base + 1); + + // Round up to a multiple of 2 so the SIMD loop + // processes shifts in 2-lane pairs. Out-of-range + // positions are guarded by `pos <= last_start`. + let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); + while s > 0 { + s -= 2; + + let win0 = if s == 0 { + w0 + } else { + (w0 >> s) | (w1 << (WORD_BITS - s)) + }; + let win1 = if s + 1 >= WORD_BITS { + 0 + } else { + (w0 >> (s + 1)) | (w1 << (WORD_BITS - (s + 1))) + }; + let windows = unsafe { _mm_loadu_si128([win0, win1].as_ptr().cast::<__m128i>()) }; + let m = _mm_and_si128(windows, mask); + let c = unsafe { _mm_cmpeq_epi64(m, needle) }; + let hits = _mm_movemask_epi8(c) as u32; + + // Check higher shift (lane 1) first for rightmost. + if hits & 0xff00 != 0 { + let pos = base + s + 1; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + if hits & 0xff != 0 { + let pos = base + s; + if pos <= last_start && verify(pos) { + return Some(pos); + } + } + } + } + + None +} diff --git a/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs b/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs index 7b84a25..c4b7079 100644 --- a/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs @@ -1,6 +1,6 @@ //! Verify that `find_last_word` produces correct rightmost matches //! against brute-force, implicitly validating whichever SIMD backend is -//! active (SSE2, AVX2, or NEON) against the scalar oracle. +//! active (SSE4.1, AVX2, or NEON) against the scalar oracle. use proptest::collection::vec; use proptest::prelude::*; diff --git a/src/traits/words_ord.rs b/src/traits/words_ord.rs index 97e7bf9..8863a09 100644 --- a/src/traits/words_ord.rs +++ b/src/traits/words_ord.rs @@ -2,25 +2,25 @@ use core::cmp::Ordering; /// Word-level lexicographic comparison on `[u64]` backing storage. /// -/// Compares words from index 0 upward (LSB-first bit order within each word). -/// Returns `Some(Ordering)` at the first differing word, or `None` when all -/// `count` words are identical. +/// Compares `full_words` logical haystack words with `needle` from index 0 +/// upward (LSB-first bit order within each word). Returns `Some(Ordering)` at +/// the first differing word, or `None` when all logical words are identical. /// -/// `other` must be word-aligned. `self` may have an intra-word `offset` -/// (0 = word-aligned), in which case each logical word is reconstructed as -/// a shifted window `(self[i] >> shift) | (self[i+1] << (64-shift))`. +/// `needle` is word-aligned. `self` may have a non-zero physical start shift, +/// in which case each logical word is reconstructed as a shifted window +/// `(self[i] >> shift) | (self[i+1] << (64-shift))`. /// /// Both paths dispatch to SIMD backends when available: /// - AVX2 (x86/x86_64, 4×u64 per iteration) -/// - SSE2 (x86/x86_64, 2×u64 per iteration) +/// - SSE4.1 (x86/x86_64, 2×u64 per iteration) /// - NEON (aarch64, 2×u64 per iteration) /// -/// Short inputs fall back to scalar in all backends. +/// Short inputs fall back to scalar before SIMD dispatch. pub(crate) trait WordsOrd { - /// `self` is pre-trimmed haystack `words[base..]`. + /// `self` is the pre-trimmed haystack slice `words[base..]`. /// `needle` is always word-aligned (pre-trimmed by the caller). /// `full_words` is the number of complete u64 words to compare. - /// `haystack_shift` is the intra-word offset within the first word. + /// `haystack_shift` is the physical start modulo `WORD_BITS`. /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is /// guaranteed and the aligned backend is used unconditionally. /// When it is `false`, no alignment guarantee is made. diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs index d27def7..665f996 100644 --- a/src/traits/words_ord/funcs_for_cmp_aligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs @@ -1,7 +1,6 @@ use core::cmp::Ordering; use crate::SMALL_WORDS; -use crate::traits::WordOrd; /// Returns `Some(Ordering)` if the first `count` aligned words of `src` /// and `other` differ, otherwise `None` (all equal). @@ -11,7 +10,7 @@ use crate::traits::WordOrd; #[inline] pub(super) fn cmp_aligned_words(src: &[u64], other: &[u64], count: usize) -> Option { if count < SMALL_WORDS { - return scalar_cmp_aligned(src, other, count); + return scalar::cmp_aligned(src, other, count); } #[cfg(all( @@ -43,18 +42,11 @@ pub(super) fn cmp_aligned_words(src: &[u64], other: &[u64], count: usize) -> Opt } #[allow(unreachable_code)] - scalar_cmp_aligned(src, other, count) + scalar::cmp_aligned(src, other, count) } -#[inline] -fn scalar_cmp_aligned(src: &[u64], other: &[u64], count: usize) -> Option { - for i in 0..count { - if src[i] != other[i] { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - } - None -} +#[allow(unused)] +mod scalar; // --------------------------------------------------------------------------- // AVX2 — 4 × u64 per iteration @@ -62,81 +54,15 @@ fn scalar_cmp_aligned(src: &[u64], other: &[u64], count: usize) -> Option Option { - let mut i = 0; - while i + 4 <= len { - let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; - let mask = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; - if mask != 0b1111 { - let lane = mask.trailing_ones() as usize; - return Some(WordOrd::bitwise_cmp(src[i + lane], other[i + lane])); - } - i += 4; - } - while i < len { - if src[i] != other[i] { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - i += 1; - } - None - } -} +mod avx2; // --------------------------------------------------------------------------- -// SSE2 — 2 × u64 per iteration +// SSE4.1 — 2 × u64 per iteration // --------------------------------------------------------------------------- #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use core::cmp::Ordering; - - use crate::traits::WordOrd; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { - let mut i = 0; - while i + 2 <= len { - let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; - let mask = unsafe { _mm_movemask_epi8(cmp) } as u32; - if mask != 0xFFFF { - // Each lane is 8 bytes → 8 high bits. The first zero byte - // tells us which lane differs. - let lane = mask.trailing_ones() as usize / 8; - return Some(WordOrd::bitwise_cmp(src[i + lane], other[i + lane])); - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - i += 1; - } - None - } -} +mod sse41; // --------------------------------------------------------------------------- // NEON — 2 × u64 per iteration @@ -144,36 +70,7 @@ mod sse41 { #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use core::cmp::Ordering; - - use crate::traits::WordOrd; - use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { - let mut i = 0; - while i + 2 <= len { - let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - let cmp = unsafe { vceqq_u64(a, b) }; - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - return Some(WordOrd::bitwise_cmp(src[i + 1], other[i + 1])); - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return Some(WordOrd::bitwise_cmp(src[i], other[i])); - } - i += 1; - } - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core/avx2.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core/avx2.rs new file mode 100644 index 0000000..4815fff --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core/avx2.rs @@ -0,0 +1,31 @@ +use core::cmp::Ordering; + +use crate::traits::WordOrd; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { + let mut i = 0; + while i + 4 <= len { + let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; + let mask = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; + if mask != 0b1111 { + let lane = mask.trailing_ones() as usize; + return Some(WordOrd::bitwise_cmp(src[i + lane], other[i + lane])); + } + i += 4; + } + while i < len { + if src[i] != other[i] { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core/neon.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core/neon.rs new file mode 100644 index 0000000..e71bee1 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core/neon.rs @@ -0,0 +1,28 @@ +use core::cmp::Ordering; + +use crate::traits::WordOrd; +use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { + let mut i = 0; + while i + 2 <= len { + let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + let cmp = unsafe { vceqq_u64(a, b) }; + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + return Some(WordOrd::bitwise_cmp(src[i + 1], other[i + 1])); + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core/scalar.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core/scalar.rs new file mode 100644 index 0000000..28cb150 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core/scalar.rs @@ -0,0 +1,13 @@ +use core::cmp::Ordering; + +use crate::traits::WordOrd; + +#[inline] +pub(super) fn cmp_aligned(src: &[u64], other: &[u64], count: usize) -> Option { + for i in 0..count { + if src[i] != other[i] { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_aligned_core/sse41.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core/sse41.rs new file mode 100644 index 0000000..bc52aa4 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core/sse41.rs @@ -0,0 +1,33 @@ +use core::cmp::Ordering; + +use crate::traits::WordOrd; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn cmp_aligned(src: &[u64], other: &[u64], len: usize) -> Option { + let mut i = 0; + while i + 2 <= len { + let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; + let mask = unsafe { _mm_movemask_epi8(cmp) } as u32; + if mask != 0xFFFF { + // Each lane is 8 bytes → 8 high bits. The first zero byte + // tells us which lane differs. + let lane = mask.trailing_ones() as usize / 8; + return Some(WordOrd::bitwise_cmp(src[i + lane], other[i + lane])); + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return Some(WordOrd::bitwise_cmp(src[i], other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs index cbb6fa4..2563f7c 100644 --- a/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core.rs @@ -6,7 +6,6 @@ use core::cmp::Ordering; -use crate::traits::WordOrd; use crate::{SMALL_WORDS, WORD_BITS}; /// Returns `Some(Ordering)` at the first differing word, or `None` when all @@ -21,7 +20,7 @@ pub(super) fn cmp_unaligned_words( debug_assert!(shift > 0 && shift < WORD_BITS); if count < SMALL_WORDS { - return scalar_cmp_unaligned(src, other, count, shift); + return scalar::cmp_unaligned(src, other, count, shift); } #[cfg(all( @@ -53,203 +52,23 @@ pub(super) fn cmp_unaligned_words( } #[allow(unreachable_code)] - scalar_cmp_unaligned(src, other, count, shift) + scalar::cmp_unaligned(src, other, count, shift) } -#[inline] -fn scalar_cmp_unaligned( - src: &[u64], - other: &[u64], - count: usize, - shift: usize, -) -> Option { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - let window = (w0 >> shift) | (w1 << (WORD_BITS - shift)); - if window != other[i] { - return Some(WordOrd::bitwise_cmp(window, other[i])); - } - } - None -} - -// --------------------------------------------------------------------------- -// AVX2 — 4 × u64 per iteration -// --------------------------------------------------------------------------- +#[allow(unused)] +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use core::cmp::Ordering; - - use crate::WORD_BITS; - use crate::traits::WordOrd; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, - _mm256_movemask_pd, _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, - _mm256_movemask_pd, _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, - }; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn cmp_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> Option { - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 4 <= len { - // Load src[i..i+4] and src[i+1..i+5]. - let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; - let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; - let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; - let window = unsafe { _mm256_or_si256(lo, hi) }; - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; - let mask = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; - if mask != 0b1111 { - let lane = mask.trailing_ones() as usize; - let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); - return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); - } - i += 4; - } - while i < len { - let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - if sw != other[i] { - return Some(WordOrd::bitwise_cmp(sw, other[i])); - } - i += 1; - } - None - } -} - -// --------------------------------------------------------------------------- -// SSE2 — 2 × u64 per iteration -// --------------------------------------------------------------------------- +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - use core::cmp::Ordering; - - use crate::WORD_BITS; - use crate::traits::WordOrd; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, - _mm_set1_epi64x, _mm_sll_epi64, _mm_srl_epi64, - }; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn cmp_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> Option { - let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; - let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 2 <= len { - // Load src[i..i+2] and src[i+1..i+3]. - let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; - let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; - let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; - let window = unsafe { _mm_or_si128(lo, hi) }; - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; - let mask = unsafe { _mm_movemask_epi8(cmp) } as u32; - if mask != 0xFFFF { - let lane = mask.trailing_ones() as usize / 8; - let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); - return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); - } - i += 2; - } - while i < len { - let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - if sw != other[i] { - return Some(WordOrd::bitwise_cmp(sw, other[i])); - } - i += 1; - } - None - } -} - -// --------------------------------------------------------------------------- -// NEON — 2 × u64 per iteration -// --------------------------------------------------------------------------- +mod sse41; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use core::cmp::Ordering; - - use crate::WORD_BITS; - use crate::traits::WordOrd; - - use core::arch::aarch64::{ - vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, - }; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn cmp_unaligned( - src: &[u64], - other: &[u64], - len: usize, - shift: usize, - ) -> Option { - let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; - let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; - let mut i = 0; - while i + 2 <= len { - let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; - let lo = unsafe { vshlq_u64(w0, neg_shift) }; - let hi = unsafe { vshlq_u64(w1, pos_shift) }; - let window = unsafe { vorrq_u64(lo, hi) }; - let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - let cmp = unsafe { vceqq_u64(window, expected) }; - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { - let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - return Some(WordOrd::bitwise_cmp(sw, other[i])); - } - if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - let sw = (src[i + 1] >> shift) | (src[i + 2] << (WORD_BITS - shift)); - return Some(WordOrd::bitwise_cmp(sw, other[i + 1])); - } - i += 2; - } - while i < len { - let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - if sw != other[i] { - return Some(WordOrd::bitwise_cmp(sw, other[i])); - } - i += 1; - } - None - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core/avx2.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/avx2.rs new file mode 100644 index 0000000..5f6e39d --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core/avx2.rs @@ -0,0 +1,52 @@ +use core::cmp::Ordering; + +use crate::WORD_BITS; +use crate::traits::WordOrd; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_set1_epi64x, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd, + _mm256_or_si256, _mm256_sll_epi64, _mm256_srl_epi64, +}; + +#[target_feature(enable = "avx2")] +pub(super) unsafe fn cmp_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> Option { + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 4 <= len { + // Load src[i..i+4] and src[i+1..i+5]. + let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; + let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + let window = unsafe { _mm256_or_si256(lo, hi) }; + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; + let mask = unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32; + if mask != 0b1111 { + let lane = mask.trailing_ones() as usize; + let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); + return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); + } + i += 4; + } + while i < len { + let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + if sw != other[i] { + return Some(WordOrd::bitwise_cmp(sw, other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core/neon.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/neon.rs new file mode 100644 index 0000000..b4b3a91 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core/neon.rs @@ -0,0 +1,46 @@ +use core::cmp::Ordering; + +use crate::WORD_BITS; +use crate::traits::WordOrd; + +use core::arch::aarch64::{ + vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, +}; + +#[target_feature(enable = "neon")] +pub(super) unsafe fn cmp_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> Option { + let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 2 <= len { + let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; + let lo = unsafe { vshlq_u64(w0, neg_shift) }; + let hi = unsafe { vshlq_u64(w1, pos_shift) }; + let window = unsafe { vorrq_u64(lo, hi) }; + let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + let cmp = unsafe { vceqq_u64(window, expected) }; + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { + let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + return Some(WordOrd::bitwise_cmp(sw, other[i])); + } + if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + let sw = (src[i + 1] >> shift) | (src[i + 2] << (WORD_BITS - shift)); + return Some(WordOrd::bitwise_cmp(sw, other[i + 1])); + } + i += 2; + } + while i < len { + let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + if sw != other[i] { + return Some(WordOrd::bitwise_cmp(sw, other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core/scalar.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/scalar.rs new file mode 100644 index 0000000..e4aa31c --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core/scalar.rs @@ -0,0 +1,22 @@ +use core::cmp::Ordering; + +use crate::WORD_BITS; +use crate::traits::WordOrd; + +#[inline] +pub(super) fn cmp_unaligned( + src: &[u64], + other: &[u64], + count: usize, + shift: usize, +) -> Option { + for i in 0..count { + let w0 = src[i]; + let w1 = src[i + 1]; + let window = (w0 >> shift) | (w1 << (WORD_BITS - shift)); + if window != other[i] { + return Some(WordOrd::bitwise_cmp(window, other[i])); + } + } + None +} diff --git a/src/traits/words_ord/funcs_for_cmp_unaligned_core/sse41.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/sse41.rs new file mode 100644 index 0000000..e56e920 --- /dev/null +++ b/src/traits/words_ord/funcs_for_cmp_unaligned_core/sse41.rs @@ -0,0 +1,52 @@ +use core::cmp::Ordering; + +use crate::WORD_BITS; +use crate::traits::WordOrd; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi64x, + _mm_sll_epi64, _mm_srl_epi64, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi64x, + _mm_sll_epi64, _mm_srl_epi64, +}; + +#[target_feature(enable = "sse4.1")] +pub(super) unsafe fn cmp_unaligned( + src: &[u64], + other: &[u64], + len: usize, + shift: usize, +) -> Option { + let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; + let mut i = 0; + while i + 2 <= len { + // Load src[i..i+2] and src[i+1..i+3]. + let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; + let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; + let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; + let window = unsafe { _mm_or_si128(lo, hi) }; + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; + let mask = unsafe { _mm_movemask_epi8(cmp) } as u32; + if mask != 0xFFFF { + let lane = mask.trailing_ones() as usize / 8; + let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); + return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); + } + i += 2; + } + while i < len { + let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + if sw != other[i] { + return Some(WordOrd::bitwise_cmp(sw, other[i])); + } + i += 1; + } + None +} diff --git a/src/traits/words_scan/funcs_for_count_ones.rs b/src/traits/words_scan/funcs_for_count_ones.rs index e23ac4f..c9ee69e 100644 --- a/src/traits/words_scan/funcs_for_count_ones.rs +++ b/src/traits/words_scan/funcs_for_count_ones.rs @@ -44,8 +44,8 @@ fn count_full_words(words: &[u64]) -> usize { /// - `src` must be valid for reads of `len` initialized `u64` values. #[inline] unsafe fn dispatch(src: *const u64, len: usize) -> usize { - // Small inputs: skip SIMD setup overhead, go straight to scalar popcnt. - // Threshold equals the backend's LANES count. + // Small inputs: skip SIMD setup overhead, go straight to scalar popcount. + // The threshold is selected by the active target-feature configuration. #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), @@ -86,243 +86,19 @@ unsafe fn dispatch(src: *const u64, len: usize) -> usize { } #[allow(unused)] -mod scalar { - /// Scalar backend for counting set bits in `src[0..len]`. - /// - /// # Safety - /// - /// - `src` must be valid for reads of `len` initialized `u64` values. - #[inline] - pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { - let mut count = 0usize; - - for i in 0..len { - // SAFETY: - // - `i < len`. - // - Pointer validity is guaranteed by the caller. - unsafe { - count += src.add(i).read().count_ones() as usize; - } - } - - count - } -} +mod scalar; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_add_epi8, _mm256_add_epi64, _mm256_and_si256, _mm256_loadu_si256, - _mm256_sad_epu8, _mm256_set1_epi8, _mm256_setr_epi8, _mm256_setzero_si256, - _mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_storeu_si256, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_add_epi8, _mm256_add_epi64, _mm256_and_si256, _mm256_loadu_si256, - _mm256_sad_epu8, _mm256_set1_epi8, _mm256_setr_epi8, _mm256_setzero_si256, - _mm256_shuffle_epi8, _mm256_srli_epi16, _mm256_storeu_si256, - }; - - const LANES: usize = 4; - - /// AVX2 backend for counting set bits in `src[0..len]`. - /// - /// Uses nibble lookup with `vpshufb`, then sums byte counts with `vpsadbw`. - /// - /// # Safety - /// - /// - Caller must only call this when AVX2 is available. - /// - `src` must be valid for reads of `len` initialized `u64` values. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { - let chunks = len / LANES; - - // SAFETY: - // - These constructors require AVX2 to be available. - // - This function is compiled with `target_feature = "avx2"`. - let lookup = _mm256_setr_epi8( - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, // - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, - ); - let low_mask = _mm256_set1_epi8(0x0F); - let zero = _mm256_setzero_si256(); - - // Accumulate popcounts in a SIMD register to avoid per-chunk - // store-to-memory round-trips. Reduced to scalar only once at the end. - let mut acc = zero; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm256_loadu_si256` permits unaligned reads. - // - `src` validity is guaranteed by the caller. - unsafe { - let bytes = _mm256_loadu_si256(src.add(offset).cast::<__m256i>()); - - let low = _mm256_and_si256(bytes, low_mask); - let high = _mm256_and_si256(_mm256_srli_epi16(bytes, 4), low_mask); - - let low_counts = _mm256_shuffle_epi8(lookup, low); - let high_counts = _mm256_shuffle_epi8(lookup, high); - let byte_counts = _mm256_add_epi8(low_counts, high_counts); - - let sums = _mm256_sad_epu8(byte_counts, zero); - acc = _mm256_add_epi64(acc, sums); - } - } - - let mut lane_sums = [0u64; LANES]; - // SAFETY: `lane_sums` has sufficient space for 4 u64 values. - unsafe { _mm256_storeu_si256(lane_sums.as_mut_ptr().cast::<__m256i>(), acc) }; - let mut count = lane_sums.iter().map(|&sum| sum as usize).sum::(); - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity is guaranteed by the caller. - count += unsafe { scalar::count_words(src.add(done), len - done) }; - count - } -} +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod ssse3 { - use super::scalar; - - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_add_epi8, _mm_add_epi64, _mm_and_si128, _mm_loadu_si128, _mm_sad_epu8, - _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, - _mm_storeu_si128, - }; - - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_add_epi8, _mm_add_epi64, _mm_and_si128, _mm_loadu_si128, _mm_sad_epu8, - _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, - _mm_storeu_si128, - }; - - const LANES: usize = 2; - - /// SSSE3 backend for counting set bits in `src[0..len]`. - /// - /// Uses nibble lookup with `pshufb`, then sums byte counts with `psadbw`. - /// - /// # Safety - /// - /// - Caller must only call this when SSSE3 is available. - /// - `src` must be valid for reads of `len` initialized `u64` values. - #[target_feature(enable = "ssse3")] - pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { - let chunks = len / LANES; - - let lookup = _mm_setr_epi8( - 0, 1, 1, 2, 1, 2, 2, 3, // - 1, 2, 2, 3, 2, 3, 3, 4, - ); - let low_mask = _mm_set1_epi8(0x0F); - let zero = _mm_setzero_si128(); - - // Accumulate popcounts in a SIMD register to avoid per-chunk - // store-to-memory round-trips. Reduced to scalar only once at the end. - let mut acc = zero; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `_mm_loadu_si128` permits unaligned reads. - // - `src` validity is guaranteed by the caller. - unsafe { - let bytes = _mm_loadu_si128(src.add(offset).cast::<__m128i>()); - - let low = _mm_and_si128(bytes, low_mask); - let high = _mm_and_si128(_mm_srli_epi16(bytes, 4), low_mask); - - let low_counts = _mm_shuffle_epi8(lookup, low); - let high_counts = _mm_shuffle_epi8(lookup, high); - let byte_counts = _mm_add_epi8(low_counts, high_counts); - - let sums = _mm_sad_epu8(byte_counts, zero); - acc = _mm_add_epi64(acc, sums); - } - } - - let mut lane_sums = [0u64; LANES]; - // SAFETY: `lane_sums` has sufficient space for 2 u64 values. - unsafe { _mm_storeu_si128(lane_sums.as_mut_ptr().cast::<__m128i>(), acc) }; - let mut count = lane_sums.iter().map(|&sum| sum as usize).sum::(); - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity is guaranteed by the caller. - count += unsafe { scalar::count_words(src.add(done), len - done) }; - count - } -} +mod ssse3; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use super::scalar; - - use core::arch::aarch64::{vaddvq_u8, vcntq_u8, vld1q_u64, vreinterpretq_u8_u64}; - - const LANES: usize = 2; - - /// NEON backend for counting set bits in `src[0..len]`. - /// - /// Counts bits per byte with `vcntq_u8`, then horizontally sums byte counts. - /// - /// # Safety - /// - /// - Caller must only call this when NEON is available. - /// - `src` must be valid for reads of `len` initialized `u64` values. - #[target_feature(enable = "neon")] - pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { - let chunks = len / LANES; - let mut count = 0usize; - - for chunk in 0..chunks { - let offset = chunk * LANES; - - // SAFETY: - // - `offset + LANES <= len`. - // - `vld1q_u64` reads exactly 2 u64 values. - // - `src` validity is guaranteed by the caller. - unsafe { - let words = vld1q_u64(src.add(offset)); - let bytes = vreinterpretq_u8_u64(words); - let byte_counts = vcntq_u8(bytes); - - count += vaddvq_u8(byte_counts) as usize; - } - } - - let done = chunks * LANES; - - // SAFETY: - // - `done <= len`. - // - Tail range is `done..len`. - // - Pointer validity is guaranteed by the caller. - count + unsafe { scalar::count_words(src.add(done), len - done) } - } -} +mod neon; #[cfg(test)] mod tests_for_backend_equivalence; diff --git a/src/traits/words_scan/funcs_for_count_ones/avx2.rs b/src/traits/words_scan/funcs_for_count_ones/avx2.rs new file mode 100644 index 0000000..78cf8e6 --- /dev/null +++ b/src/traits/words_scan/funcs_for_count_ones/avx2.rs @@ -0,0 +1,80 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_add_epi8, _mm256_add_epi64, _mm256_and_si256, _mm256_loadu_si256, + _mm256_sad_epu8, _mm256_set1_epi8, _mm256_setr_epi8, _mm256_setzero_si256, _mm256_shuffle_epi8, + _mm256_srli_epi16, _mm256_storeu_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_add_epi8, _mm256_add_epi64, _mm256_and_si256, _mm256_loadu_si256, + _mm256_sad_epu8, _mm256_set1_epi8, _mm256_setr_epi8, _mm256_setzero_si256, _mm256_shuffle_epi8, + _mm256_srli_epi16, _mm256_storeu_si256, +}; + +const LANES: usize = 4; + +/// AVX2 backend for counting set bits in `src[0..len]`. +/// +/// Uses nibble lookup with `vpshufb`, then sums byte counts with `vpsadbw`. +/// +/// # Safety +/// +/// - Caller must only call this when AVX2 is available. +/// - `src` must be valid for reads of `len` initialized `u64` values. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { + let chunks = len / LANES; + + // SAFETY: + // - These constructors require AVX2 to be available. + // - This function is compiled with `target_feature = "avx2"`. + let lookup = _mm256_setr_epi8( + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, // + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + ); + let low_mask = _mm256_set1_epi8(0x0F); + let zero = _mm256_setzero_si256(); + + // Accumulate popcounts in a SIMD register to avoid per-chunk + // store-to-memory round-trips. Reduced to scalar only once at the end. + let mut acc = zero; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm256_loadu_si256` permits unaligned reads. + // - `src` validity is guaranteed by the caller. + unsafe { + let bytes = _mm256_loadu_si256(src.add(offset).cast::<__m256i>()); + + let low = _mm256_and_si256(bytes, low_mask); + let high = _mm256_and_si256(_mm256_srli_epi16(bytes, 4), low_mask); + + let low_counts = _mm256_shuffle_epi8(lookup, low); + let high_counts = _mm256_shuffle_epi8(lookup, high); + let byte_counts = _mm256_add_epi8(low_counts, high_counts); + + let sums = _mm256_sad_epu8(byte_counts, zero); + acc = _mm256_add_epi64(acc, sums); + } + } + + let mut lane_sums = [0u64; LANES]; + // SAFETY: `lane_sums` has sufficient space for 4 u64 values. + unsafe { _mm256_storeu_si256(lane_sums.as_mut_ptr().cast::<__m256i>(), acc) }; + let mut count = lane_sums.iter().map(|&sum| sum as usize).sum::(); + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity is guaranteed by the caller. + count += unsafe { scalar::count_words(src.add(done), len - done) }; + count +} diff --git a/src/traits/words_scan/funcs_for_count_ones/neon.rs b/src/traits/words_scan/funcs_for_count_ones/neon.rs new file mode 100644 index 0000000..62ff9dd --- /dev/null +++ b/src/traits/words_scan/funcs_for_count_ones/neon.rs @@ -0,0 +1,43 @@ +use super::scalar; + +use core::arch::aarch64::{vaddvq_u8, vcntq_u8, vld1q_u64, vreinterpretq_u8_u64}; + +const LANES: usize = 2; + +/// NEON backend for counting set bits in `src[0..len]`. +/// +/// Counts bits per byte with `vcntq_u8`, then horizontally sums byte counts. +/// +/// # Safety +/// +/// - Caller must only call this when NEON is available. +/// - `src` must be valid for reads of `len` initialized `u64` values. +#[target_feature(enable = "neon")] +pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { + let chunks = len / LANES; + let mut count = 0usize; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `vld1q_u64` reads exactly 2 u64 values. + // - `src` validity is guaranteed by the caller. + unsafe { + let words = vld1q_u64(src.add(offset)); + let bytes = vreinterpretq_u8_u64(words); + let byte_counts = vcntq_u8(bytes); + + count += vaddvq_u8(byte_counts) as usize; + } + } + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity is guaranteed by the caller. + count + unsafe { scalar::count_words(src.add(done), len - done) } +} diff --git a/src/traits/words_scan/funcs_for_count_ones/scalar.rs b/src/traits/words_scan/funcs_for_count_ones/scalar.rs new file mode 100644 index 0000000..9fcf732 --- /dev/null +++ b/src/traits/words_scan/funcs_for_count_ones/scalar.rs @@ -0,0 +1,20 @@ +/// Scalar backend for counting set bits in `src[0..len]`. +/// +/// # Safety +/// +/// - `src` must be valid for reads of `len` initialized `u64` values. +#[inline] +pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { + let mut count = 0usize; + + for i in 0..len { + // SAFETY: + // - `i < len`. + // - Pointer validity is guaranteed by the caller. + unsafe { + count += src.add(i).read().count_ones() as usize; + } + } + + count +} diff --git a/src/traits/words_scan/funcs_for_count_ones/ssse3.rs b/src/traits/words_scan/funcs_for_count_ones/ssse3.rs new file mode 100644 index 0000000..f677134 --- /dev/null +++ b/src/traits/words_scan/funcs_for_count_ones/ssse3.rs @@ -0,0 +1,77 @@ +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_add_epi8, _mm_add_epi64, _mm_and_si128, _mm_loadu_si128, _mm_sad_epu8, + _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, + _mm_storeu_si128, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_add_epi8, _mm_add_epi64, _mm_and_si128, _mm_loadu_si128, _mm_sad_epu8, + _mm_set1_epi8, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, + _mm_storeu_si128, +}; + +const LANES: usize = 2; + +/// SSSE3 backend for counting set bits in `src[0..len]`. +/// +/// Uses nibble lookup with `pshufb`, then sums byte counts with `psadbw`. +/// +/// # Safety +/// +/// - Caller must only call this when SSSE3 is available. +/// - `src` must be valid for reads of `len` initialized `u64` values. +#[target_feature(enable = "ssse3")] +pub(super) unsafe fn count_words(src: *const u64, len: usize) -> usize { + let chunks = len / LANES; + + let lookup = _mm_setr_epi8( + 0, 1, 1, 2, 1, 2, 2, 3, // + 1, 2, 2, 3, 2, 3, 3, 4, + ); + let low_mask = _mm_set1_epi8(0x0F); + let zero = _mm_setzero_si128(); + + // Accumulate popcounts in a SIMD register to avoid per-chunk + // store-to-memory round-trips. Reduced to scalar only once at the end. + let mut acc = zero; + + for chunk in 0..chunks { + let offset = chunk * LANES; + + // SAFETY: + // - `offset + LANES <= len`. + // - `_mm_loadu_si128` permits unaligned reads. + // - `src` validity is guaranteed by the caller. + unsafe { + let bytes = _mm_loadu_si128(src.add(offset).cast::<__m128i>()); + + let low = _mm_and_si128(bytes, low_mask); + let high = _mm_and_si128(_mm_srli_epi16(bytes, 4), low_mask); + + let low_counts = _mm_shuffle_epi8(lookup, low); + let high_counts = _mm_shuffle_epi8(lookup, high); + let byte_counts = _mm_add_epi8(low_counts, high_counts); + + let sums = _mm_sad_epu8(byte_counts, zero); + acc = _mm_add_epi64(acc, sums); + } + } + + let mut lane_sums = [0u64; LANES]; + // SAFETY: `lane_sums` has sufficient space for 2 u64 values. + unsafe { _mm_storeu_si128(lane_sums.as_mut_ptr().cast::<__m128i>(), acc) }; + let mut count = lane_sums.iter().map(|&sum| sum as usize).sum::(); + + let done = chunks * LANES; + + // SAFETY: + // - `done <= len`. + // - Tail range is `done..len`. + // - Pointer validity is guaranteed by the caller. + count += unsafe { scalar::count_words(src.add(done), len - done) }; + count +} diff --git a/src/traits/words_scan/funcs_for_ends/mod.rs b/src/traits/words_scan/funcs_for_ends.rs similarity index 100% rename from src/traits/words_scan/funcs_for_ends/mod.rs rename to src/traits/words_scan/funcs_for_ends.rs diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs index a53f490..b0df422 100644 --- a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs @@ -137,281 +137,14 @@ pub(crate) fn leading( scanned.min(bit_len) } -// ═══════════════════════════════════════════════════════════════════════ -// AVX2 backend — 256-bit / 4-lane, unaligned loads only. -// ═══════════════════════════════════════════════════════════════════════ - #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, - _mm256_testz_si256, _mm256_xor_si256, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, - _mm256_testz_si256, _mm256_xor_si256, - }; - - const LANES: usize = 4; - const STRIDE: usize = 16; // 4 × LANES for unrolled iteration - const ALIGN_THRESHOLD: usize = 128; - - /// AVX2 forward scan: advances `p` past all-FILL 256-bit chunks. - /// - /// For large inputs (≥ ALIGN_THRESHOLD words), aligns `p` to a - /// 32-byte boundary and uses `_mm256_load_si256` (aligned) for the - /// hot loop. Smaller inputs use `_mm256_loadu_si256` (unaligned) - /// to avoid the alignment prefix overhead. - /// - /// # Safety - /// - /// Caller must ensure AVX2 is enabled for the compilation target. - /// `p` through `end` must be valid for u64 reads. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn leading_scan( - mut p: *const u64, - end: *const u64, - base: *const u64, - total: usize, - ) -> *const u64 { - // SAFETY: AVX2 is enabled for this compilation target. All pointer - // arithmetic stays within bounds. - unsafe { - if total >= ALIGN_THRESHOLD { - // Distance in words to the next 32-byte boundary. - let words_to_align = (4usize.wrapping_sub((base as usize / 8) % 4)) % 4; - if words_to_align > 0 { - let prefix_end = base.add(words_to_align); - while p < prefix_end { - if *p != FILL { - return p; - } - p = p.add(1); - } - } - let mut iters = - (end as usize - p as usize) / (STRIDE * core::mem::size_of::()); - while iters > 0 { - if FILL == 0 { - let d0 = _mm256_load_si256(p.cast::<__m256i>()); - let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); - let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); - let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); - let any01 = _mm256_or_si256(d0, d1); - let any23 = _mm256_or_si256(d2, d3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d0 = _mm256_load_si256(p.cast::<__m256i>()); - let x0 = _mm256_xor_si256(d0, fill_vec); - let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); - let x1 = _mm256_xor_si256(d1, fill_vec); - let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); - let x2 = _mm256_xor_si256(d2, fill_vec); - let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); - let x3 = _mm256_xor_si256(d3, fill_vec); - let any01 = _mm256_or_si256(x0, x1); - let any23 = _mm256_or_si256(x2, x3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } - p = p.add(STRIDE); - iters -= 1; - } - } else { - // 4×-unrolled unaligned path. - let mut iters = total / STRIDE; - while iters > 0 { - if FILL == 0 { - let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); - let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); - let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); - let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); - let any01 = _mm256_or_si256(d0, d1); - let any23 = _mm256_or_si256(d2, d3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); - let x0 = _mm256_xor_si256(d0, fill_vec); - let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); - let x1 = _mm256_xor_si256(d1, fill_vec); - let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); - let x2 = _mm256_xor_si256(d2, fill_vec); - let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); - let x3 = _mm256_xor_si256(d3, fill_vec); - let any01 = _mm256_or_si256(x0, x1); - let any23 = _mm256_or_si256(x2, x3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } - p = p.add(STRIDE); - iters -= 1; - } - } - // Single-chunk remainder (LANES = 4 words). - let limit = end.sub(LANES); - while p <= limit { - if FILL == 0 { - let d = _mm256_loadu_si256(p.cast::<__m256i>()); - if _mm256_testz_si256(d, d) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d = _mm256_loadu_si256(p.cast::<__m256i>()); - let x = _mm256_xor_si256(d, fill_vec); - if _mm256_testz_si256(x, x) == 0 { - break; - } - } - p = p.add(LANES); - } - p - } - } -} - -// ═══════════════════════════════════════════════════════════════════════ -// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). -// ═══════════════════════════════════════════════════════════════════════ +mod avx2; #[allow(unused)] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - _mm_setzero_si128, _mm_xor_si128, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - _mm_setzero_si128, _mm_xor_si128, - }; - - const LANES: usize = 2; - const LANES_2X: usize = LANES * 2; - - #[inline(always)] - unsafe fn chunk_eq(ptr: *const u64) -> bool { - // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and - // SSE2 is available. - unsafe { - let data = _mm_loadu_si128(ptr.cast::<__m128i>()); - if FILL == 0 { - let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); - _mm_movemask_epi8(cmp) == 0xFFFF - } else { - let fill_vec = _mm_set1_epi64x(FILL as i64); - let xor = _mm_xor_si128(data, fill_vec); - let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); - _mm_movemask_epi8(cmp) == 0xFFFF - } - } - } - - /// SSE2 forward scan: advances `p` past all-FILL chunks. - /// - /// # Safety - /// - /// Caller must ensure SSE2 is available (baseline on x86-64). - /// `p` through `end` must be valid for u64 reads. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn leading_scan( - mut p: *const u64, - end: *const u64, - total: usize, - ) -> *const u64 { - // SAFETY: SSE2 is enabled for this compilation target. - unsafe { - let mut iters = total / LANES_2X; - while iters > 0 { - if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { - return p; - } - p = p.add(LANES_2X); - iters -= 1; - } - let limit = end.sub(LANES); - while p <= limit { - if !chunk_eq::(p) { - break; - } - p = p.add(LANES); - } - p - } - } -} - -// ═══════════════════════════════════════════════════════════════════════ -// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). -// ═══════════════════════════════════════════════════════════════════════ +mod sse2; #[allow(unused)] #[cfg(target_arch = "aarch64")] -mod neon { - use core::arch::aarch64::{uint64x2_t, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; - - const LANES: usize = 2; - const LANES_2X: usize = LANES * 2; - - #[inline(always)] - unsafe fn chunk_eq(ptr: *const u64) -> bool { - // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and - // NEON is available. - unsafe { - let data = vld1q_u64(ptr); - let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); - vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 - } - } - - /// NEON forward scan: advances `p` past all-FILL chunks. - /// - /// # Safety - /// - /// Caller must ensure NEON is available. - /// `p` through `end` must be valid for u64 reads. - #[target_feature(enable = "neon")] - pub(super) unsafe fn leading_scan( - mut p: *const u64, - end: *const u64, - total: usize, - ) -> *const u64 { - // SAFETY: only callable when NEON is available. All pointer - // arithmetic stays within `[p, end)`. - unsafe { - let mut iters = total / LANES_2X; - while iters > 0 { - if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { - return p; - } - p = p.add(LANES_2X); - iters -= 1; - } - let limit = end.sub(LANES); - while p <= limit { - if !chunk_eq::(p) { - break; - } - p = p.add(LANES); - } - p - } - } -} +mod neon; diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/avx2.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/avx2.rs new file mode 100644 index 0000000..019acdc --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/avx2.rs @@ -0,0 +1,142 @@ +// ═══════════════════════════════════════════════════════════════════════ +// AVX2 backend — 256-bit / 4-lane, unaligned loads only. +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, + _mm256_testz_si256, _mm256_xor_si256, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, + _mm256_testz_si256, _mm256_xor_si256, +}; + +const LANES: usize = 4; +const STRIDE: usize = 16; // 4 × LANES for unrolled iteration +const ALIGN_THRESHOLD: usize = 128; + +/// AVX2 forward scan: advances `p` past all-FILL 256-bit chunks. +/// +/// For large inputs (≥ ALIGN_THRESHOLD words), aligns `p` to a +/// 32-byte boundary and uses `_mm256_load_si256` (aligned) for the +/// hot loop. Smaller inputs use `_mm256_loadu_si256` (unaligned) +/// to avoid the alignment prefix overhead. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is enabled for the compilation target. +/// `p` through `end` must be valid for u64 reads. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + base: *const u64, + total: usize, +) -> *const u64 { + // SAFETY: AVX2 is enabled for this compilation target. All pointer + // arithmetic stays within bounds. + unsafe { + if total >= ALIGN_THRESHOLD { + // Distance in words to the next 32-byte boundary. + let words_to_align = (4usize.wrapping_sub((base as usize / 8) % 4)) % 4; + if words_to_align > 0 { + let prefix_end = base.add(words_to_align); + while p < prefix_end { + if *p != FILL { + return p; + } + p = p.add(1); + } + } + let mut iters = (end as usize - p as usize) / (STRIDE * core::mem::size_of::()); + while iters > 0 { + if FILL == 0 { + let d0 = _mm256_load_si256(p.cast::<__m256i>()); + let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); + let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); + let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d0 = _mm256_load_si256(p.cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + let d2 = _mm256_load_si256(p.add(LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_load_si256(p.add(LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } + p = p.add(STRIDE); + iters -= 1; + } + } else { + // 4×-unrolled unaligned path. + let mut iters = total / STRIDE; + while iters > 0 { + if FILL == 0 { + let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); + let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); + let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); + let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + let d2 = _mm256_loadu_si256(p.add(LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_loadu_si256(p.add(LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } + p = p.add(STRIDE); + iters -= 1; + } + } + // Single-chunk remainder (LANES = 4 words). + let limit = end.sub(LANES); + while p <= limit { + if FILL == 0 { + let d = _mm256_loadu_si256(p.cast::<__m256i>()); + if _mm256_testz_si256(d, d) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d = _mm256_loadu_si256(p.cast::<__m256i>()); + let x = _mm256_xor_si256(d, fill_vec); + if _mm256_testz_si256(x, x) == 0 { + break; + } + } + p = p.add(LANES); + } + p + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/neon.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/neon.rs new file mode 100644 index 0000000..f65d698 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/neon.rs @@ -0,0 +1,53 @@ +// ═══════════════════════════════════════════════════════════════════════ +// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +use core::arch::aarch64::{uint64x2_t, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; + +const LANES: usize = 2; +const LANES_2X: usize = LANES * 2; + +#[inline(always)] +unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // NEON is available. + unsafe { + let data = vld1q_u64(ptr); + let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); + vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 + } +} + +/// NEON forward scan: advances `p` past all-FILL chunks. +/// +/// # Safety +/// +/// Caller must ensure NEON is available. +/// `p` through `end` must be valid for u64 reads. +#[target_feature(enable = "neon")] +pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + total: usize, +) -> *const u64 { + // SAFETY: only callable when NEON is available. All pointer + // arithmetic stays within `[p, end)`. + unsafe { + let mut iters = total / LANES_2X; + while iters > 0 { + if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { + return p; + } + p = p.add(LANES_2X); + iters -= 1; + } + let limit = end.sub(LANES); + while p <= limit { + if !chunk_eq::(p) { + break; + } + p = p.add(LANES); + } + p + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/sse2.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/sse2.rs new file mode 100644 index 0000000..8e36b87 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core/sse2.rs @@ -0,0 +1,68 @@ +// ═══════════════════════════════════════════════════════════════════════ +// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, +}; + +const LANES: usize = 2; +const LANES_2X: usize = LANES * 2; + +#[inline(always)] +unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // SSE2 is available. + unsafe { + let data = _mm_loadu_si128(ptr.cast::<__m128i>()); + if FILL == 0 { + let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } else { + let fill_vec = _mm_set1_epi64x(FILL as i64); + let xor = _mm_xor_si128(data, fill_vec); + let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } + } +} + +/// SSE2 forward scan: advances `p` past all-FILL chunks. +/// +/// # Safety +/// +/// Caller must ensure SSE2 is available (baseline on x86-64). +/// `p` through `end` must be valid for u64 reads. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + total: usize, +) -> *const u64 { + // SAFETY: SSE2 is enabled for this compilation target. + unsafe { + let mut iters = total / LANES_2X; + while iters > 0 { + if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { + return p; + } + p = p.add(LANES_2X); + iters -= 1; + } + let limit = end.sub(LANES); + while p <= limit { + if !chunk_eq::(p) { + break; + } + p = p.add(LANES); + } + p + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs index 884d8c2..3750bbb 100644 --- a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs @@ -7,6 +7,18 @@ use super::count_matching; use crate::{SMALL_WORDS, WORD_BITS}; +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2; + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod sse2; + +#[allow(unused)] +#[cfg(target_arch = "aarch64")] +mod neon; + // ── Dispatch ─────────────────────────────────────────────────────────── #[inline] @@ -140,237 +152,3 @@ pub(crate) fn trailing( scanned.min(bit_len) } - -// ═══════════════════════════════════════════════════════════════════════ -// AVX2 backend — 256-bit / 4-lane reverse scan. -// ═══════════════════════════════════════════════════════════════════════ - -#[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, - _mm256_xor_si256, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, - _mm256_xor_si256, - }; - - const LANES: usize = 4; - const STRIDE: usize = 16; - - /// AVX2 reverse scan: scans backwards from `wi_end` and advances `done` - /// past all-FILL 256-bit chunks. - /// - /// Returns the updated `done` count (total words consumed from right). - /// - /// # Safety - /// - /// Caller must ensure AVX2 is enabled for the compilation target. - /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn trailing_scan( - ptr: *const u64, - wi_end: usize, - mut done: usize, - total_words: usize, - ) -> usize { - // SAFETY: AVX2 is enabled for this compilation target. All pointer - // arithmetic stays within bounds. - unsafe { - // Four vectors are combined before testing. This keeps the - // all-matching hot path to one branch per 16 words, matching - // the forward scanner's throughput-oriented layout. - while done + STRIDE <= total_words { - let chunk_start = wi_end + 1 - (done + STRIDE); - if FILL == 0 { - let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); - let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); - let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); - let any01 = _mm256_or_si256(d0, d1); - let any23 = _mm256_or_si256(d2, d3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - let x0 = _mm256_xor_si256(d0, fill_vec); - let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); - let x1 = _mm256_xor_si256(d1, fill_vec); - let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); - let x2 = _mm256_xor_si256(d2, fill_vec); - let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); - let x3 = _mm256_xor_si256(d3, fill_vec); - let any01 = _mm256_or_si256(x0, x1); - let any23 = _mm256_or_si256(x2, x3); - let any = _mm256_or_si256(any01, any23); - if _mm256_testz_si256(any, any) == 0 { - break; - } - } - done += STRIDE; - } - // Single-chunk remainder - while done + LANES <= total_words { - let chunk_start = wi_end + 1 - (done + LANES); - if FILL == 0 { - let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - if _mm256_testz_si256(d, d) == 0 { - break; - } - } else { - let fill_vec = _mm256_set1_epi64x(FILL as i64); - let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); - let x = _mm256_xor_si256(d, fill_vec); - if _mm256_testz_si256(x, x) == 0 { - break; - } - } - done += LANES; - } - - done - } - } -} - -// ═══════════════════════════════════════════════════════════════════════ -// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). -// ═══════════════════════════════════════════════════════════════════════ - -#[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - _mm_setzero_si128, _mm_xor_si128, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - _mm_setzero_si128, _mm_xor_si128, - }; - - const LANES: usize = 2; - const LANES_2X: usize = LANES * 2; - - #[inline(always)] - unsafe fn chunk_eq(ptr: *const u64) -> bool { - // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and - // SSE2 is available. - unsafe { - let data = _mm_loadu_si128(ptr.cast::<__m128i>()); - if FILL == 0 { - let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); - _mm_movemask_epi8(cmp) == 0xFFFF - } else { - let fill_vec = _mm_set1_epi64x(FILL as i64); - let xor = _mm_xor_si128(data, fill_vec); - let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); - _mm_movemask_epi8(cmp) == 0xFFFF - } - } - } - - /// SSE2 reverse scan: advances `done` past all-FILL chunks from the right. - /// - /// # Safety - /// - /// Caller must ensure SSE2 is available (baseline on x86-64). - /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. - #[target_feature(enable = "sse2")] - pub(super) unsafe fn trailing_scan( - ptr: *const u64, - wi_end: usize, - mut done: usize, - total_words: usize, - ) -> usize { - // SAFETY: SSE2 is enabled for this compilation target. - unsafe { - while done + LANES_2X <= total_words { - let chunk_start = wi_end + 1 - (done + LANES_2X); - if !chunk_eq::(ptr.add(chunk_start)) - || !chunk_eq::(ptr.add(chunk_start + LANES)) - { - return done; - } - done += LANES_2X; - } - while done + LANES <= total_words { - let chunk_start = wi_end + 1 - (done + LANES); - if !chunk_eq::(ptr.add(chunk_start)) { - break; - } - done += LANES; - } - done - } - } -} - -// ═══════════════════════════════════════════════════════════════════════ -// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). -// ═══════════════════════════════════════════════════════════════════════ - -#[allow(unused)] -#[cfg(target_arch = "aarch64")] -mod neon { - use core::arch::aarch64::{vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; - - const LANES: usize = 2; - const LANES_2X: usize = LANES * 2; - - #[inline(always)] - unsafe fn chunk_eq(ptr: *const u64) -> bool { - // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and - // NEON is available. - unsafe { - let data = vld1q_u64(ptr); - let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); - vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 - } - } - - /// NEON reverse scan: advances `done` past all-FILL chunks from the right. - /// - /// # Safety - /// - /// Caller must ensure NEON is available. - /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. - #[target_feature(enable = "neon")] - pub(super) unsafe fn trailing_scan( - ptr: *const u64, - wi_end: usize, - mut done: usize, - total_words: usize, - ) -> usize { - // SAFETY: only callable when NEON is available. All pointer - // arithmetic stays within bounds. - unsafe { - while done + LANES_2X <= total_words { - let chunk_start = wi_end + 1 - (done + LANES_2X); - if !chunk_eq::(ptr.add(chunk_start)) - || !chunk_eq::(ptr.add(chunk_start + LANES)) - { - return done; - } - done += LANES_2X; - } - while done + LANES <= total_words { - let chunk_start = wi_end + 1 - (done + LANES); - if !chunk_eq::(ptr.add(chunk_start)) { - break; - } - done += LANES; - } - done - } - } -} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/avx2.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/avx2.rs new file mode 100644 index 0000000..802ea69 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/avx2.rs @@ -0,0 +1,94 @@ +// ═══════════════════════════════════════════════════════════════════════ +// AVX2 backend — 256-bit / 4-lane reverse scan. +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, + _mm256_xor_si256, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m256i, _mm256_loadu_si256, _mm256_or_si256, _mm256_set1_epi64x, _mm256_testz_si256, + _mm256_xor_si256, +}; + +const LANES: usize = 4; +const STRIDE: usize = 16; + +/// AVX2 reverse scan: scans backwards from `wi_end` and advances `done` +/// past all-FILL 256-bit chunks. +/// +/// Returns the updated `done` count (total words consumed from right). +/// +/// # Safety +/// +/// Caller must ensure AVX2 is enabled for the compilation target. +/// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, +) -> usize { + // SAFETY: AVX2 is enabled for this compilation target. All pointer + // arithmetic stays within bounds. + unsafe { + // Four vectors are combined before testing. This keeps the + // all-matching hot path to one branch per 16 words, matching + // the forward scanner's throughput-oriented layout. + while done + STRIDE <= total_words { + let chunk_start = wi_end + 1 - (done + STRIDE); + if FILL == 0 { + let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); + let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); + let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); + let any01 = _mm256_or_si256(d0, d1); + let any23 = _mm256_or_si256(d2, d3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d0 = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + let d2 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 2).cast::<__m256i>()); + let x2 = _mm256_xor_si256(d2, fill_vec); + let d3 = _mm256_loadu_si256(ptr.add(chunk_start + LANES * 3).cast::<__m256i>()); + let x3 = _mm256_xor_si256(d3, fill_vec); + let any01 = _mm256_or_si256(x0, x1); + let any23 = _mm256_or_si256(x2, x3); + let any = _mm256_or_si256(any01, any23); + if _mm256_testz_si256(any, any) == 0 { + break; + } + } + done += STRIDE; + } + // Single-chunk remainder + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if FILL == 0 { + let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + if _mm256_testz_si256(d, d) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let x = _mm256_xor_si256(d, fill_vec); + if _mm256_testz_si256(x, x) == 0 { + break; + } + } + done += LANES; + } + + done + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/neon.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/neon.rs new file mode 100644 index 0000000..c5274f4 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/neon.rs @@ -0,0 +1,55 @@ +// ═══════════════════════════════════════════════════════════════════════ +// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +use core::arch::aarch64::{vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; + +const LANES: usize = 2; +const LANES_2X: usize = LANES * 2; + +#[inline(always)] +unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // NEON is available. + unsafe { + let data = vld1q_u64(ptr); + let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); + vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 + } +} + +/// NEON reverse scan: advances `done` past all-FILL chunks from the right. +/// +/// # Safety +/// +/// Caller must ensure NEON is available. +/// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. +#[target_feature(enable = "neon")] +pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, +) -> usize { + // SAFETY: only callable when NEON is available. All pointer + // arithmetic stays within bounds. + unsafe { + while done + LANES_2X <= total_words { + let chunk_start = wi_end + 1 - (done + LANES_2X); + if !chunk_eq::(ptr.add(chunk_start)) + || !chunk_eq::(ptr.add(chunk_start + LANES)) + { + return done; + } + done += LANES_2X; + } + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if !chunk_eq::(ptr.add(chunk_start)) { + break; + } + done += LANES; + } + done + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/sse2.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/sse2.rs new file mode 100644 index 0000000..267a4b0 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core/sse2.rs @@ -0,0 +1,70 @@ +// ═══════════════════════════════════════════════════════════════════════ +// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, +}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, +}; + +const LANES: usize = 2; +const LANES_2X: usize = LANES * 2; + +#[inline(always)] +unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // SSE2 is available. + unsafe { + let data = _mm_loadu_si128(ptr.cast::<__m128i>()); + if FILL == 0 { + let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } else { + let fill_vec = _mm_set1_epi64x(FILL as i64); + let xor = _mm_xor_si128(data, fill_vec); + let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } + } +} + +/// SSE2 reverse scan: advances `done` past all-FILL chunks from the right. +/// +/// # Safety +/// +/// Caller must ensure SSE2 is available (baseline on x86-64). +/// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. +#[target_feature(enable = "sse2")] +pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, +) -> usize { + // SAFETY: SSE2 is enabled for this compilation target. + unsafe { + while done + LANES_2X <= total_words { + let chunk_start = wi_end + 1 - (done + LANES_2X); + if !chunk_eq::(ptr.add(chunk_start)) + || !chunk_eq::(ptr.add(chunk_start + LANES)) + { + return done; + } + done += LANES_2X; + } + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if !chunk_eq::(ptr.add(chunk_start)) { + break; + } + done += LANES; + } + done + } +} diff --git a/tests/adversarial/tests_for_access.rs b/tests/adversarial/tests_for_access.rs index ca23465..4c85a56 100644 --- a/tests/adversarial/tests_for_access.rs +++ b/tests/adversarial/tests_for_access.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_get_out_of_bounds() { diff --git a/tests/adversarial/tests_for_bitstr.rs b/tests/adversarial/tests_for_bitstr.rs index e382d82..a5111ae 100644 --- a/tests/adversarial/tests_for_bitstr.rs +++ b/tests/adversarial/tests_for_bitstr.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_clone_invariants() { diff --git a/tests/adversarial/tests_for_bugs.rs b/tests/adversarial/tests_for_bugs.rs index 527cae1..4dadb29 100644 --- a/tests/adversarial/tests_for_bugs.rs +++ b/tests/adversarial/tests_for_bugs.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; /// Regression test for B1: `set_chunk` must mask unused high bits after writing. #[test] diff --git a/tests/adversarial/tests_for_count.rs b/tests/adversarial/tests_for_count.rs index 63ecea5..d0a1d4a 100644 --- a/tests/adversarial/tests_for_count.rs +++ b/tests/adversarial/tests_for_count.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_count_ones_zeros() { diff --git a/tests/adversarial/tests_for_fmt.rs b/tests/adversarial/tests_for_fmt.rs index cce1e3f..39ba508 100644 --- a/tests/adversarial/tests_for_fmt.rs +++ b/tests/adversarial/tests_for_fmt.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_display_roundtrip() { diff --git a/tests/adversarial/tests_for_iter.rs b/tests/adversarial/tests_for_iter.rs index 98f1d3b..10d523d 100644 --- a/tests/adversarial/tests_for_iter.rs +++ b/tests/adversarial/tests_for_iter.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_iter_empty() { diff --git a/tests/adversarial/tests_for_leading_trailing.rs b/tests/adversarial/tests_for_leading_trailing.rs index 8439c64..5f713ee 100644 --- a/tests/adversarial/tests_for_leading_trailing.rs +++ b/tests/adversarial/tests_for_leading_trailing.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; // =========================================================================== // A. BitString leading/trailing — exhaustive small-length coverage diff --git a/tests/adversarial/tests_for_matching.rs b/tests/adversarial/tests_for_matching.rs index 0ebc21e..0927f29 100644 --- a/tests/adversarial/tests_for_matching.rs +++ b/tests/adversarial/tests_for_matching.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn alignment_pattern(len: usize) -> String { (0..len) diff --git a/tests/adversarial/tests_for_ord_hash.rs b/tests/adversarial/tests_for_ord_hash.rs index 8235582..83e6f5c 100644 --- a/tests/adversarial/tests_for_ord_hash.rs +++ b/tests/adversarial/tests_for_ord_hash.rs @@ -1,6 +1,6 @@ use super::*; use core::cmp::Ordering; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; fn alignment_pattern(len: usize) -> String { (0..len) diff --git a/tests/adversarial/tests_for_predicates.rs b/tests/adversarial/tests_for_predicates.rs index 206b20b..6fec058 100644 --- a/tests/adversarial/tests_for_predicates.rs +++ b/tests/adversarial/tests_for_predicates.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_predicates_corner() { diff --git a/tests/adversarial/tests_for_replace.rs b/tests/adversarial/tests_for_replace.rs index 99a60a4..bc7e689 100644 --- a/tests/adversarial/tests_for_replace.rs +++ b/tests/adversarial/tests_for_replace.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_replace_equal_length() { diff --git a/tests/adversarial/tests_for_slice_drain.rs b/tests/adversarial/tests_for_slice_drain.rs index 7ac5205..ec8d59b 100644 --- a/tests/adversarial/tests_for_slice_drain.rs +++ b/tests/adversarial/tests_for_slice_drain.rs @@ -1,5 +1,5 @@ use super::*; -use int_interval::UsizeCO; +use int_intervals::UsizeCO; #[test] fn attack_slice_interval_clamping() { diff --git a/tests/packed_char.rs b/tests/packed_char.rs new file mode 100644 index 0000000..9ae43ce --- /dev/null +++ b/tests/packed_char.rs @@ -0,0 +1,461 @@ +use bit_string::{BitString, PackedString, packed, traits::PackedChar}; +use int_intervals::UsizeCO; +use proptest::prelude::*; + +#[packed(bits = 1)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PackedSymbol { + Zero = 0, + One = 1, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[packed(bits = 2)] +enum Symbol { + Zero = 0, + One = 1, + Two = 2, +} + +#[packed(bits = 3)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Oct { + V0 = 0, + V1 = 1, + V2 = 2, + V3 = 3, + V4 = 4, + V5 = 5, + V6 = 6, + V7 = 7, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct WideCode(u8); + +impl PackedChar<7> for WideCode { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + (code < 128).then_some(Self(code)) + } +} + +fn symbol(code: u8) -> Symbol { + match code { + 0 => Symbol::Zero, + 1 => Symbol::One, + 2 => Symbol::Two, + _ => unreachable!(), + } +} + +fn oct(code: u8) -> Oct { + match code { + 0 => Oct::V0, + 1 => Oct::V1, + 2 => Oct::V2, + 3 => Oct::V3, + 4 => Oct::V4, + 5 => Oct::V5, + 6 => Oct::V6, + 7 => Oct::V7, + _ => unreachable!(), + } +} + +fn wide(code: u8) -> WideCode { + WideCode(code) +} + +fn packed(codes: &[u8]) -> PackedString { + PackedString::from_chars(codes.iter().copied().map(symbol)) +} + +fn packed_as(codes: &[u8], decode: fn(u8) -> C) -> PackedString +where + C: PackedChar, +{ + PackedString::from_chars(codes.iter().copied().map(decode)) +} + +fn matches_at(haystack: &[u8], start: usize, needle: &[u8]) -> bool { + haystack + .get(start..start.saturating_add(needle.len())) + .is_some_and(|window| window == needle) +} + +fn assert_access_slice_order( + left: &[u8], + right: &[u8], + start: usize, + len: usize, + decode: fn(u8) -> C, +) where + C: PackedChar + core::fmt::Debug, +{ + let string = packed_as(left, decode); + assert_eq!(string.char_len(), left.len()); + let view = string.as_packed_str(); + for index in 0..=left.len() + 1 { + assert_eq!(string.get(index), left.get(index).copied().map(decode)); + assert_eq!(view.get(index), left.get(index).copied().map(decode)); + } + + let oracle_start = start.min(left.len()); + let oracle_end = start.saturating_add(len).min(left.len()).max(oracle_start); + let slice = string.slice(UsizeCO::checked_from_start_len(start, len).unwrap()); + assert_eq!( + slice.to_vec(), + left[oracle_start..oracle_end] + .iter() + .copied() + .map(decode) + .collect::>() + ); + + let right_string = packed_as(right, decode); + assert_eq!(string.cmp(&right_string), left.cmp(&right)); + assert_eq!( + string.as_packed_str().cmp(&right_string.as_packed_str()), + left.cmp(&right) + ); +} + +fn assert_matching(haystack: &[u8], needle: &[u8], decode: fn(u8) -> C) +where + C: PackedChar, +{ + let haystack_string = packed_as(haystack, decode); + let needle_string = packed_as(needle, decode); + let haystack_view = haystack_string.as_packed_str(); + let needle_view = needle_string.as_packed_str(); + let max_start = haystack.len().saturating_sub(needle.len()); + let expected_find = (0..=max_start).find(|&start| matches_at(haystack, start, needle)); + let expected_rfind = (0..=max_start) + .rev() + .find(|&start| matches_at(haystack, start, needle)); + + assert_eq!(haystack_view.find(needle_view), expected_find); + assert_eq!(haystack_view.rfind(needle_view), expected_rfind); + assert_eq!(haystack_view.contains(needle_view), expected_find.is_some()); +} + +fn assert_edits( + initial: &[u8], + replacement: &[u8], + insert_index: usize, + remove_index: usize, + replace_start: usize, + inserted_code: u8, + decode: fn(u8) -> C, +) where + C: PackedChar + core::fmt::Debug, +{ + let mut string = packed_as(initial, decode); + let mut oracle = initial.to_vec(); + + let oracle_insert_index = insert_index.min(oracle.len()); + string.insert(insert_index, decode(inserted_code)); + oracle.insert(oracle_insert_index, inserted_code); + + if !oracle.is_empty() { + let remove_index = remove_index.min(oracle.len() - 1); + assert_eq!( + string.remove(remove_index).code(), + oracle.remove(remove_index) + ); + } + + let oracle_replace_start = replace_start.min(oracle.len()); + let replace_end = oracle_replace_start + .saturating_add(replacement.len()) + .min(oracle.len()); + string.replace_assign(replace_start, &packed_as(replacement, decode)); + oracle.splice( + oracle_replace_start..replace_end, + replacement.iter().copied(), + ); + assert_eq!( + string + .to_vec() + .iter() + .copied() + .map(|value| value.code()) + .collect::>(), + oracle + ); + + let drain_len = insert_index.max(1); + let oracle_drain_start = remove_index.min(oracle.len()); + let oracle_drain_end = remove_index + .saturating_add(drain_len) + .min(oracle.len()) + .max(oracle_drain_start); + string.drain_interval_assign(UsizeCO::checked_from_start_len(remove_index, drain_len).unwrap()); + oracle.drain(oracle_drain_start..oracle_drain_end); + assert_eq!( + string + .to_vec() + .iter() + .copied() + .map(|value| value.code()) + .collect::>(), + oracle + ); +} + +proptest! { + #[test] + fn packed_access_slice_and_order_match_vec_oracles( + left in prop::collection::vec(0u8..=2, 0..=32), + right in prop::collection::vec(0u8..=2, 0..=32), + start in 0usize..40, + len in 1usize..40, + ) { + let string = packed(&left); + prop_assert_eq!(string.char_len(), left.len()); + let view = string.as_packed_str(); + for index in 0..=left.len() + 1 { + prop_assert_eq!(string.get(index), left.get(index).copied().map(symbol)); + prop_assert_eq!(view.get(index), left.get(index).copied().map(symbol)); + } + + let oracle_start = start.min(left.len()); + let oracle_end = start.saturating_add(len).min(left.len()).max(oracle_start); + let slice = string.slice(UsizeCO::checked_from_start_len(start, len).unwrap()); + prop_assert_eq!(slice.to_vec(), left[oracle_start..oracle_end].iter().copied().map(symbol).collect::>()); + + let right_string = packed(&right); + prop_assert_eq!(string.cmp(&right_string), left.cmp(&right)); + prop_assert_eq!(string.as_packed_str().cmp(&right_string.as_packed_str()), left.cmp(&right)); + } + + #[test] + fn packed_matching_matches_vec_sliding_windows( + haystack in prop::collection::vec(0u8..=2, 0..=32), + needle in prop::collection::vec(0u8..=2, 0..=16), + ) { + let haystack_string = packed(&haystack); + let needle_string = packed(&needle); + let haystack_view = haystack_string.as_packed_str(); + let needle_view = needle_string.as_packed_str(); + let max_start = haystack.len().saturating_sub(needle.len()); + let expected_find = (0..=max_start).find(|&start| matches_at(&haystack, start, &needle)); + let expected_rfind = (0..=max_start).rev().find(|&start| matches_at(&haystack, start, &needle)); + + prop_assert_eq!(haystack_view.find(needle_view), expected_find); + prop_assert_eq!(haystack_view.rfind(needle_view), expected_rfind); + prop_assert_eq!(haystack_view.contains(needle_view), expected_find.is_some()); + } + + #[test] + fn packed_edits_match_vec_splice_semantics( + initial in prop::collection::vec(0u8..=2, 0..=24), + replacement in prop::collection::vec(0u8..=2, 0..=12), + insert_index in 0usize..32, + remove_index in 0usize..32, + replace_start in 0usize..32, + ) { + let mut string = packed(&initial); + let mut oracle = initial.clone(); + + let oracle_insert_index = insert_index.min(oracle.len()); + let inserted = symbol(2); + string.insert(insert_index, inserted); + oracle.insert(oracle_insert_index, 2); + + if !oracle.is_empty() { + let remove_index = remove_index.min(oracle.len() - 1); + prop_assert_eq!(string.remove(remove_index).code(), oracle.remove(remove_index)); + } + + let oracle_replace_start = replace_start.min(oracle.len()); + let replace_end = oracle_replace_start + .saturating_add(replacement.len()) + .min(oracle.len()); + string.replace_assign(replace_start, &packed(&replacement)); + oracle.splice(oracle_replace_start..replace_end, replacement.iter().copied()); + prop_assert_eq!( + string.to_vec().iter().copied().map(|value| value.code()).collect::>(), + oracle.clone(), + ); + + let drain_len = insert_index.max(1); + let oracle_drain_start = remove_index.min(oracle.len()); + let oracle_drain_end = remove_index + .saturating_add(drain_len) + .min(oracle.len()) + .max(oracle_drain_start); + string.drain_interval_assign( + UsizeCO::checked_from_start_len(remove_index, drain_len).unwrap(), + ); + oracle.drain(oracle_drain_start..oracle_drain_end); + prop_assert_eq!( + string.to_vec().iter().copied().map(|value| value.code()).collect::>(), + oracle.clone(), + ); + } +} + +proptest! { + #[test] + fn three_and_seven_bit_access_slice_and_order_cross_word_boundaries( + left3 in prop::collection::vec(0u8..=7, 22..=128), + right3 in prop::collection::vec(0u8..=7, 22..=128), + left7 in prop::collection::vec(0u8..=127, 10..=128), + right7 in prop::collection::vec(0u8..=127, 10..=128), + start in 0usize..160, + len in 1usize..80, + ) { + assert_access_slice_order::(&left3, &right3, start, len, oct); + assert_access_slice_order::(&left7, &right7, start, len, wide); + } + + #[test] + fn three_and_seven_bit_matching_crosses_word_boundaries( + haystack3 in prop::collection::vec(0u8..=7, 22..=128), + needle3 in prop::collection::vec(0u8..=7, 0..=32), + haystack7 in prop::collection::vec(0u8..=127, 10..=128), + needle7 in prop::collection::vec(0u8..=127, 0..=16), + ) { + assert_matching::(&haystack3, &needle3, oct); + assert_matching::(&haystack7, &needle7, wide); + } + + #[test] + fn three_and_seven_bit_edits_cross_word_boundaries( + initial3 in prop::collection::vec(0u8..=7, 22..=128), + replacement3 in prop::collection::vec(0u8..=7, 0..=64), + initial7 in prop::collection::vec(0u8..=127, 10..=128), + replacement7 in prop::collection::vec(0u8..=127, 0..=32), + insert_index in 0usize..160, + remove_index in 0usize..160, + replace_start in 0usize..160, + ) { + assert_edits::( + &initial3, + &replacement3, + insert_index, + remove_index, + replace_start, + 2, + oct, + ); + assert_edits::( + &initial7, + &replacement7, + insert_index, + remove_index, + replace_start, + 2, + wide, + ); + } +} + +#[test] +fn packed_out_of_bounds_intervals_preserve_empty_semantics() { + let string = packed(&[0, 1, 2]); + let out_of_bounds = UsizeCO::checked_from_start_len(99, 1).unwrap(); + + assert!(string.slice(out_of_bounds).is_empty()); + + let mut drained = string.clone(); + drained.drain_interval_assign(out_of_bounds); + assert_eq!(drained, string); +} + +#[test] +fn packed_attribute_encoding_round_trips_through_packed_string() { + let string = PackedString::::from_chars([Symbol::Two, Symbol::Zero, Symbol::One]); + assert_eq!(string.get(0), Some(Symbol::Two)); + assert_eq!(string.get(1), Some(Symbol::Zero)); + assert_eq!(string.get(2), Some(Symbol::One)); +} + +#[derive(Clone, Copy, PartialEq, Eq)] +struct ManualSymbol; + +impl PackedChar<1> for ManualSymbol { + fn code(self) -> u8 { + 0 + } + + fn from_code(code: u8) -> Option { + (code == 0).then_some(Self) + } +} + +#[test] +fn manual_packed_char_implementations_remain_supported() { + assert_eq!( + PackedString::::from_chars([ManualSymbol]).char_len(), + 1 + ); +} + +#[test] +fn attribute_macro_generates_packed_char_impl() { + let string = + PackedString::::from_chars([PackedSymbol::One, PackedSymbol::Zero]); + assert_eq!(string.get(0), Some(PackedSymbol::One)); +} + +#[test] +fn packed_views_and_editing_remain_character_aligned() { + let mut string = PackedString::::from_chars([ + Symbol::Zero, + Symbol::One, + Symbol::Two, + Symbol::One, + ]); + + let view = string.as_packed_str(); + assert_eq!(view.char_len(), 4); + assert_eq!( + view.slice(UsizeCO::checked_from_start_len(1, 2).unwrap()) + .get(0), + Some(Symbol::One) + ); + assert_eq!(view.find(view.slice_until(1)), Some(0)); + assert!(view.contains(view.slice(UsizeCO::checked_from_start_len(1, 1).unwrap()))); + + let haystack_string = + PackedString::::from_chars([Symbol::Zero, Symbol::Zero, Symbol::One]); + let needle_string = PackedString::::from_chars([Symbol::Two]); + let haystack = haystack_string.as_packed_str(); + let needle = needle_string.as_packed_str(); + assert!(!haystack.contains(needle)); + assert_eq!(haystack.find(needle), None); + assert_eq!(haystack.rfind(needle), None); + assert!(!haystack.matches_at(1, needle)); + + string.insert(2, Symbol::Zero); + assert_eq!(string.remove(2), Symbol::Zero); + assert_eq!( + string.reverse().to_vec(), + vec![Symbol::One, Symbol::Two, Symbol::One, Symbol::Zero] + ); + string.retain(|symbol| symbol != Symbol::Two); + assert_eq!( + string.to_vec(), + vec![Symbol::Zero, Symbol::One, Symbol::One] + ); +} + +#[test] +fn from_bits_rejects_misaligned_and_unknown_codes() { + assert!(PackedString::::from_bits(BitString::from_iter([true])).is_none()); + assert!(PackedString::::from_bits(BitString::from_iter([true, true])).is_none()); +} + +#[test] +fn ordering_uses_packed_code_values() { + let one = PackedString::::from_chars([Symbol::One]); + let two = PackedString::::from_chars([Symbol::Two]); + assert!(one < two); + assert!(one.as_packed_str() < two.as_packed_str()); +} diff --git a/uv.lock b/uv.lock index 47ff850..2d9e4f8 100644 --- a/uv.lock +++ b/uv.lock @@ -3,6 +3,6 @@ revision = 3 requires-python = ">=3.11" [[package]] -name = "bit-string" +name = "bit-string-dev-tools" version = "0.1.0" source = { virtual = "." }