diff --git a/Cargo.lock b/Cargo.lock index 35d8034..8706156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,7 +147,7 @@ dependencies = [ [[package]] name = "bit-string" -version = "0.6.0" +version = "0.6.1" dependencies = [ "bit-string-derive", "bitvec_simd", @@ -163,7 +163,7 @@ dependencies = [ [[package]] name = "bit-string-derive" -version = "0.6.0" +version = "0.6.1" dependencies = [ "proc-macro-crate", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 3150b2e..d015105 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ exclude = ["/benches", "/src/**/tests_for_*", "/tests", ".github/"] members = ["derive"] [workspace.package] -version = "0.6.0" +version = "0.6.1" [features] characterization = [] @@ -128,6 +128,10 @@ harness = false name = "packed_construction" harness = false +[[bench]] +name = "packed_thresholds" +harness = false + [[bench]] name = "packed_access" harness = false @@ -136,6 +140,10 @@ harness = false name = "packed_iteration" harness = false +[[bench]] +name = "packed_conversion" +harness = false + [[bench]] name = "packed_editing" harness = false @@ -152,6 +160,10 @@ harness = false name = "bit_ops_shl_assign" harness = false +[[bench]] +name = "bit_ops_shr_assign" +harness = false + [[bench]] name = "hash" harness = false diff --git a/benches/bit_ops_shr_assign.rs b/benches/bit_ops_shr_assign.rs new file mode 100644 index 0000000..c144651 --- /dev/null +++ b/benches/bit_ops_shr_assign.rs @@ -0,0 +1,63 @@ +use bit_string::BitString; +use divan::{Bencher, black_box}; + +fn main() { + divan::main(); +} + +// Preparation is outside timing. Repeated shifts reuse one allocation, so +// they measure steady-state throughput separately from a single operation. +fn assign(bencher: Bencher, len: usize, amount: usize, count: usize) { + let input = make_bit_string(len); + bencher.with_inputs(|| input.clone()).bench_refs(|bits| { + for _ in 0..count { + bits.shr_assign(black_box(amount)); + } + black_box(&*bits); + }); +} + +fn owned(bencher: Bencher, len: usize, amount: usize) { + let input = make_bit_string(len); + // Output allocation is part of the owned operation. + bencher.bench(|| black_box(black_box(&input).shr(black_box(amount)))); +} + +fn make_bit_string(len: usize) -> BitString { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state & 1 != 0 + }) + .collect() +} + +macro_rules! define_shift { + ($case:ident, $amount:literal) => { + mod $case { + use super::*; + + #[divan::bench(args = [4096, 65_536, 1_048_576])] + fn assign(bencher: Bencher, len: usize) { + super::assign(bencher, len, $amount, 1); + } + + #[divan::bench(args = [4096, 65_536, 1_048_576])] + fn repeated_256(bencher: Bencher, len: usize) { + super::assign(bencher, len, $amount, 256); + } + + #[divan::bench(args = [4096, 65_536, 1_048_576])] + fn owned(bencher: Bencher, len: usize) { + super::owned(bencher, len, $amount); + } + } + }; +} + +define_shift!(by_1, 1); +define_shift!(by_17, 17); +define_shift!(by_65, 65); diff --git a/benches/packed_competitors.rs b/benches/packed_competitors.rs index deb2d95..51613e7 100644 --- a/benches/packed_competitors.rs +++ b/benches/packed_competitors.rs @@ -5,8 +5,8 @@ use compressed_intvec::fixed::UFixedVec; use divan::{Bencher, black_box}; use grit_bitvec::{TypedBitElem, TypedBitVec}; use support::{ - aligned_index, aligned_index_for, codes, cross_word_index, cross_word_index_for, indices, - unaligned_index, unaligned_index_for, + Code, aligned_index, aligned_index_for, codes, cross_word_index, cross_word_index_for, indices, + packed, unaligned_index, unaligned_index_for, }; use sux::bits::BitFieldVec; use value_traits::slices::{SliceByValue, SliceByValueMut}; @@ -309,6 +309,89 @@ where }); } +// Batch timings include every insertion and any capacity growth it triggers. +// Keep the batch no larger than the initial length: truncating a cloned packed +// input then retains enough capacity even with BitString's lazy shrink policy. +const INSERT_BATCH_LEN: usize = 256; + +fn insert_reserved_packed(bencher: Bencher, len: usize) { + insert_sequence_packed::(bencher, len, 1, true); +} + +fn insert_repeated_packed(bencher: Bencher, len: usize) { + insert_sequence_packed::(bencher, len, len.min(INSERT_BATCH_LEN), false); +} + +fn insert_repeated_reserved_packed(bencher: Bencher, len: usize) { + insert_sequence_packed::(bencher, len, len.min(INSERT_BATCH_LEN), true); +} + +fn insert_sequence_packed( + bencher: Bencher, + len: usize, + count: usize, + reserve: bool, +) { + let input = packed::(&codes(BITS, len + if reserve { count } else { 0 })); + let index = unaligned_index::(len); + bencher + .with_inputs(|| { + let mut value = input.clone(); + // Clone before truncating: cloning the shorter value would discard + // the spare allocation. Setup is excluded from the timed region. + value.truncate(len); + value + }) + .bench_local_refs(|value| { + // A fixed insertion point keeps shifting the original suffix plus + // the characters inserted earlier in this batch. + for _ in 0..count { + value.insert(index, Code(0)); + } + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn insert_reserved_grit + 'static>(bencher: Bencher, len: usize) { + insert_sequence_grit::(bencher, len, 1, true); +} + +fn insert_repeated_grit + 'static>(bencher: Bencher, len: usize) { + insert_sequence_grit::(bencher, len, len.min(INSERT_BATCH_LEN), false); +} + +fn insert_repeated_reserved_grit + 'static>( + bencher: Bencher, + len: usize, +) { + insert_sequence_grit::(bencher, len, len.min(INSERT_BATCH_LEN), true); +} + +fn insert_sequence_grit + 'static>( + bencher: Bencher, + len: usize, + count: usize, + reserve: bool, +) { + let width = grit_width::(); + let input = codes(width, len); + let index = unaligned_index_for(usize::from(width), len); + bencher + .with_inputs(|| { + let mut value = TypedBitVec::::with_capacity(len + if reserve { count } else { 0 }); + value.append_iter(input.iter().copied()).unwrap(); + value + }) + .bench_local_refs(|value| { + for _ in 0..count { + value.insert(index, 0).unwrap(); + } + black_box(&*value); + black_box(value.len()) + }); +} + fn remove_grit(bencher: Bencher, len: usize) where T: TypedBitElem + 'static, @@ -376,6 +459,9 @@ macro_rules! define_packed_case { benchmark!("editing/pop", pop_fixed, "compressed_intvec"); benchmark!("editing/extend_bulk_25pct", extend_fixed, "compressed_intvec"); benchmark!("editing/insert_position/unaligned", insert_fixed, "compressed_intvec"); + benchmark!("editing/insert_reserved/unaligned", insert_reserved_packed, "ours_packed_string"); + benchmark!("editing/insert_repeated/unaligned", insert_repeated_packed, "ours_packed_string"); + benchmark!("editing/insert_repeated_reserved/unaligned", insert_repeated_reserved_packed, "ours_packed_string"); benchmark!("editing/remove_position/word_boundary_or_fallback", remove_fixed, "compressed_intvec"); benchmark!("construction/from_chars", construct_sux, "sux"); @@ -422,6 +508,9 @@ macro_rules! define_grit_case { benchmark!("editing/push_boundary_probe", push_grit); benchmark!("editing/pop", pop_grit); benchmark!("editing/insert_position/unaligned", insert_grit); + benchmark!("editing/insert_reserved/unaligned", insert_reserved_grit); + benchmark!("editing/insert_repeated/unaligned", insert_repeated_grit); + benchmark!("editing/insert_repeated_reserved/unaligned", insert_repeated_reserved_grit); benchmark!("editing/remove_position/word_boundary_or_fallback", remove_grit); benchmark!("editing/extend_bulk_25pct", append_grit); } diff --git a/benches/packed_conversion.rs b/benches/packed_conversion.rs new file mode 100644 index 0000000..9f75637 --- /dev/null +++ b/benches/packed_conversion.rs @@ -0,0 +1,106 @@ +#[path = "packed_support/mod.rs"] +mod support; + +use divan::{Bencher, black_box}; +use support::{codes, packed}; + +fn main() { + divan::main(); +} + +fn collect_via_iter(bencher: Bencher, len: usize) { + let input = codes(BITS, len); + let value = packed::(&input); + bencher.bench(|| black_box(&value).iter().collect::>()); +} + +fn convert_to_vec(bencher: Bencher, len: usize) { + let input = codes(BITS, len); + let value = packed::(&input); + bencher.bench(|| black_box(&value).to_vec()); +} + +macro_rules! define_case { + ($case:ident, $bits:literal, $len:literal) => { + mod $case { + use super::*; + + #[divan::bench(name = concat!("packed_conversion/to_vec/", stringify!($case), "/iter_collect"))] + fn iter_collect(bencher: Bencher) { + super::collect_via_iter::<$bits>(bencher, $len); + } + + #[divan::bench(name = concat!("packed_conversion/to_vec/", stringify!($case), "/to_vec"))] + fn to_vec(bencher: Bencher) { + super::convert_to_vec::<$bits>(bencher, $len); + } + } + }; +} + +crate::for_each_packed_bench_case!(define_case); + +macro_rules! define_boundary_case { + ($case:ident, $width:ident, $bits:literal, $len:literal) => { + mod $case { + use super::*; + + #[divan::bench(name = concat!("packed_conversion/to_vec_boundary/", stringify!($width), "/", stringify!($case), "/iter_collect"))] + fn iter_collect(bencher: Bencher) { + super::super::collect_via_iter::<$bits>(bencher, $len); + } + + #[divan::bench(name = concat!("packed_conversion/to_vec_boundary/", stringify!($width), "/", stringify!($case), "/to_vec"))] + fn to_vec(bencher: Bencher) { + super::super::convert_to_vec::<$bits>(bencher, $len); + } + } + }; +} + +macro_rules! define_width_boundaries { + ($width:ident, $bits:literal; $($case:ident: $len:literal),+ $(,)?) => { + mod $width { + use super::*; + + $(define_boundary_case!($case, $width, $bits, $len);)+ + } + }; +} + +mod boundaries { + use super::*; + + define_width_boundaries!(bits_1, 1; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_2, 2; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_3, 3; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_4, 4; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_5, 5; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_6, 6; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_7, 7; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, + ); + define_width_boundaries!(bits_8, 8; + len_16: 16, len_31: 31, len_32: 32, len_63: 63, len_64: 64, + len_65: 65, len_95: 95, len_96: 96, len_127: 127, len_128: 128, len_129: 129, + ); +} diff --git a/benches/packed_editing.rs b/benches/packed_editing.rs index 6d739c6..6fca073 100644 --- a/benches/packed_editing.rs +++ b/benches/packed_editing.rs @@ -1,6 +1,7 @@ #[path = "packed_support/mod.rs"] mod support; +use bit_string::PackedString; use divan::{Bencher, black_box}; use support::{Code, aligned_index, codes, cross_word_index, packed, unaligned_index}; @@ -32,23 +33,83 @@ fn set_middle_vec(bencher: Bencher, len: usize) { } fn push_packed(bencher: Bencher, len: usize) { + push_packed_value::(bencher, len, 0); +} + +fn push_packed_value(bencher: Bencher, len: usize, code: u8) { let input = packed::(&codes(BITS, len)); bencher.with_inputs(|| input.clone()).bench_refs(|value| { - value.push(Code(0)); + value.push(Code(code)); black_box(&*value); black_box(value.char_len()) }); } fn push_amortized_packed(bencher: Bencher, len: usize) { + push_amortized_packed_value::(bencher, len, 0); +} + +fn push_amortized_packed_value(bencher: Bencher, len: usize, code: u8) { let input = packed::(&codes(BITS, len - 1)); bencher.with_inputs(|| input.clone()).bench_refs(|value| { - value.push(Code(0)); + value.push(Code(code)); black_box(&*value); black_box(value.char_len()) }); } +fn push_existing_word(bencher: Bencher, code: u8) { + push_packed_value::(bencher, 1, code); +} + +// Build the backing word first, then truncate so the timed push grows the +// logical word slice without triggering a heap reallocation. +fn push_word_growth(bencher: Bencher, code: u8) { + const PREFIX_LEN: usize = 64; + let input = packed::(&codes(BITS, PREFIX_LEN + 1)); + bencher + .with_inputs(|| { + let mut value = input.clone(); + value.truncate(PREFIX_LEN); + value + }) + .bench_refs(|value| { + value.push(Code(code)); + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn push_cross_word(bencher: Bencher, code: u8) { + let prefix_len = match BITS { + 3 => 21, + 5 => 12, + 6 => 10, + 7 => 9, + _ => unreachable!("cross-word character only exists for irregular widths"), + }; + let input = packed::(&codes(BITS, prefix_len + 1)); + bencher + .with_inputs(|| { + let mut value = input.clone(); + value.truncate(prefix_len); + value + }) + .bench_refs(|value| { + value.push(Code(code)); + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn max_code(bits: u8) -> u8 { + if bits == 8 { + u8::MAX + } else { + ((1u16 << bits) - 1) as u8 + } +} + fn push_vec(bencher: Bencher, len: usize) { let input = codes(BITS, len); bencher.with_inputs(|| input.clone()).bench_refs(|value| { @@ -58,6 +119,35 @@ fn push_vec(bencher: Bencher, len: usize) { }); } +fn push_existing_word_vec(bencher: Bencher, code: u8) { + bencher + .with_inputs(|| { + let mut value = vec![0u8]; + value.reserve(1); + value + }) + .bench_refs(|value| { + value.push(code); + black_box(&*value); + black_box(value.len()) + }); +} + +fn push_word_growth_vec(bencher: Bencher, code: u8) { + let input = vec![0u8; 65]; + bencher + .with_inputs(|| { + let mut value = input.clone(); + value.truncate(64); + value + }) + .bench_refs(|value| { + value.push(code); + black_box(&*value); + black_box(value.len()) + }); +} + fn push_amortized_vec(bencher: Bencher, len: usize) { let input = codes(BITS, len - 1); bencher @@ -132,7 +222,15 @@ fn remove_middle_vec(bencher: Bencher, len: usize) { } fn extend_packed(bencher: Bencher, len: usize) { - let input = packed::(&codes(BITS, len)); + extend_packed_at::(bencher, len, len); +} + +fn extend_packed_unaligned(bencher: Bencher, len: usize) { + extend_packed_at::(bencher, len, len + 1); +} + +fn extend_packed_at(bencher: Bencher, len: usize, prefix_len: usize) { + let input = packed::(&codes(BITS, prefix_len)); let extension = codes(BITS, len / 4); bencher.with_inputs(|| input.clone()).bench_refs(|value| { value.extend(extension.iter().copied().map(Code)); @@ -141,6 +239,72 @@ fn extend_packed(bencher: Bencher, len: usize) { }); } +fn extend_from_chars_concat(bencher: Bencher, len: usize) { + extend_from_chars_concat_at::(bencher, len, len); +} + +fn extend_from_chars_concat_unaligned(bencher: Bencher, len: usize) { + extend_from_chars_concat_at::(bencher, len, len + 1); +} + +fn extend_from_chars_concat_at(bencher: Bencher, len: usize, prefix_len: usize) { + let input = packed::(&codes(BITS, prefix_len)); + let extension = codes(BITS, len / 4); + bencher.with_inputs(|| input.clone()).bench_refs(|value| { + let suffix = packed::(&extension); + value.push_packed_string(&suffix); + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn extend_with_length(bencher: Bencher, extension_len: usize, concat: bool) { + let input = extend_length_input::(extension_len); + let extension = codes(BITS, extension_len); + bencher + .with_inputs(|| extend_length_bench_input(&input)) + .bench_refs(|value| { + if concat { + let suffix = packed::(&extension); + value.push_packed_string(&suffix); + } else { + value.extend(extension.iter().copied().map(Code)); + } + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn extend_per_character_with_length(bencher: Bencher, extension_len: usize) { + let input = extend_length_input::(extension_len); + let extension = codes(BITS, extension_len); + bencher + .with_inputs(|| extend_length_bench_input(&input)) + .bench_refs(|value| { + for code in extension.iter().copied() { + value.push(Code(code)); + } + black_box(&*value); + black_box(value.char_len()) + }); +} + +fn extend_length_input(extension_len: usize) -> PackedString { + const PREFIX_LEN: usize = 1_048_576; + packed::(&codes(BITS, PREFIX_LEN + extension_len)) +} + +fn extend_length_bench_input( + input: &PackedString, +) -> PackedString { + const PREFIX_LEN: usize = 1_048_576; + // The source was built with spare logical words. Clone before truncating so + // every candidate gets the same retained capacity outside the timed region. + let mut value = input.clone(); + value.truncate(PREFIX_LEN); + value +} + fn extend_vec(bencher: Bencher, len: usize) { let input = codes(BITS, len); let extension = codes(BITS, len / 4); @@ -189,12 +353,177 @@ macro_rules! define_case { pair!("set_position/aligned", set_middle_packed, set_middle_vec); pair!("push_boundary_probe", push_packed, push_vec); pair!("push_amortized", push_amortized_packed, push_amortized_vec); + #[divan::bench(name = concat!("packed_editing/push_boundary_probe_nonzero/", stringify!($case), "/ours_packed_string"))] + fn push_boundary_probe_nonzero(bencher: Bencher) { + crate::push_packed_value::<$bits>(bencher, $len, crate::max_code($bits)); + } + + #[divan::bench(name = concat!("packed_editing/push_amortized_nonzero/", stringify!($case), "/ours_packed_string"))] + fn push_amortized_nonzero(bencher: Bencher) { + crate::push_amortized_packed_value::<$bits>(bencher, $len, crate::max_code($bits)); + } pair!("pop", pop_packed, pop_vec); pair!("insert_position/unaligned", insert_middle_packed, insert_middle_vec); pair!("remove_position/word_boundary_or_fallback", remove_middle_packed, remove_middle_vec); pair!("extend_bulk_25pct", extend_packed, extend_vec); + #[divan::bench(name = concat!("packed_editing/extend_bulk_25pct_unaligned/", stringify!($case), "/ours_packed_string"))] + fn extend_bulk_25pct_unaligned(bencher: Bencher) { + crate::extend_packed_unaligned::<$bits>(bencher, $len); + } + #[divan::bench(name = concat!("packed_editing/extend_from_chars_concat/", stringify!($case), "/ours_packed_string"))] + fn extend_from_chars_concat(bencher: Bencher) { + crate::extend_from_chars_concat::<$bits>(bencher, $len); + } + #[divan::bench(name = concat!("packed_editing/extend_from_chars_concat_unaligned/", stringify!($case), "/ours_packed_string"))] + fn extend_from_chars_concat_unaligned(bencher: Bencher) { + crate::extend_from_chars_concat_unaligned::<$bits>(bencher, $len); + } } }; } crate::for_each_packed_bench_case!(define_case); + +macro_rules! define_extend_length_case { + ($case:ident, $bits:literal) => { + mod $case { + use super::*; + + #[divan::bench(args = [1, 16, 63, 64, 262144])] + fn streaming_batches(bencher: Bencher, extension_len: usize) { + crate::extend_with_length::<$bits>(bencher, extension_len, false); + } + + #[divan::bench(args = [1, 16, 63, 64, 262144])] + fn suffix_concat(bencher: Bencher, extension_len: usize) { + crate::extend_with_length::<$bits>(bencher, extension_len, true); + } + + #[divan::bench(args = [1, 16, 63, 64, 262144])] + fn per_character_preallocated(bencher: Bencher, extension_len: usize) { + crate::extend_per_character_with_length::<$bits>(bencher, extension_len); + } + } + }; +} + +define_extend_length_case!(extend_length_bits_1, 1); +define_extend_length_case!(extend_length_bits_2, 2); +define_extend_length_case!(extend_length_bits_3, 3); +define_extend_length_case!(extend_length_bits_4, 4); +define_extend_length_case!(extend_length_bits_5, 5); +define_extend_length_case!(extend_length_bits_6, 6); +define_extend_length_case!(extend_length_bits_7, 7); +define_extend_length_case!(extend_length_bits_8, 8); + +// Adjacent lengths exercise backing-word growth/shrink and partial tails. +macro_rules! define_edit_boundary_case { + ($case:ident, $bits:literal) => { + mod $case { + use super::*; + + #[divan::bench(args = [63, 64, 65, 127, 128, 129])] + fn insert_middle(bencher: Bencher, len: usize) { + crate::insert_middle_packed::<$bits>(bencher, len); + } + + #[divan::bench(args = [63, 64, 65, 127, 128, 129])] + fn remove_middle(bencher: Bencher, len: usize) { + crate::remove_middle_packed::<$bits>(bencher, len); + } + } + }; +} + +define_edit_boundary_case!(edit_boundary_bits_1, 1); +define_edit_boundary_case!(edit_boundary_bits_2, 2); +define_edit_boundary_case!(edit_boundary_bits_3, 3); +define_edit_boundary_case!(edit_boundary_bits_4, 4); +define_edit_boundary_case!(edit_boundary_bits_5, 5); +define_edit_boundary_case!(edit_boundary_bits_6, 6); +define_edit_boundary_case!(edit_boundary_bits_7, 7); +define_edit_boundary_case!(edit_boundary_bits_8, 8); + +macro_rules! define_push_shape_case { + ($case:ident, $bits:literal) => { + mod $case { + use super::*; + + #[divan::bench(name = concat!("packed_editing/push_existing_word_zero/", stringify!($case), "/ours_packed_string"))] + fn existing_word_zero(bencher: Bencher) { + crate::push_existing_word::<$bits>(bencher, 0); + } + + #[divan::bench(name = concat!("packed_editing/push_existing_word_nonzero/", stringify!($case), "/ours_packed_string"))] + fn existing_word_nonzero(bencher: Bencher) { + crate::push_existing_word::<$bits>(bencher, crate::max_code($bits)); + } + + #[divan::bench(name = concat!("packed_editing/push_word_growth_zero/", stringify!($case), "/ours_packed_string"))] + fn word_growth_zero(bencher: Bencher) { + crate::push_word_growth::<$bits>(bencher, 0); + } + + #[divan::bench(name = concat!("packed_editing/push_word_growth_nonzero/", stringify!($case), "/ours_packed_string"))] + fn word_growth_nonzero(bencher: Bencher) { + crate::push_word_growth::<$bits>(bencher, crate::max_code($bits)); + } + } + }; +} + +macro_rules! define_cross_word_case { + ($case:ident, $bits:literal) => { + mod $case { + use super::*; + + #[divan::bench(name = concat!("packed_editing/push_cross_word_zero/", stringify!($case), "/ours_packed_string"))] + fn cross_word_zero(bencher: Bencher) { + crate::push_cross_word::<$bits>(bencher, 0); + } + + #[divan::bench(name = concat!("packed_editing/push_cross_word_nonzero/", stringify!($case), "/ours_packed_string"))] + fn cross_word_nonzero(bencher: Bencher) { + crate::push_cross_word::<$bits>(bencher, crate::max_code($bits)); + } + } + }; +} + +define_push_shape_case!(push_shape_bits_1, 1); +define_push_shape_case!(push_shape_bits_2, 2); +define_push_shape_case!(push_shape_bits_3, 3); +define_push_shape_case!(push_shape_bits_4, 4); +define_push_shape_case!(push_shape_bits_5, 5); +define_push_shape_case!(push_shape_bits_6, 6); +define_push_shape_case!(push_shape_bits_7, 7); +define_push_shape_case!(push_shape_bits_8, 8); + +mod push_vec_bits_8 { + use super::*; + + #[divan::bench(name = "packed_editing/push_existing_word_zero/push_shape_bits_8/vec_u8")] + fn existing_word_zero(bencher: Bencher) { + crate::push_existing_word_vec(bencher, 0); + } + + #[divan::bench(name = "packed_editing/push_existing_word_nonzero/push_shape_bits_8/vec_u8")] + fn existing_word_nonzero(bencher: Bencher) { + crate::push_existing_word_vec(bencher, u8::MAX); + } + + #[divan::bench(name = "packed_editing/push_word_growth_zero/push_shape_bits_8/vec_u8")] + fn word_growth_zero(bencher: Bencher) { + crate::push_word_growth_vec(bencher, 0); + } + + #[divan::bench(name = "packed_editing/push_word_growth_nonzero/push_shape_bits_8/vec_u8")] + fn word_growth_nonzero(bencher: Bencher) { + crate::push_word_growth_vec(bencher, u8::MAX); + } +} + +define_cross_word_case!(cross_word_bits_3, 3); +define_cross_word_case!(cross_word_bits_5, 5); +define_cross_word_case!(cross_word_bits_6, 6); +define_cross_word_case!(cross_word_bits_7, 7); diff --git a/benches/packed_support/mod.rs b/benches/packed_support/mod.rs index e234fd2..e1feb95 100644 --- a/benches/packed_support/mod.rs +++ b/benches/packed_support/mod.rs @@ -39,6 +39,16 @@ macro_rules! for_each_packed_case { $macro!(bits_4_len_1024, 4, 1_024); $macro!(bits_4_len_65536, 4, 65_536); $macro!(bits_4_len_1048576, 4, 1_048_576); + $macro!(bits_5_len_16, 5, 16); + $macro!(bits_5_len_64, 5, 64); + $macro!(bits_5_len_1024, 5, 1_024); + $macro!(bits_5_len_65536, 5, 65_536); + $macro!(bits_5_len_1048576, 5, 1_048_576); + $macro!(bits_6_len_16, 6, 16); + $macro!(bits_6_len_64, 6, 64); + $macro!(bits_6_len_1024, 6, 1_024); + $macro!(bits_6_len_65536, 6, 65_536); + $macro!(bits_6_len_1048576, 6, 1_048_576); $macro!(bits_7_len_16, 7, 16); $macro!(bits_7_len_64, 7, 64); $macro!(bits_7_len_1024, 7, 1_024); @@ -75,6 +85,16 @@ macro_rules! for_each_grit_case { $macro!(bits_4_len_1024, 4, 1_024, grit_bitvec::u8_as_u4); $macro!(bits_4_len_65536, 4, 65_536, grit_bitvec::u8_as_u4); $macro!(bits_4_len_1048576, 4, 1_048_576, grit_bitvec::u8_as_u4); + $macro!(bits_5_len_16, 5, 16, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_64, 5, 64, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_1024, 5, 1_024, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_65536, 5, 65_536, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_1048576, 5, 1_048_576, grit_bitvec::u8_as_u5); + $macro!(bits_6_len_16, 6, 16, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_64, 6, 64, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_1024, 6, 1_024, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_65536, 6, 65_536, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_1048576, 6, 1_048_576, grit_bitvec::u8_as_u6); $macro!(bits_7_len_16, 7, 16, grit_bitvec::u8_as_u7); $macro!(bits_7_len_64, 7, 64, grit_bitvec::u8_as_u7); $macro!(bits_7_len_1024, 7, 1_024, grit_bitvec::u8_as_u7); @@ -98,6 +118,10 @@ macro_rules! for_each_packed_regression_case { $macro!(bits_3_len_1048576, 3, 1_048_576); $macro!(bits_4_len_64, 4, 64); $macro!(bits_4_len_1048576, 4, 1_048_576); + $macro!(bits_5_len_1024, 5, 1_024); + $macro!(bits_5_len_1048576, 5, 1_048_576); + $macro!(bits_6_len_1024, 6, 1_024); + $macro!(bits_6_len_1048576, 6, 1_048_576); $macro!(bits_7_len_1024, 7, 1_024); $macro!(bits_7_len_1048576, 7, 1_048_576); $macro!(bits_8_len_64, 8, 64); @@ -116,6 +140,10 @@ macro_rules! for_each_grit_regression_case { $macro!(bits_3_len_1048576, 3, 1_048_576, grit_bitvec::u8_as_u3); $macro!(bits_4_len_64, 4, 64, grit_bitvec::u8_as_u4); $macro!(bits_4_len_1048576, 4, 1_048_576, grit_bitvec::u8_as_u4); + $macro!(bits_5_len_1024, 5, 1_024, grit_bitvec::u8_as_u5); + $macro!(bits_5_len_1048576, 5, 1_048_576, grit_bitvec::u8_as_u5); + $macro!(bits_6_len_1024, 6, 1_024, grit_bitvec::u8_as_u6); + $macro!(bits_6_len_1048576, 6, 1_048_576, grit_bitvec::u8_as_u6); $macro!(bits_7_len_1024, 7, 1_024, grit_bitvec::u8_as_u7); $macro!(bits_7_len_1048576, 7, 1_048_576, grit_bitvec::u8_as_u7); }; diff --git a/benches/packed_thresholds.rs b/benches/packed_thresholds.rs new file mode 100644 index 0000000..2d08e80 --- /dev/null +++ b/benches/packed_thresholds.rs @@ -0,0 +1,137 @@ +//! Compare packed construction with and without AVX2 enabled. +//! +//! Run this benchmark without `-C target-feature=+avx2` for the scalar +//! baseline, then with that target feature to measure the AVX2 dispatch. +//! Widths 2 and 8 are intentionally omitted because their AVX2 prototypes did +//! not show a stable construction-speed improvement; see the AVX2 backend +//! documentation. + +#[path = "packed_support/mod.rs"] +mod support; + +use divan::{Bencher, black_box}; +use support::{codes, packed}; + +fn main() { + divan::main(); +} + +macro_rules! define_width { + ($module:ident, $bits:literal, [$($name:ident => $len:literal),+ $(,)?]) => { + mod $module { + use super::*; + + $( + #[divan::bench( + name = concat!( + "packed_thresholds/bits_", + stringify!($bits), + "/len_", + stringify!($len) + ) + )] + fn $name(bencher: Bencher) { + let input = codes($bits, $len); + bencher.bench(|| black_box(packed::<$bits>(&input))); + } + )+ + } + }; +} + +define_width!( + bits_1, + 1, + [ + len_64 => 64, + len_128 => 128, + len_192 => 192, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_3, + 3, + [ + len_64 => 64, + len_128 => 128, + len_192 => 192, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_4, + 4, + [ + len_16 => 16, + len_32 => 32, + len_48 => 48, + len_64 => 64, + len_96 => 96, + len_128 => 128, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_5, + 5, + [ + len_64 => 64, + len_128 => 128, + len_192 => 192, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_6, + 6, + [ + len_16 => 16, + len_32 => 32, + len_48 => 48, + len_64 => 64, + len_96 => 96, + len_128 => 128, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); +define_width!( + bits_7, + 7, + [ + len_64 => 64, + len_128 => 128, + len_192 => 192, + len_256 => 256, + len_512 => 512, + len_1024 => 1024, + len_4096 => 4096, + len_16384 => 16384, + len_1048576 => 1_048_576, + ] +); diff --git a/src/bit_string/impls_for_construction/impls_for_from_words.rs b/src/bit_string/impls_for_construction/impls_for_from_words.rs index 9ecb30d..1467e18 100644 --- a/src/bit_string/impls_for_construction/impls_for_from_words.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_words.rs @@ -1,3 +1,5 @@ +use alloc::vec::Vec; + use super::*; use crate::funcs_for_bits::*; use crate::traits::*; @@ -22,6 +24,24 @@ impl BitString { bit_len: len, }) } + + /// Constructs a bit string by taking ownership of packed little-endian words. + /// + /// The input must contain exactly enough words for `len`. Unused high bits + /// in the last word are masked out before the vector is adopted. + #[inline] + pub(crate) fn from_owned_words(mut words: Vec, len: usize) -> Option { + if words.len() != word_len(len) { + return None; + } + + words.mask_unused_bits(len); + + Some(Self { + words, + bit_len: len, + }) + } } #[cfg(test)] diff --git a/src/bit_string/impls_for_construction/impls_for_from_words/tests_for_from_words.rs b/src/bit_string/impls_for_construction/impls_for_from_words/tests_for_from_words.rs index 2c56314..dd9672e 100644 --- a/src/bit_string/impls_for_construction/impls_for_from_words/tests_for_from_words.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_words/tests_for_from_words.rs @@ -1,4 +1,5 @@ use alloc::string::ToString; +use alloc::vec; use crate::BitString; @@ -12,6 +13,14 @@ fn constructs_empty_from_empty_words() { assert_eq!(bits.to_string(), ""); } +#[test] +fn constructs_from_owned_words() { + let bits = BitString::from_owned_words(vec![0b1011], 4).unwrap(); + + assert_eq!(bits.bit_len(), 4); + assert_eq!(bits.to_string(), "1101"); +} + #[test] fn constructs_single_partial_word() { let bits = BitString::from_words(&[0b1011], 4).unwrap(); @@ -31,6 +40,14 @@ fn masks_unused_high_bits_in_last_word() { assert!(bits.is_all_ones()); } +#[test] +fn masks_unused_high_bits_in_owned_last_word() { + let bits = BitString::from_owned_words(vec![u64::MAX], 3).unwrap(); + + assert_eq!(bits.words(), &[0b111]); + assert_eq!(bits.count_ones(), 3); +} + #[test] fn constructs_word_aligned_bits() { let bits = BitString::from_words(&[u64::MAX], 64).unwrap(); @@ -56,6 +73,8 @@ fn constructs_across_multiple_words() { fn rejects_too_few_words() { assert!(BitString::from_words(&[], 1).is_none()); assert!(BitString::from_words(&[0], 65).is_none()); + assert!(BitString::from_owned_words(vec![], 1).is_none()); + assert!(BitString::from_owned_words(vec![0], 65).is_none()); } #[test] @@ -63,4 +82,7 @@ fn rejects_too_many_words() { assert!(BitString::from_words(&[0], 0).is_none()); assert!(BitString::from_words(&[0, 0], 64).is_none()); assert!(BitString::from_words(&[0, 0, 0], 65).is_none()); + assert!(BitString::from_owned_words(vec![0], 0).is_none()); + assert!(BitString::from_owned_words(vec![0, 0], 64).is_none()); + assert!(BitString::from_owned_words(vec![0, 0, 0], 65).is_none()); } diff --git a/src/bit_string/impls_for_editing/impls_for_concat.rs b/src/bit_string/impls_for_editing/impls_for_concat.rs index 92e0731..a229070 100644 --- a/src/bit_string/impls_for_editing/impls_for_concat.rs +++ b/src/bit_string/impls_for_editing/impls_for_concat.rs @@ -49,6 +49,26 @@ impl BitString { self.bit_len = new_len; } + /// Appends a word-aligned packed block whose final word may be partial. + pub(crate) fn push_packed_words(&mut self, rhs_words: &[u64], rhs_bit_len: usize) { + if rhs_bit_len == 0 { + return; + } + + assert_eq!(rhs_words.len(), word_len(rhs_bit_len)); + + let old_bit_len = self.bit_len; + let new_bit_len = old_bit_len + .checked_add(rhs_bit_len) + .expect("bit string length overflow"); + self.words.resize(word_len(new_bit_len), 0); + rhs_words + .copy_bits(0, rhs_bit_len) + .paste_to(&mut self.words, old_bit_len); + self.bit_len = new_bit_len; + self.words.mask_unused_bits(new_bit_len); + } + /// Inserts the bits of `rhs` at `index`, shifting the tail right. /// /// `index` is clamped to `[0, self.bit_len()]` — an out-of-bounds index diff --git a/src/bit_string/impls_for_editing/impls_for_insert_remove.rs b/src/bit_string/impls_for_editing/impls_for_insert_remove.rs index 01eddcb..6c614cc 100644 --- a/src/bit_string/impls_for_editing/impls_for_insert_remove.rs +++ b/src/bit_string/impls_for_editing/impls_for_insert_remove.rs @@ -4,6 +4,79 @@ use crate::traits::*; use super::*; impl BitString { + /// Inserts the low `width` bits of `value`, shifting the suffix once. + /// + /// `width` must be at most one word. The index is clamped to the end; + /// zero width is a no-op. Growth is checked before modifying the payload. + pub(crate) fn insert_bits(&mut self, index: usize, value: u64, width: usize) { + assert!( + width <= WORD_BITS, + "cannot insert more than one word of bits" + ); + let index = index.min(self.bit_len); + if index == self.bit_len { + self.push_bits(value, width); + return; + } + if width == 0 { + return; + } + + let new_len = self + .bit_len + .checked_add(width) + .expect("bit string length overflow"); + // Allocate before changing existing bits, retaining spare capacity. + self.words.resize(word_len(new_len), 0); + + let first_word = index / WORD_BITS; + let prefix_mask = crate::low_mask(index % WORD_BITS); + let prefix = self.words[first_word] & prefix_mask; + // Clear the boundary prefix before shifting so it cannot enter the gap. + // The word-level shift preserves overlapping input. + self.words[first_word] &= !prefix_mask; + self.words[first_word..].shl_assign(new_len - first_word * WORD_BITS, width); + self.words[first_word] |= prefix; + // The shift leaves [index, index + width) zero, as required by OR writes. + self.words.write_word_at::(index, value, width); + self.bit_len = new_len; + } + + /// Removes `width` bits at `index`, shifting the suffix once. + /// + /// `width` must be at most one word. Zero width is a no-op; otherwise + /// the entire range must exist. Invalid arguments panic before mutation. + pub(crate) fn remove_bits(&mut self, index: usize, width: usize) { + assert!( + width <= WORD_BITS, + "cannot remove more than one word of bits" + ); + if width == 0 { + return; + } + assert!( + index <= self.bit_len && width <= self.bit_len - index, + "bit string removal range out of bounds" + ); + + let first_word = index / WORD_BITS; + let prefix_mask = crate::low_mask(index % WORD_BITS); + let prefix = self.words[first_word] & prefix_mask; + // The word-level shift runs low to high, preserving overlapping input. + // At every bit position >= index, it reads the old position + width. + self.words[first_word..].shr_assign(self.bit_len - first_word * WORD_BITS, width); + // Removed bits may have shifted into the prefix: replace, rather than OR. + self.words[first_word] = (self.words[first_word] & !prefix_mask) | prefix; + + let new_len = self.bit_len - width; + let new_words = word_len(new_len); + if new_words < self.words.len() { + self.truncate_words(new_words); + } + self.words.mask_unused_bits(new_len); + self.bit_len = new_len; + } + /// Inserts a single bit at `index`, shifting the tail right. /// /// `index` is clamped to `[0, self.bit_len()]` — an out-of-bounds index @@ -86,3 +159,6 @@ mod tests_for_insert; #[cfg(test)] mod tests_for_remove; + +#[cfg(test)] +mod tests_for_edit_bits; diff --git a/src/bit_string/impls_for_editing/impls_for_insert_remove/tests_for_edit_bits.rs b/src/bit_string/impls_for_editing/impls_for_insert_remove/tests_for_edit_bits.rs new file mode 100644 index 0000000..4e4d7bd --- /dev/null +++ b/src/bit_string/impls_for_editing/impls_for_insert_remove/tests_for_edit_bits.rs @@ -0,0 +1,95 @@ +use crate::{BitString, low_mask, word_len}; +use alloc::vec::Vec; + +const LENGTHS: &[usize] = &[0, 1, 7, 8, 15, 31, 32, 63, 64, 65, 127, 128, 129, 193]; +const WIDTHS: &[usize] = &[0, 1, 2, 3, 5, 6, 7, 8, 31, 32, 63, 64]; + +fn input(len: usize) -> Vec { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state & 1 != 0 + }) + .collect() +} + +fn assert_bits(actual: &BitString, expected: &[bool]) { + assert_eq!(actual.bit_len(), expected.len()); + assert_eq!(actual.words().len(), word_len(expected.len())); + assert_eq!(actual.iter().collect::>(), expected); + if let Some(&last) = actual.words().last() { + let used = (expected.len() - 1) % 64 + 1; + assert_eq!(last & !low_mask(used), 0, "unused bits must remain zero"); + } +} + +#[test] +fn insert_bits_matches_boolean_model_at_every_position() { + for &len in LENGTHS { + let original = input(len); + let source = BitString::from_bool_iter(original.iter().copied()); + for &width in WIDTHS { + for index in 0..=len + 1 { + for code in [0, u64::MAX, 0xa53c_79e1_d24b_860fu64] { + let mut expected = original.clone(); + let bits = (0..width).map(|shift| (code >> shift) & 1 != 0); + expected.splice(index.min(len)..index.min(len), bits); + let mut actual = source.clone(); + actual.insert_bits(index, code, width); + assert_bits(&actual, &expected); + } + } + } + } +} + +#[test] +fn remove_bits_matches_boolean_model_at_every_position() { + for &len in LENGTHS { + let original = input(len); + let source = BitString::from_bool_iter(original.iter().copied()); + for &width in WIDTHS { + if width > len { + continue; + } + for index in 0..=len - width { + let mut expected = original.clone(); + expected.drain(index..index + width); + let mut actual = source.clone(); + actual.remove_bits(index, width); + assert_bits(&actual, &expected); + } + } + } +} + +#[test] +fn repeated_edits_preserve_zero_padding() { + let mut actual = BitString::from_bool_iter(input(193)); + let original = actual.clone(); + for &width in WIDTHS { + actual.insert_bits(63, u64::MAX, width); + actual.remove_bits(63, width); + assert_eq!(actual, original); + } + actual.insert_bits(usize::MAX, 0, 0); + actual.remove_bits(usize::MAX, 0); + assert_eq!(actual, original); +} + +#[test] +fn invalid_arguments_leave_payload_unchanged() { + extern crate std; + use std::panic::{AssertUnwindSafe, catch_unwind}; + let original = BitString::from_bool_iter(input(129)); + let mut actual = original.clone(); + assert!(catch_unwind(AssertUnwindSafe(|| actual.insert_bits(0, 1, 65))).is_err()); + assert_eq!(actual, original); + for (index, width) in [(0, 65), (129, 1), (128, 2), (usize::MAX, 1)] { + assert!(catch_unwind(AssertUnwindSafe(|| actual.remove_bits(index, width))).is_err()); + assert_eq!(actual, original); + } +} diff --git a/src/bit_string/impls_for_editing/impls_for_push_pop.rs b/src/bit_string/impls_for_editing/impls_for_push_pop.rs index 7745e27..522b17b 100644 --- a/src/bit_string/impls_for_editing/impls_for_push_pop.rs +++ b/src/bit_string/impls_for_editing/impls_for_push_pop.rs @@ -1,9 +1,75 @@ use crate::funcs_for_bits::*; use crate::traits::*; +use crate::{WORD_BITS, low_mask}; use super::*; impl BitString { + /// Appends one byte to a byte-aligned bit string. + #[inline] + pub(crate) fn push_byte(&mut self, value: u8) { + debug_assert_eq!(self.bit_len % 8, 0); + + let new_bit_len = self + .bit_len + .checked_add(8) + .expect("bit string length overflow"); + let word_index = self.bit_len / WORD_BITS; + let bit_offset = self.bit_len % WORD_BITS; + + if word_index == self.words.len() { + self.words.push(0); + } + + self.words[word_index] |= u64::from(value) << bit_offset; + self.bit_len = new_bit_len; + } + + /// Reserves backing storage for an additional number of bits. + #[inline] + pub(crate) fn reserve_bits(&mut self, additional_bits: usize) { + if let Some(total_bits) = self.bit_len.checked_add(additional_bits) { + let additional_words = word_len(total_bits).saturating_sub(self.words.len()); + self.words.reserve(additional_words); + } + } + + /// Appends the low `bits` bits of `value` in one or two word writes. + #[inline] + pub(crate) fn push_bits(&mut self, value: u64, bits: usize) { + assert!( + bits <= WORD_BITS, + "cannot append more than one word of bits" + ); + if bits == 0 { + return; + } + + let new_bit_len = self + .bit_len + .checked_add(bits) + .expect("bit string length overflow"); + let word_index = self.bit_len / WORD_BITS; + let bit_offset = self.bit_len % WORD_BITS; + + if word_index == self.words.len() { + self.words.push(0); + } + + let value = value & low_mask(bits); + self.words[word_index] |= value << bit_offset; + + let next_offset = bit_offset + bits; + if next_offset > WORD_BITS { + if word_index + 1 == self.words.len() { + self.words.push(0); + } + self.words[word_index + 1] |= value >> (WORD_BITS - bit_offset); + } + + self.bit_len = new_bit_len; + } + /// Appends a single bit to the end. /// /// Panics if the bit string length would overflow `usize`. diff --git a/src/bit_string/impls_for_editing/impls_for_push_pop/tests_for_push.rs b/src/bit_string/impls_for_editing/impls_for_push_pop/tests_for_push.rs index 682f8b7..2200a53 100644 --- a/src/bit_string/impls_for_editing/impls_for_push_pop/tests_for_push.rs +++ b/src/bit_string/impls_for_editing/impls_for_push_pop/tests_for_push.rs @@ -74,3 +74,25 @@ fn pushing_true_increments_count_ones() { assert_eq!(bits.count_zeros(), 1); assert_eq!(bits.to_string(), "10111"); } + +#[test] +fn push_byte_appends_at_byte_aligned_offsets() { + for prefix_len in [0, 8, 56, 64] { + for value in [0, 0xa5, u8::MAX] { + let mut bits = BitString::ones(prefix_len); + + bits.push_byte(value); + + assert_eq!(bits.bit_len(), prefix_len + 8); + for index in 0..prefix_len { + assert_eq!(bits.get(index), Some(true)); + } + for offset in 0..8 { + assert_eq!( + bits.get(prefix_len + offset), + Some(value & (1 << offset) != 0) + ); + } + } + } +} diff --git a/src/funcs_for_packed.rs b/src/funcs_for_packed.rs index 65464f9..ff6eb22 100644 --- a/src/funcs_for_packed.rs +++ b/src/funcs_for_packed.rs @@ -16,5 +16,39 @@ pub(crate) const fn code_mask() -> u8 { } } +/// Extracts one fixed-width packed code from a word-aligned or offset view. +/// +/// # Safety +/// +/// `BITS` must be in `1..=8`, `bit_start` must be a multiple of `BITS`, and the +/// complete `BITS`-wide code at `bit_start + index * BITS` must lie within the +/// logical bit range represented by `words`. The arithmetic for that range +/// must fit in `usize`. These conditions ensure that the first backing word +/// exists and, when the code crosses a word boundary, that the following +/// backing word exists as well. +#[inline] +pub(crate) unsafe fn extract_code_unchecked( + words: &[u64], + bit_start: usize, + index: usize, +) -> u8 { + let bits = usize::from(BITS); + let bit_index = bit_start + index * bits; + let word_index = bit_index / 64; + let offset = bit_index % 64; + + // SAFETY: The caller guarantees that the complete code lies in the logical + // bit range, so its first backing word is present. + let mut code = unsafe { *words.get_unchecked(word_index) } >> offset; + if 64 % bits != 0 && offset + bits > 64 { + // SAFETY: This branch is taken only when the code crosses the current + // word boundary. Since the complete code lies in the logical bit + // range, the following backing word is present. + code |= unsafe { *words.get_unchecked(word_index + 1) } << (64 - offset); + } + + (code & u64::from(code_mask::())) as u8 +} + #[cfg(test)] mod tests_for_packed; diff --git a/src/packed_str.rs b/src/packed_str.rs index 507a3af..aee5b5b 100644 --- a/src/packed_str.rs +++ b/src/packed_str.rs @@ -5,6 +5,13 @@ use core::marker::PhantomData; use crate::{BitStr, PackedString, traits::PackedChar}; /// A character-aligned borrowed view into a [`PackedString`]. +/// +/// The view may be unaligned to a `u64` word boundary, but never to a packed +/// character boundary. Its representation maintains these invariants: +/// +/// - `bits.start()` is a multiple of `BITS`; +/// - `bits.bit_len()` is a multiple of `BITS`; +/// - the bit range lies within its source [`BitString`](crate::BitString). #[derive(Clone, Copy)] pub struct PackedStr<'ps, C, const BITS: u8> where diff --git a/src/packed_str/impls_for_access.rs b/src/packed_str/impls_for_access.rs index 613e80a..e9fdbb9 100644 --- a/src/packed_str/impls_for_access.rs +++ b/src/packed_str/impls_for_access.rs @@ -1,4 +1,5 @@ use super::*; +use crate::extract_code_unchecked; impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> where @@ -16,13 +17,19 @@ where if index >= self.char_len() { return None; } - Some( - C::from_code( - (self.bits.get_chunk(index * usize::from(BITS)) - & u64::from(crate::code_mask::())) as u8, - ) - .expect("PackedChar rejected a code in a PackedStr invariant"), - ) + // SAFETY: `PackedStr` maintains a character-aligned BitStr start, so + // `self.bits.start()` is a multiple of BITS as required by the helper. + // `index < char_len()` implies + // `(index + 1) * BITS <= self.bits.bit_len()`, so the complete code lies + // inside this view. The BitStr invariant places + // `[self.bits.start(), self.bits.start() + self.bits.bit_len())` inside + // the source bit range. Therefore the code's first word and, when it + // crosses a word boundary, its successor are present in the source + // words, including when the view starts at a non-word offset. + let code = unsafe { + extract_code_unchecked::(self.bits.source().words(), self.bits.start(), index) + }; + Some(C::from_code(code).expect("PackedChar rejected a code in a PackedStr invariant")) } pub fn first(&self) -> Option { @@ -35,3 +42,6 @@ where .and_then(|index| self.get(index)) } } + +#[cfg(test)] +mod tests_for_access; diff --git a/src/packed_str/impls_for_access/tests_for_access.rs b/src/packed_str/impls_for_access/tests_for_access.rs new file mode 100644 index 0000000..54bb21c --- /dev/null +++ b/src/packed_str/impls_for_access/tests_for_access.rs @@ -0,0 +1,45 @@ +use crate::{PackedStr, PackedString, traits::PackedChar}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct TestChar(u8); + +impl PackedChar for TestChar { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} + +fn check_access() { + let mask = (1u16 << BITS) - 1; + let values: alloc::vec::Vec<_> = (0..96) + .map(|index| TestChar(((index * 29 + 7) as u16 & mask) as u8)) + .collect(); + let packed = PackedString::::from_chars(values.iter().copied()); + let view: PackedStr<'_, TestChar, BITS> = packed.as_packed_str().slice_from(1); + assert_eq!(view.bits.start() % usize::from(BITS), 0); + assert_ne!(view.bits.start() % 64, 0); + + for (index, expected) in values.iter().copied().skip(1).enumerate() { + assert_eq!(packed.get(index + 1), Some(expected)); + assert_eq!(view.get(index), Some(expected)); + } + + assert_eq!(view.get(view.char_len()), None); + assert_eq!(packed.get(packed.char_len()), None); +} + +#[test] +fn checked_get_supports_all_widths_and_offset_views() { + check_access::<1>(); + check_access::<2>(); + check_access::<3>(); + check_access::<4>(); + check_access::<5>(); + check_access::<6>(); + check_access::<7>(); + check_access::<8>(); +} diff --git a/src/packed_str/impls_for_iter.rs b/src/packed_str/impls_for_iter.rs index 8eb20d6..b928990 100644 --- a/src/packed_str/impls_for_iter.rs +++ b/src/packed_str/impls_for_iter.rs @@ -1,5 +1,7 @@ use core::iter::FusedIterator; +use crate::{WORD_BITS, code_mask}; + use super::*; impl<'ps, C, const BITS: u8> PackedStr<'ps, C, BITS> @@ -7,15 +9,46 @@ where C: PackedChar, { pub fn iter(&self) -> Iter<'ps, C, BITS> { + let char_len = self.char_len(); + let bit_start = self.bits.start(); + let front_cursor = if char_len == 0 { + FrontCursor { + pending_bits: 0, + pending_bit_count: 0, + next_word_index: 0, + } + } else { + let word_index = bit_start / WORD_BITS; + let bit_offset = bit_start % WORD_BITS; + let first_word = self.bits.source().words()[word_index]; + FrontCursor { + pending_bits: first_word >> bit_offset, + pending_bit_count: WORD_BITS - bit_offset, + next_word_index: word_index + 1, + } + }; + Iter { view: *self, front: 0, - back: self.char_len(), + back: char_len, + front_cursor, } } } -#[allow(dead_code)] +struct FrontCursor { + // Loaded but unconsumed bits; the next code starts at bit 0. + pending_bits: u64, + // Includes source bits past this view's end; front/back prevents yielding them. + pending_bit_count: usize, + // Index of the next backing word not yet loaded into `pending_bits`. + next_word_index: usize, + // For a non-empty iterator, with `Iter::front` and the view start: + // `view.bits.start() + front * BITS + pending_bit_count` + // equals `next_word_index * WORD_BITS`. +} + pub struct Iter<'ps, C, const BITS: u8> where C: PackedChar, @@ -23,6 +56,7 @@ where view: PackedStr<'ps, C, BITS>, front: usize, back: usize, + front_cursor: FrontCursor, } impl Iterator for Iter<'_, C, BITS> @@ -33,12 +67,41 @@ where fn next(&mut self) -> Option { if self.front == self.back { - None - } else { - let c = self.view.get(self.front); - self.front += 1; - c + return None; } + + let width = usize::from(BITS); + let words = self.view.bits.source().words(); + let code = if self.front_cursor.pending_bit_count >= width { + let code = self.front_cursor.pending_bits; + self.front_cursor.pending_bits >>= width; + self.front_cursor.pending_bit_count -= width; + code + } else { + let old_fill = self.front_cursor.pending_bit_count; + // SAFETY: This refill branch requires `old_fill < width`, and + // `front < back`. The cursor invariant gives + // `front_cursor.next_word_index * WORD_BITS = view.bits.start() + // + front * BITS + old_fill`, which is less than + // `view.bits.start() + (front + 1) * BITS + // <= view.bits.start() + back * BITS + // <= view.bits.start() + view.bits.bit_len() + // <= words.len() * WORD_BITS`. Thus this word index is in bounds. + let next_word = unsafe { *words.get_unchecked(self.front_cursor.next_word_index) }; + self.front_cursor.next_word_index += 1; + + let code = self.front_cursor.pending_bits | (next_word << old_fill); + let used_from_next = width - old_fill; + self.front_cursor.pending_bits = next_word >> used_from_next; + self.front_cursor.pending_bit_count = WORD_BITS - used_from_next; + code + }; + + let character = C::from_code((code & u64::from(code_mask::())) as u8) + .expect("PackedChar rejected a code in a PackedStr invariant"); + self.front += 1; + + Some(character) } fn size_hint(&self) -> (usize, Option) { @@ -75,3 +138,6 @@ where self.iter() } } + +#[cfg(test)] +mod tests_for_iter; diff --git a/src/packed_str/impls_for_iter/tests_for_iter.rs b/src/packed_str/impls_for_iter/tests_for_iter.rs new file mode 100644 index 0000000..9b5b0a5 --- /dev/null +++ b/src/packed_str/impls_for_iter/tests_for_iter.rs @@ -0,0 +1,106 @@ +use crate::{PackedStr, PackedString, WORD_BITS, traits::PackedChar}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} + +fn assert_front_window(iter: &super::Iter<'_, Code, BITS>, front: usize) { + assert_eq!( + iter.view.bits.start() + front * usize::from(BITS) + iter.front_cursor.pending_bit_count, + iter.front_cursor.next_word_index * WORD_BITS + ); + assert!(iter.front_cursor.pending_bit_count <= WORD_BITS); +} + +fn check_mixed_iteration(word_unaligned: bool) { + let width = usize::from(BITS); + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let codes_per_word = WORD_BITS / width; + let lengths = [ + 0, + 1, + codes_per_word.saturating_sub(1), + codes_per_word, + codes_per_word + 1, + codes_per_word * 2 - 1, + codes_per_word * 2, + codes_per_word * 2 + 1, + 95, + ]; + + for len in lengths { + let expected = (0..len) + .map(|index| Code(((index * 17 + 3) as u8) & mask)) + .collect::>(); + let mut source = alloc::vec::Vec::with_capacity(len + usize::from(word_unaligned)); + if word_unaligned { + source.push(Code(0)); + } + source.extend_from_slice(&expected); + let owner = PackedString::::from_chars(source); + let packed_str = owner.as_packed_str(); + let view: PackedStr<'_, Code, BITS> = if word_unaligned { + packed_str.slice_from(1) + } else { + packed_str + }; + let mut iter = view.iter(); + let mut front = 0; + let mut back = len; + + if !word_unaligned || len > 0 { + assert_eq!(view.bits.start() % width, 0); + if word_unaligned { + assert_ne!(view.bits.start() % WORD_BITS, 0); + } else { + assert_eq!(view.bits.start(), 0); + } + assert_front_window(&iter, front); + } + + while front < back { + assert_eq!(iter.next(), Some(expected[front])); + front += 1; + assert_front_window(&iter, front); + if front < back { + back -= 1; + assert_eq!(iter.next_back(), Some(expected[back])); + assert_front_window(&iter, front); + } + } + + assert_eq!(iter.next(), None); + assert_eq!(iter.next_back(), None); + assert_eq!(iter.len(), 0); + } +} + +#[test] +fn rolling_window_handles_aligned_and_offset_views_with_mixed_iteration() { + fn check_width() { + check_mixed_iteration::(false); + check_mixed_iteration::(true); + } + + check_width::<1>(); + check_width::<2>(); + check_width::<3>(); + check_width::<4>(); + check_width::<5>(); + check_width::<6>(); + check_width::<7>(); + check_width::<8>(); +} diff --git a/src/packed_string.rs b/src/packed_string.rs index 9e4b78e..8b2e645 100644 --- a/src/packed_string.rs +++ b/src/packed_string.rs @@ -21,6 +21,7 @@ where mod funcs_for_code; mod impls_for_access; mod impls_for_construction; +mod impls_for_conversion; mod impls_for_editing; mod impls_for_eq; mod impls_for_fmt; diff --git a/src/packed_string/funcs_for_code.rs b/src/packed_string/funcs_for_code.rs index 04d47bb..5bab330 100644 --- a/src/packed_string/funcs_for_code.rs +++ b/src/packed_string/funcs_for_code.rs @@ -1,5 +1,104 @@ +use alloc::vec::Vec; + +use crate::traits::{WordsPack, layout_block_len}; +use crate::{BitString, WORD_BITS, word_len}; + use super::*; +// Encoding batch size shared by construction and extension. A batch contains +// complete layout blocks for BITS=1..=8; tune independently of decoding batches. +pub(super) const PACK_CODE_BATCH_LEN: usize = 64; + +pub(super) fn pack_codes(chars: I) -> BitString +where + C: PackedChar, + I: IntoIterator, +{ + let width = usize::from(BITS); + let chars = chars.into_iter(); + let (lower_bound, _) = chars.size_hint(); + let mut words = Vec::new(); + if let Some(bit_len) = lower_bound.checked_mul(width) { + words.reserve(word_len(bit_len)); + } + + let (tail_codes, tail_len) = for_each_aligned_code_batch::(chars, |codes| { + let start = words.len(); + words.resize(start + codes.len() * width / WORD_BITS, 0); + words[start..].pack_codes::(codes); + }); + let mut bit_len = words.len() * WORD_BITS; + for &code in &tail_codes[..tail_len] { + append_code::(&mut words, &mut bit_len, u64::from(code)); + } + + BitString::from_owned_words(words, bit_len).expect("packed words have an invalid length") +} + +/// Passes full code batches and final complete layout blocks to `consume`. +/// Returns any remaining codes that cannot form a word-aligned layout block. +/// +/// Construction and extension share this batching so extension can pack a +/// bounded batch at a time instead of materializing the entire input first. +pub(super) fn for_each_aligned_code_batch( + chars: I, + mut consume: F, +) -> ([u8; PACK_CODE_BATCH_LEN], usize) +where + C: PackedChar, + I: IntoIterator, + F: FnMut(&[u8]), +{ + let layout_len = layout_block_len::(); + let mut batch = [0u8; PACK_CODE_BATCH_LEN]; + let mut batch_len = 0; + + for character in chars { + batch[batch_len] = character.code(); + batch_len += 1; + + if batch_len == PACK_CODE_BATCH_LEN { + consume(&batch); + batch_len = 0; + } + } + + let aligned_count = batch_len / layout_len * layout_len; + if aligned_count != 0 { + consume(&batch[..aligned_count]); + } + + let tail_len = batch_len - aligned_count; + batch.copy_within(aligned_count..batch_len, 0); + (batch, tail_len) +} + +#[inline] +fn append_code(words: &mut Vec, bit_len: &mut usize, code: u64) { + let width = usize::from(BITS); + let new_bit_len = bit_len + .checked_add(width) + .expect("packed string length overflow"); + let word_index = *bit_len / WORD_BITS; + let bit_offset = *bit_len % WORD_BITS; + + if word_index == words.len() { + words.push(0); + } + + words[word_index] |= code << bit_offset; + + let next_offset = bit_offset + width; + if next_offset > WORD_BITS { + if word_index + 1 == words.len() { + words.push(0); + } + words[word_index + 1] |= code >> (WORD_BITS - bit_offset); + } + + *bit_len = new_bit_len; +} + #[inline] pub(super) fn write_code(bits: &mut BitString, position: usize, code: u8) { let start = position * usize::from(BITS); diff --git a/src/packed_string/impls_for_access.rs b/src/packed_string/impls_for_access.rs index 3ee54d3..64b7b93 100644 --- a/src/packed_string/impls_for_access.rs +++ b/src/packed_string/impls_for_access.rs @@ -1,4 +1,5 @@ use super::*; +use crate::extract_code_unchecked; impl PackedString where @@ -30,10 +31,12 @@ where if index >= self.char_len() { return None; } - Some( - C::from_code(self.code_at(index)) - .expect("PackedChar rejected a code it previously produced"), - ) + // SAFETY: `index < char_len()` implies + // `(index + 1) * BITS <= self.bits.bit_len()`, so this character's + // complete range lies in the owned packed bit range. Its first word, + // and the next word if it crosses a boundary, are therefore present. + let code = unsafe { extract_code_unchecked::(self.bits.words(), 0, index) }; + Some(C::from_code(code).expect("PackedChar rejected a code it previously produced")) } #[inline] @@ -45,12 +48,6 @@ where pub fn last(&self) -> Option { self.char_len().checked_sub(1).and_then(|i| self.get(i)) } - - #[inline] - fn code_at(&self, index: usize) -> u8 { - let start = index * usize::from(BITS); - (self.bits.get_chunk(start) & u64::from(code_mask::())) as u8 - } } #[cfg(test)] diff --git a/src/packed_string/impls_for_construction.rs b/src/packed_string/impls_for_construction.rs index df209f8..9371a58 100644 --- a/src/packed_string/impls_for_construction.rs +++ b/src/packed_string/impls_for_construction.rs @@ -1,4 +1,5 @@ use super::*; +use crate::packed_string::funcs_for_code::pack_codes; impl PackedString where @@ -18,9 +19,7 @@ where where I: IntoIterator, { - let mut result = Self::new(); - result.extend(chars); - result + Self::from_valid_bits(pack_codes::(chars)) } } diff --git a/src/packed_string/impls_for_construction/tests_for_construction.rs b/src/packed_string/impls_for_construction/tests_for_construction.rs index d39fedb..6702868 100644 --- a/src/packed_string/impls_for_construction/tests_for_construction.rs +++ b/src/packed_string/impls_for_construction/tests_for_construction.rs @@ -1,7 +1,21 @@ #[cfg(debug_assertions)] use crate::BitString; -use crate::PackedString; use crate::packed_string::tests_for_support::{Letter, LetterString}; +use crate::traits::layout_block_len; +use crate::{PackedString, traits::PackedChar}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} #[test] fn enum_discriminants_are_stored_directly() { @@ -18,6 +32,83 @@ fn collect_constructs_a_packed_string() { assert_eq!(value.get(0), Some(Letter::A)); } +#[test] +fn construction_and_extend_preserve_values_across_word_boundaries() { + fn check() { + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let initial = (0..7) + .map(|index| Code(((index * 3 + 1) as u8) & mask)) + .collect::>(); + let suffix = (7..30) + .map(|index| Code(((index * 5 + 2) as u8) & mask)) + .collect::>(); + let expected = initial + .iter() + .chain(&suffix) + .map(|code| code.0) + .collect::>(); + + let mut value = PackedString::::from_chars(initial.iter().copied()); + value.extend(suffix.iter().copied()); + + assert_eq!(value.bits().bit_len(), expected.len() * usize::from(BITS)); + assert_eq!( + value + .iter() + .map(|code| code.0) + .collect::>(), + expected + ); + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} + +#[test] +fn construction_preserves_values_across_complete_pack_blocks() { + fn check() { + let block_len = layout_block_len::(); + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let expected = (0..(block_len * 2 + 3)) + .map(|index| ((index * 11 + 5) as u8) & mask) + .collect::>(); + let input = expected.iter().copied().map(Code); + let value = PackedString::::from_chars(input); + + assert_eq!( + value + .iter() + .map(|code| code.0) + .collect::>(), + expected + ); + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} + #[test] fn trusted_constructor_adopts_aligned_payload_unchanged() { let source = LetterString::from_chars([Letter::A, Letter::D, Letter::B, Letter::C]); diff --git a/src/packed_string/impls_for_conversion.rs b/src/packed_string/impls_for_conversion.rs new file mode 100644 index 0000000..1ce4355 --- /dev/null +++ b/src/packed_string/impls_for_conversion.rs @@ -0,0 +1,70 @@ +use crate::traits::{WordsUnpack, has_accelerated_bulk_unpack, layout_block_len}; + +use super::*; + +impl PackedString +where + C: PackedChar, +{ + /// Collects the decoded characters into a vector. + pub fn to_vec(&self) -> alloc::vec::Vec { + // Decoding batch size, tuned independently of construction and extension. + // 64 codes is a multiple of every layout block for BITS=1..=8. + const UNPACK_CODE_BATCH_LEN: usize = 64; + + if !has_accelerated_bulk_unpack::() { + return self.iter().collect(); + } + + debug_assert!(UNPACK_CODE_BATCH_LEN.is_multiple_of(layout_block_len::())); + + let char_len = self.char_len(); + if char_len < UNPACK_CODE_BATCH_LEN { + return self.iter().collect(); + } + + let bulk_code_len = char_len / UNPACK_CODE_BATCH_LEN * UNPACK_CODE_BATCH_LEN; + let words = self.bits.words(); + let words_per_batch = UNPACK_CODE_BATCH_LEN * usize::from(BITS) / crate::WORD_BITS; + let mut decoded = [0u8; UNPACK_CODE_BATCH_LEN]; + let mut result = alloc::vec::Vec::with_capacity(char_len); + + for batch_index in 0..bulk_code_len / UNPACK_CODE_BATCH_LEN { + let word_start = batch_index * words_per_batch; + let word_end = word_start + words_per_batch; + words[word_start..word_end].unpack_codes::(&mut decoded); + + result.extend(decoded.iter().map(|&code| { + C::from_code(code).expect("PackedChar rejected a code it previously produced") + })); + } + + let layout_len = layout_block_len::(); + let remainder_code_len = char_len - bulk_code_len; + let unpacked_remainder_code_len = remainder_code_len / layout_len * layout_len; + + if unpacked_remainder_code_len > 0 { + let remainder_word_start = bulk_code_len * usize::from(BITS) / crate::WORD_BITS; + let remainder_word_len = + unpacked_remainder_code_len * usize::from(BITS) / crate::WORD_BITS; + let remainder_word_end = remainder_word_start + remainder_word_len; + words[remainder_word_start..remainder_word_end] + .unpack_codes::(&mut decoded[..unpacked_remainder_code_len]); + + result.extend(decoded[..unpacked_remainder_code_len].iter().map(|&code| { + C::from_code(code).expect("PackedChar rejected a code it previously produced") + })); + } + + let tail_start = bulk_code_len + unpacked_remainder_code_len; + result.extend((tail_start..char_len).map(|index| { + self.get(index) + .expect("packed character index is within the string") + })); + + result + } +} + +#[cfg(test)] +mod tests_for_conversion; diff --git a/src/packed_string/impls_for_conversion/tests_for_conversion.rs b/src/packed_string/impls_for_conversion/tests_for_conversion.rs new file mode 100644 index 0000000..235dec3 --- /dev/null +++ b/src/packed_string/impls_for_conversion/tests_for_conversion.rs @@ -0,0 +1,38 @@ +use crate::PackedString; +use crate::traits::PackedChar; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} + +#[test] +fn to_vec_matches_codes_across_batches_and_tails() { + fn check() { + let mask = crate::code_mask::(); + for len in [0, 1, 15, 16, 31, 32, 63, 64, 65, 127, 128, 129, 257] { + let expected = (0..len) + .map(|index| Code(((index * 29 + 5) as u8) & mask)) + .collect::>(); + let value = PackedString::::from_chars(expected.iter().copied()); + assert_eq!(value.to_vec(), expected, "BITS={BITS}, len={len}"); + } + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} diff --git a/src/packed_string/impls_for_editing.rs b/src/packed_string/impls_for_editing.rs index 000f4f1..fe9c4a5 100644 --- a/src/packed_string/impls_for_editing.rs +++ b/src/packed_string/impls_for_editing.rs @@ -1,4 +1,6 @@ +use super::funcs_for_code::{PACK_CODE_BATCH_LEN, for_each_aligned_code_batch}; use super::*; +use crate::traits::WordsPack; impl PackedString where @@ -7,12 +9,10 @@ where /// Appends the character's code directly to the payload. pub fn push(&mut self, character: C) { let code = character.code(); - self.bits - .bit_len() - .checked_add(usize::from(BITS)) - .expect("packed string length overflow"); - for shift in 0..BITS { - self.bits.push((code >> shift) & 1 != 0); + if BITS == 8 { + self.bits.push_byte(code); + } else { + self.bits.push_bits(u64::from(code), usize::from(BITS)); } } @@ -46,8 +46,41 @@ where C: PackedChar, { fn extend>(&mut self, iter: I) { - for character in iter { - self.push(character); + let iter = iter.into_iter(); + let width = usize::from(BITS); + let (lower_bound, upper_bound) = iter.size_hint(); + + if let Some(additional_bits) = lower_bound.checked_mul(width) { + self.bits.reserve_bits(additional_bits); + } + + // Avoid batch setup when the iterator can yield fewer than one batch. + if matches!(upper_bound, Some(upper_bound) if upper_bound < PACK_CODE_BATCH_LEN) { + for character in iter { + self.push(character); + } + return; + } + + let mut packed_words = [0u64; PACK_CODE_BATCH_LEN / 8]; + let bits = &mut self.bits; + // Keep scratch space bounded; collecting the whole iterator first would + // require a temporary PackedString for the complete suffix. + // If iteration panics, completed batches remain appended and the partial + // batch is discarded during unwinding. + let (tail_codes, tail_len) = for_each_aligned_code_batch::(iter, |codes| { + let packed_bit_len = codes.len() * width; + let packed_word_len = packed_bit_len / crate::WORD_BITS; + packed_words[..packed_word_len].pack_codes::(codes); + bits.push_packed_words(&packed_words[..packed_word_len], packed_bit_len); + }); + + for &code in &tail_codes[..tail_len] { + if BITS == 8 { + self.bits.push_byte(code); + } else { + self.bits.push_bits(u64::from(code), width); + } } } } diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs index 03f4f02..8f80508 100644 --- a/src/packed_string/impls_for_editing/impls_for_insert_remove.rs +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove.rs @@ -6,27 +6,18 @@ where { pub fn insert(&mut self, index: usize, character: C) { let index = index.min(self.char_len()); - self.bits - .bit_len() - .checked_add(usize::from(BITS)) - .expect("packed string length overflow"); let start = index * usize::from(BITS); - for shift in (0..BITS).rev() { - self.bits - .insert(start, (character.code() >> shift) & 1 != 0); - } + self.bits + .insert_bits(start, u64::from(character.code()), usize::from(BITS)); } pub fn remove(&mut self, index: usize) -> C { let character = self.get(index).expect("packed string index out of bounds"); let start = index * usize::from(BITS); - let mut code = 0; - for shift in 0..BITS { - if self.bits.remove(start) { - code |= 1 << shift; - } - } - debug_assert_eq!(code, character.code()); + self.bits.remove_bits(start, usize::from(BITS)); character } } + +#[cfg(test)] +mod tests_for_insert_remove; diff --git a/src/packed_string/impls_for_editing/impls_for_insert_remove/tests_for_insert_remove.rs b/src/packed_string/impls_for_editing/impls_for_insert_remove/tests_for_insert_remove.rs new file mode 100644 index 0000000..e1a571c --- /dev/null +++ b/src/packed_string/impls_for_editing/impls_for_insert_remove/tests_for_insert_remove.rs @@ -0,0 +1,78 @@ +use crate::{PackedString, code_mask, traits::PackedChar}; +use alloc::vec::Vec; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + fn from_code(code: u8) -> Option { + (code <= code_mask::()).then_some(Self(code)) + } +} + +fn check() { + for len in [0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 65, 127, 128, 129] { + let mut state = 0x1234_5678_9abc_def0u64; + let expected = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + Code(state as u8 & code_mask::()) + }) + .collect::>(); + let source = PackedString::::from_chars(expected.iter().copied()); + for index in 0..=len + 1 { + for code in [Code(0), Code(code_mask::())] { + let mut actual = source.clone(); + let mut model = expected.clone(); + let at = index.min(len); + model.insert(at, code); + actual.insert(index, code); + assert_eq!(actual.iter().collect::>(), model); + assert_eq!( + actual.bits(), + PackedString::::from_chars(model).bits() + ); + assert_eq!(actual.remove(at), code); + assert_eq!(actual.bits(), source.bits()); + } + if index < len { + let mut actual = source.clone(); + let mut model = expected.clone(); + assert_eq!(actual.remove(index), model.remove(index)); + assert_eq!( + actual.bits(), + PackedString::::from_chars(model).bits() + ); + } + } + } +} + +#[test] +fn fixed_width_edits_match_character_model() { + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} + +#[test] +fn remove_out_of_bounds_preserves_payload() { + extern crate std; + use std::panic::{AssertUnwindSafe, catch_unwind}; + let mut actual = PackedString::::from_chars([Code(7), Code(31)]); + let original = actual.bits().clone(); + for index in [2, usize::MAX] { + assert!(catch_unwind(AssertUnwindSafe(|| actual.remove(index))).is_err()); + assert_eq!(actual.bits(), &original); + } +} diff --git a/src/packed_string/impls_for_editing/tests_for_editing.rs b/src/packed_string/impls_for_editing/tests_for_editing.rs index 06fad30..4189d91 100644 --- a/src/packed_string/impls_for_editing/tests_for_editing.rs +++ b/src/packed_string/impls_for_editing/tests_for_editing.rs @@ -1,4 +1,5 @@ use crate::packed_string::tests_for_support::{Letter, LetterString}; +use crate::traits::PackedChar; #[test] fn push_set_pop_and_truncate_preserve_values() { @@ -14,3 +15,111 @@ fn push_set_pop_and_truncate_preserve_values() { value.clear(); assert!(value.is_empty()); } + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + (code <= crate::code_mask::()).then_some(Self(code)) + } +} + +fn assert_extend_preserves_codes() { + for prefix_len in [0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 65] { + for extension_len in [0, 1, 7, 8, 15, 16, 31, 32, 63, 64, 65, 127, 128, 129, 193] { + let code_at = |index: usize| { + (index.wrapping_mul(37).wrapping_add(13) as u8) & crate::code_mask::() + }; + let prefix = (0..prefix_len).map(|index| Code::(code_at(index))); + let extension = + (prefix_len..prefix_len + extension_len).map(|index| Code::(code_at(index))); + let expected = prefix + .clone() + .chain(extension.clone()) + .collect::>(); + + let mut actual = super::super::PackedString::, BITS>::from_chars(prefix); + actual.extend(extension); + + assert_eq!(actual.iter().collect::>(), expected); + } + } +} + +#[test] +fn extend_batches_preserve_codes_at_unaligned_tails_for_every_width() { + assert_extend_preserves_codes::<1>(); + assert_extend_preserves_codes::<2>(); + assert_extend_preserves_codes::<3>(); + assert_extend_preserves_codes::<4>(); + assert_extend_preserves_codes::<5>(); + assert_extend_preserves_codes::<6>(); + assert_extend_preserves_codes::<7>(); + assert_extend_preserves_codes::<8>(); +} + +fn assert_extend_with_zero_size_hint() { + let prefix_len = 31; + let extension_len = 193; + let code_at = + |index: usize| (index.wrapping_mul(37).wrapping_add(13) as u8) & crate::code_mask::(); + let prefix = (0..prefix_len).map(|index| Code::(code_at(index))); + let extension = (prefix_len..prefix_len + extension_len) + .map(|index| Code::(code_at(index))) + .collect::>(); + let expected = prefix + .clone() + .chain(extension.iter().copied()) + .collect::>(); + let mut actual = super::super::PackedString::, BITS>::from_chars(prefix); + let filtered = extension.iter().copied().filter(|_| true); + assert_eq!(filtered.size_hint().0, 0); + actual.extend(filtered); + + assert_eq!(actual.iter().collect::>(), expected); +} + +#[test] +fn extend_batches_do_not_depend_on_size_hint_lower_bound() { + assert_extend_with_zero_size_hint::<1>(); + assert_extend_with_zero_size_hint::<2>(); + assert_extend_with_zero_size_hint::<3>(); + assert_extend_with_zero_size_hint::<4>(); + assert_extend_with_zero_size_hint::<5>(); + assert_extend_with_zero_size_hint::<6>(); + assert_extend_with_zero_size_hint::<7>(); + assert_extend_with_zero_size_hint::<8>(); +} + +#[test] +fn extend_panic_keeps_only_completed_batches() { + extern crate std; + + let mut value = super::super::PackedString::, 5>::new(); + let mut index = 0; + let iter = core::iter::from_fn(|| { + if index == 70 { + panic!("test iterator panic"); + } + + let code = Code::<5>((index as u8) & crate::code_mask::<5>()); + index += 1; + Some(code) + }); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| value.extend(iter))); + assert!(result.is_err()); + assert_eq!(index, 70); + assert_eq!(value.char_len(), 64); + assert_eq!( + value.iter().collect::>(), + (0..64) + .map(|index| Code::<5>((index as u8) & crate::code_mask::<5>())) + .collect::>() + ); +} diff --git a/src/packed_string/impls_for_iter.rs b/src/packed_string/impls_for_iter.rs index 48273e3..b2d890a 100644 --- a/src/packed_string/impls_for_iter.rs +++ b/src/packed_string/impls_for_iter.rs @@ -1,5 +1,7 @@ use core::iter::FusedIterator; +use crate::{WORD_BITS, code_mask}; + use super::*; impl PackedString @@ -12,13 +14,11 @@ where string: self, front: 0, back: self.char_len(), + front_pending_bits: 0, + front_pending_bit_count: 0, + front_next_word_index: 0, } } - - /// Collects the decoded characters into a vector. - pub fn to_vec(&self) -> alloc::vec::Vec { - self.iter().collect() - } } #[derive(Clone)] @@ -29,6 +29,14 @@ where string: &'a PackedString, front: usize, back: usize, + // Inspired by sux's bit-field iterator: cache unread bits and refill on demand. + // Loaded but unconsumed bits; the next code starts at bit 0. + front_pending_bits: u64, + // Number of buffered, unconsumed low bits; the final word may include tail padding. + front_pending_bit_count: usize, + // Index of the next backing word not yet loaded into `front_pending_bits`. + front_next_word_index: usize, + // Invariant: `front * BITS + front_pending_bit_count == front_next_word_index * WORD_BITS`. } impl Iterator for Iter<'_, C, BITS> @@ -41,9 +49,36 @@ where if self.front == self.back { return None; } - let character = self.string.get(self.front); + + let width = usize::from(BITS); + let words = self.string.bits.words(); + let code = if self.front_pending_bit_count >= width { + let code = self.front_pending_bits; + self.front_pending_bits >>= width; + self.front_pending_bit_count -= width; + code + } else { + let old_fill = self.front_pending_bit_count; + // SAFETY: This refill branch requires `old_fill < width`, and + // `front < back`. The cursor invariant gives + // `front_next_word_index * WORD_BITS = front * BITS + old_fill`, + // which is less than `(front + 1) * BITS <= back * BITS <= + // words.len() * WORD_BITS`; therefore this word index is in bounds. + let next_word = unsafe { *words.get_unchecked(self.front_next_word_index) }; + self.front_next_word_index += 1; + + let code = self.front_pending_bits | (next_word << old_fill); + let used_from_next = width - old_fill; + self.front_pending_bits = next_word >> used_from_next; + self.front_pending_bit_count = WORD_BITS - used_from_next; + code + }; + + let character = C::from_code((code & u64::from(code_mask::())) as u8) + .expect("PackedChar rejected a code it previously produced"); self.front += 1; - character + + Some(character) } fn size_hint(&self) -> (usize, Option) { diff --git a/src/packed_string/impls_for_iter/tests_for_iter.rs b/src/packed_string/impls_for_iter/tests_for_iter.rs index 22f755b..4a3622a 100644 --- a/src/packed_string/impls_for_iter/tests_for_iter.rs +++ b/src/packed_string/impls_for_iter/tests_for_iter.rs @@ -1,4 +1,6 @@ +use crate::PackedString; use crate::packed_string::tests_for_support::{Letter, LetterString}; +use crate::traits::PackedChar; #[test] fn iter_is_double_ended_and_exact_size() { @@ -10,3 +12,127 @@ fn iter_is_double_ended_and_exact_size() { assert_eq!(iter.next(), Some(Letter::B)); assert_eq!(iter.next(), None); } + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Code(u8); + +impl PackedChar for Code { + fn code(self) -> u8 { + self.0 + } + + fn from_code(code: u8) -> Option { + Some(Self(code)) + } +} + +fn assert_front_window(iter: &super::Iter<'_, Code, BITS>, front: usize) { + assert_eq!( + iter.front_next_word_index * crate::WORD_BITS, + front * usize::from(BITS) + iter.front_pending_bit_count + ); + assert!(iter.front_pending_bit_count <= crate::WORD_BITS); +} + +#[test] +fn mixed_iteration_preserves_order_across_word_boundaries() { + fn check() { + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let expected = (0..100) + .map(|index| Code(((index * 13 + 7) as u8) & mask)) + .collect::>(); + let value = PackedString::::from_chars(expected.iter().copied()); + let mut iter = value.iter(); + assert_front_window(&iter, 0); + + for (front, &code) in expected[..25].iter().enumerate() { + assert_eq!(iter.next(), Some(code)); + assert_front_window(&iter, front + 1); + } + for &code in expected[75..].iter().rev() { + assert_eq!(iter.next_back(), Some(code)); + assert_front_window(&iter, 25); + } + for (front, &code) in expected[25..75].iter().enumerate() { + assert_eq!(iter.next(), Some(code)); + assert_front_window(&iter, front + 26); + } + + assert_front_window(&iter, 75); + assert_eq!(iter.next(), None); + assert_eq!(iter.next_back(), None); + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} + +#[test] +fn rolling_window_handles_word_boundaries_and_mixed_iteration() { + fn check() { + let mask = if BITS == 8 { + u8::MAX + } else { + (1u16 << BITS) as u8 - 1 + }; + let codes_per_word = crate::WORD_BITS / usize::from(BITS); + let codes_per_two_words = (crate::WORD_BITS * 2).div_ceil(usize::from(BITS)); + let lengths = [ + 0, + 1, + codes_per_word.saturating_sub(1), + codes_per_word, + codes_per_word + 1, + codes_per_word * 2 - 1, + codes_per_word * 2, + codes_per_word * 2 + 1, + codes_per_two_words - 1, + codes_per_two_words, + codes_per_two_words + 1, + ]; + + for len in lengths { + let expected = (0..len) + .map(|index| Code(((index * 17 + 3) as u8) & mask)) + .collect::>(); + let value = PackedString::::from_chars(expected.iter().copied()); + let mut iter = value.iter(); + let mut front = 0; + let mut back = len; + assert_front_window(&iter, front); + + while front < back { + assert_eq!(iter.next(), Some(expected[front])); + front += 1; + assert_front_window(&iter, front); + if front < back { + back -= 1; + assert_eq!(iter.next_back(), Some(expected[back])); + assert_front_window(&iter, front); + } + } + + assert_eq!(iter.next(), None); + assert_eq!(iter.next_back(), None); + } + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +} diff --git a/src/traits.rs b/src/traits.rs index c682bf3..741c445 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,17 +1,23 @@ mod packed_char; +mod packed_layout; pub(crate) mod word_ord; pub(crate) mod words_arith; pub(crate) mod words_edit; pub(crate) mod words_eq; pub(crate) mod words_find; pub(crate) mod words_ord; +pub(crate) mod words_pack; pub(crate) mod words_scan; +pub(crate) mod words_unpack; pub use packed_char::PackedChar; +pub(crate) use packed_layout::layout_block_len; pub(crate) use word_ord::*; pub(crate) use words_arith::*; pub(crate) use words_edit::*; pub(crate) use words_eq::*; pub(crate) use words_find::*; pub(crate) use words_ord::*; +pub(crate) use words_pack::*; pub(crate) use words_scan::*; +pub(crate) use words_unpack::*; diff --git a/src/traits/packed_layout.rs b/src/traits/packed_layout.rs new file mode 100644 index 0000000..d083826 --- /dev/null +++ b/src/traits/packed_layout.rs @@ -0,0 +1,18 @@ +use crate::{WORD_BITS, assert_valid_width}; + +/// Returns the smallest number of fixed-width codes that occupies whole words. +#[inline] +pub(crate) const fn layout_block_len() -> usize { + assert_valid_width::(); + WORD_BITS / gcd(WORD_BITS, BITS as usize) +} + +#[inline] +const fn gcd(mut lhs: usize, mut rhs: usize) -> usize { + while rhs != 0 { + let remainder = lhs % rhs; + lhs = rhs; + rhs = remainder; + } + lhs +} diff --git a/src/traits/words_arith/funcs_for_shl_core.rs b/src/traits/words_arith/funcs_for_shl_core.rs index e933b47..8f21971 100644 --- a/src/traits/words_arith/funcs_for_shl_core.rs +++ b/src/traits/words_arith/funcs_for_shl_core.rs @@ -33,8 +33,8 @@ pub(super) fn assign(bits: &mut [u64], bit_len: usize, amount: usize) { // - `ptr` is valid for reads and writes of `word_len` u64 values. // - `dst == src` is explicitly allowed by `dispatch`. // - `dispatch` writes every slot in `0..word_len` exactly once. - // - The implementation writes from high word to low word, so in-place left shift - // does not overwrite source words before they are read. + // - Backends preserve overlapping input by descending writes or by saving + // each word's outgoing carry before overwriting that word. unsafe { dispatch(ptr, ptr.cast_const(), word_len, amount); } diff --git a/src/traits/words_arith/funcs_for_shl_core/avx2.rs b/src/traits/words_arith/funcs_for_shl_core/avx2.rs index b13adff..47380c0 100644 --- a/src/traits/words_arith/funcs_for_shl_core/avx2.rs +++ b/src/traits/words_arith/funcs_for_shl_core/avx2.rs @@ -39,6 +39,32 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + // In-place carry stream, inspired by grit-bitvec's insert. + // A sub-word shift needs only the current old word and its preceding carry. + // Save that carry before overwriting the word, so low-to-high is safe. + if dst.cast_const() == src && word_shift == 0 && bit_shift != 0 { + // SAFETY: dst spans word_len initialized words; end is one-past that + // same allocation. The cursor advances only while it is below end. + // Exact overlap means each old word is read before it is overwritten; + // its outgoing carry is retained before advancing to the next word. + // At word i, carry is old[i - 1] >> (64 - bit_shift), or zero for i=0. + // Thus the write produces exactly the shifted word without rereading + // overwritten input. This branch guarantees 1 <= bit_shift < 64. + unsafe { + let end = dst.add(word_len); + let mut cursor = dst; + let mut carry = 0; + while cursor < end { + let old = cursor.read(); + let next_carry = old >> (WORD_BITS - bit_shift); + cursor.write((old << bit_shift) | carry); + cursor = cursor.add(1); + carry = next_carry; + } + } + return; + } + let bulk_start = word_shift + usize::from(bit_shift != 0); let mut end = word_len; diff --git a/src/traits/words_arith/funcs_for_shl_core/sse2.rs b/src/traits/words_arith/funcs_for_shl_core/sse2.rs index 7d74057..36cc744 100644 --- a/src/traits/words_arith/funcs_for_shl_core/sse2.rs +++ b/src/traits/words_arith/funcs_for_shl_core/sse2.rs @@ -39,6 +39,11 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + // With rustc 1.98.1 and an SSE2-only target, a tested forward carry stream + // saved loads but added register copies and lane shuffles; large-input + // throughput was flat or worse. Saving overlapping loads does not imply + // fewer main-memory reads; load count alone did not predict this result. + // Keep this kernel unless a separately benchmarked SSE2 candidate wins. let bulk_start = word_shift + usize::from(bit_shift != 0); let mut end = word_len; diff --git a/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs index a33a9b6..4d993f6 100644 --- a/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs @@ -88,6 +88,7 @@ fn assert_backend_matches_scalar(backend: Backend) { #[test] fn sse2_matches_scalar() { assert_backend_matches_scalar(sse2::words); + assert_backend_matches_scalar_across_vector_boundaries(sse2::words); } #[cfg(all( @@ -97,10 +98,41 @@ fn sse2_matches_scalar() { #[test] fn avx2_matches_scalar() { assert_backend_matches_scalar(avx2::words); + assert_backend_matches_scalar_across_vector_boundaries(avx2::words); } #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] #[test] fn neon_matches_scalar() { assert_backend_matches_scalar(neon::words); + assert_backend_matches_scalar_across_vector_boundaries(neon::words); +} + +fn assert_backend_matches_scalar_across_vector_boundaries(backend: Backend) { + for len in [0, 1, 3, 4, 5, 15, 16, 17, 18, 31, 32, 33, 63, 64, 65] { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let src: Vec<_> = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }) + .collect(); + for amount in [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 31, 63, 64, 65, 127, 128, 129, 191, 256, 4096, + ] { + let expected = run_backend_owned(scalar::words, &src, amount); + assert_eq!( + run_backend_owned(backend, &src, amount), + expected, + "owned: len={len}, amount={amount}" + ); + assert_eq!( + run_backend_in_place(backend, &src, amount), + expected, + "in-place: len={len}, amount={amount}" + ); + } + } } diff --git a/src/traits/words_arith/funcs_for_shr_core/avx2.rs b/src/traits/words_arith/funcs_for_shr_core/avx2.rs index 07a8364..67d03f9 100644 --- a/src/traits/words_arith/funcs_for_shr_core/avx2.rs +++ b/src/traits/words_arith/funcs_for_shr_core/avx2.rs @@ -39,6 +39,28 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + // In-place forward right shift with one-word lookahead. + if dst.cast_const() == src && word_shift == 0 && bit_shift != 0 { + // SAFETY: word_len > 0 here, and 1 <= bit_shift < 64. At word i, + // current is old[i]. cursor < last guarantees that old[i + 1] exists + // and is unmodified by earlier writes, which only touch lower words. + // Read it before writing old[i] >> bit_shift | old[i + 1] << (64 - bit_shift). + // The last word has no following carry and is shifted on its own. + unsafe { + let last = dst.add(word_len - 1); + let mut cursor = dst; + let mut current = cursor.read(); + while cursor < last { + let next = cursor.add(1).read(); + cursor.write((current >> bit_shift) | (next << (WORD_BITS - bit_shift))); + current = next; + cursor = cursor.add(1); + } + cursor.write(current >> bit_shift); + } + return; + } + let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); let mut start = 0usize; diff --git a/src/traits/words_arith/funcs_for_shr_core/sse2.rs b/src/traits/words_arith/funcs_for_shr_core/sse2.rs index 6c2d8bd..878bcfa 100644 --- a/src/traits/words_arith/funcs_for_shr_core/sse2.rs +++ b/src/traits/words_arith/funcs_for_shr_core/sse2.rs @@ -3,14 +3,14 @@ use crate::WORD_BITS; #[cfg(target_arch = "x86")] use core::arch::x86::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, + __m128i, _mm_castpd_si128, _mm_castsi128_pd, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, + _mm_shuffle_pd, _mm_sll_epi64, _mm_srl_epi64, _mm_storeu_si128, }; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::{ - __m128i, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, _mm_sll_epi64, _mm_srl_epi64, - _mm_storeu_si128, + __m128i, _mm_castpd_si128, _mm_castsi128_pd, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_or_si128, + _mm_shuffle_pd, _mm_sll_epi64, _mm_srl_epi64, _mm_storeu_si128, }; const LANES: usize = 2; @@ -39,6 +39,63 @@ pub(super) unsafe fn words(dst: *mut u64, src: *const u64, word_len: usize, amou return; } + if dst.cast_const() == src && word_shift == 0 && bit_shift != 0 && word_len >= LANES { + // SAFETY: current holds the original words [start, start + 1]. + // Each next vector is loaded before either current word is overwritten; + // retaining next preserves that invariant when start advances by two. + // The bulk bound leaves room for both vectors inside 0..word_len. + // At loop exit, two or three initialized words remain. The optional + // last word is read before the final vector store and is shifted once. + // Exact overlap and 1 <= bit_shift < 64 make every shift valid. + unsafe { + let right_count = _mm_cvtsi64_si128(bit_shift as i64); + let left_count = _mm_cvtsi64_si128((WORD_BITS - bit_shift) as i64); + let mut start = 0; + let mut current = _mm_loadu_si128(src.cast::<__m128i>()); + while word_len - start >= 2 * LANES { + let next = _mm_loadu_si128(src.add(start + LANES).cast::<__m128i>()); + // Bit casts do not convert values. SHUFPD immediate 1 selects + // [current.high, next.low], the neighbors of both current words. + let neighbor = _mm_castpd_si128(_mm_shuffle_pd::<1>( + _mm_castsi128_pd(current), + _mm_castsi128_pd(next), + )); + let out = _mm_or_si128( + _mm_srl_epi64(current, right_count), + _mm_sll_epi64(neighbor, left_count), + ); + _mm_storeu_si128(dst.add(start).cast::<__m128i>(), out); + current = next; + start += LANES; + } + // One cached vector remains, optionally followed by one word. + let has_tail = word_len - start > LANES; + let tail = if has_tail { + src.add(start + LANES).read() + } else { + 0 + }; + let neighbor = _mm_castpd_si128(_mm_shuffle_pd::<1>( + _mm_castsi128_pd(current), + _mm_castsi128_pd(_mm_cvtsi64_si128(tail as i64)), + )); + let out = _mm_or_si128( + _mm_srl_epi64(current, right_count), + _mm_sll_epi64(neighbor, left_count), + ); + _mm_storeu_si128(dst.add(start).cast::<__m128i>(), out); + if has_tail { + dst.add(start + LANES).write(tail >> bit_shift); + } + } + return; + } + + // With rustc 1.98.1 and an SSE2-only target, tested reverse-carry and scalar + // forward-lookahead streams regressed. The latter compiled to one scalar + // SHRD per word, losing this kernel's two-word SIMD parallelism. This is a + // codegen result for those prototypes, not a limitation of SSE2 itself. + // Keep this kernel unless a separately benchmarked vector candidate wins. let bulk_end = word_len - word_shift - usize::from(bit_shift != 0); let mut start = 0usize; diff --git a/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs index a33a9b6..0d8e6ef 100644 --- a/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs @@ -88,6 +88,7 @@ fn assert_backend_matches_scalar(backend: Backend) { #[test] fn sse2_matches_scalar() { assert_backend_matches_scalar(sse2::words); + assert_backend_matches_scalar_across_vector_boundaries(sse2::words); } #[cfg(all( @@ -97,10 +98,41 @@ fn sse2_matches_scalar() { #[test] fn avx2_matches_scalar() { assert_backend_matches_scalar(avx2::words); + assert_backend_matches_scalar_across_vector_boundaries(avx2::words); } #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] #[test] fn neon_matches_scalar() { assert_backend_matches_scalar(neon::words); + assert_backend_matches_scalar_across_vector_boundaries(neon::words); +} + +fn assert_backend_matches_scalar_across_vector_boundaries(backend: Backend) { + for len in [0, 1, 2, 3, 4, 5, 15, 16, 17, 18, 31, 32, 33, 63, 64, 65] { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let src: Vec<_> = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }) + .collect(); + for amount in [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 17, 31, 63, 64, 65, 127, 128, 129, 191, 256, 4096, + ] { + let expected = run_backend_owned(scalar::words, &src, amount); + assert_eq!( + run_backend_owned(backend, &src, amount), + expected, + "owned: len={len}, amount={amount}" + ); + assert_eq!( + run_backend_in_place(backend, &src, amount), + expected, + "in-place: len={len}, amount={amount}" + ); + } + } } diff --git a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs index aa69a99..16c59ce 100644 --- a/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/avx2.rs @@ -1,3 +1,5 @@ +//! AVX2 backend for copying shifted 64-bit windows. + use crate::WORD_BITS; #[cfg(target_arch = "x86")] @@ -19,6 +21,8 @@ pub(super) unsafe fn copy_words_shifted(dst: &mut [u64], src: &[u64], len: usize let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; let mut i = 0; while i + 4 <= len { + // The loads overlap by one word: lane j combines src[i + j] with + // src[i + j + 1], producing four consecutive shifted windows. // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm256_loadu_si256` uses unaligned loads so alignment is not required. let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; // SAFETY: Same as above. diff --git a/src/traits/words_pack.rs b/src/traits/words_pack.rs new file mode 100644 index 0000000..73536e7 --- /dev/null +++ b/src/traits/words_pack.rs @@ -0,0 +1,14 @@ +/// Fixed-width code packing operations on `[u64]` backing storage. +/// +/// The destination is expected to contain exactly the words produced by the +/// complete layout blocks in `codes`. Partial blocks and bit offsets are +/// handled by the packed-domain caller instead. Every code must fit in +/// `BITS` bits; callers are responsible for preserving this invariant. Every +/// destination word is fully overwritten, and its previous contents are +/// ignored. +pub(crate) trait WordsPack { + fn pack_codes(&mut self, codes: &[u8]); +} + +pub(crate) mod funcs_for_pack_codes_core; +mod impls_for_u64_slice; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core.rs b/src/traits/words_pack/funcs_for_pack_codes_core.rs new file mode 100644 index 0000000..f1e35eb --- /dev/null +++ b/src/traits/words_pack/funcs_for_pack_codes_core.rs @@ -0,0 +1,69 @@ +use crate::{assert_valid_width, code_mask, traits::layout_block_len, word_len}; + +#[inline] +pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { + assert_valid_width::(); + let layout_len = layout_block_len::(); + let expected_words = word_len(codes.len() * usize::from(BITS)); + + assert!( + codes.len().is_multiple_of(layout_len), + "code count must contain complete packed blocks" + ); + assert_eq!( + dst.len(), + expected_words, + "destination must contain exactly the packed block words" + ); + + debug_assert!( + codes.iter().all(|&code| code <= code_mask::()), + "packed code does not fit in BITS bits" + ); + + dispatch::(dst, codes); +} + +#[inline] +fn dispatch(dst: &mut [u64], codes: &[u8]) { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // Conservative thresholds selected from `benches/packed_thresholds.rs`. + const AVX2_BITS_1_THRESHOLD: usize = 64; + const AVX2_BITS_3_THRESHOLD: usize = 64; + const AVX2_BITS_4_THRESHOLD: usize = 64; + const AVX2_BITS_5_THRESHOLD: usize = 64; + const AVX2_BITS_6_THRESHOLD: usize = 32; + const AVX2_BITS_7_THRESHOLD: usize = 64; + + let use_avx2 = match BITS { + 1 => codes.len() >= AVX2_BITS_1_THRESHOLD, + 3 => codes.len() >= AVX2_BITS_3_THRESHOLD, + 4 => codes.len() >= AVX2_BITS_4_THRESHOLD, + 5 => codes.len() >= AVX2_BITS_5_THRESHOLD, + 6 => codes.len() >= AVX2_BITS_6_THRESHOLD, + 7 => codes.len() >= AVX2_BITS_7_THRESHOLD, + _ => false, + }; + + if use_avx2 { + // SAFETY: This branch is compiled only when AVX2 is enabled. + unsafe { avx2::pack_codes::(dst, codes) }; + return; + } + } + + scalar::pack_codes::(dst, codes); +} + +mod scalar; + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2; + +#[cfg(test)] +mod tests_for_backend_equivalence; diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs new file mode 100644 index 0000000..d1087fe --- /dev/null +++ b/src/traits/words_pack/funcs_for_pack_codes_core/avx2.rs @@ -0,0 +1,359 @@ +//! AVX2 kernels are retained for `BITS=1`, `BITS=3`, `BITS=4`, `BITS=5`, +//! `BITS=6`, and `BITS=7`. +//! +//! `BITS=2` and `BITS=8` intentionally remain scalar: tested AVX2 +//! implementations showed no stable construction-speed improvement. Re-run +//! the construction benchmarks before changing this support matrix. + +use super::scalar; +use crate::{WORD_BITS, traits::layout_block_len}; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_storeu_si128, _mm256_add_epi16, _mm256_castsi256_si128, + _mm256_cvtepu32_epi64, _mm256_loadu_si256, _mm256_madd_epi16, _mm256_maddubs_epi16, + _mm256_movemask_epi8, _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_permutevar8x32_epi32, + _mm256_setr_epi8, _mm256_setr_epi16, _mm256_setr_epi32, _mm256_shuffle_epi8, _mm256_slli_epi16, + _mm256_slli_epi64, _mm256_storeu_si256, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_storeu_si128, _mm256_add_epi16, _mm256_castsi256_si128, + _mm256_cvtepu32_epi64, _mm256_loadu_si256, _mm256_madd_epi16, _mm256_maddubs_epi16, + _mm256_movemask_epi8, _mm256_or_si256, _mm256_permute4x64_epi64, _mm256_permutevar8x32_epi32, + _mm256_setr_epi8, _mm256_setr_epi16, _mm256_setr_epi32, _mm256_shuffle_epi8, _mm256_slli_epi16, + _mm256_slli_epi64, _mm256_storeu_si256, +}; + +const CODES_PER_VECTOR: usize = 32; +// Each irregular-width group contains eight codes. +const BITS_3_GROUP_BITS: usize = 3 * 8; +const BITS_5_GROUP_BITS: usize = 5 * 8; +const BITS_7_GROUP_BITS: usize = 7 * 8; + +/// Packs the BITS=1, BITS=3, BITS=4, BITS=5, BITS=6, or BITS=7 SIMD prefix +/// and delegates complete remaining blocks to scalar. +/// +/// # Safety +/// +/// The caller must only invoke this function when AVX2 is available. The +/// slices must satisfy the `WordsPack` contract, and `BITS` must be 1, 3, 4, 5, 6, or 7. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn pack_codes(dst: &mut [u64], codes: &[u8]) { + debug_assert!(matches!(BITS, 1 | 3 | 4 | 5 | 6 | 7)); + debug_assert_eq!(codes.len() % layout_block_len::(), 0); + debug_assert_eq!(dst.len(), codes.len() * usize::from(BITS) / 64); + + let simd_code_block_len = if matches!(BITS, 1 | 3 | 5 | 7) { + 64 + } else { + CODES_PER_VECTOR + }; + let simd_code_len = codes.len() / simd_code_block_len * simd_code_block_len; + let simd_word_len = simd_code_len * usize::from(BITS) / 64; + let simd_word_block_len = simd_code_block_len * usize::from(BITS) / WORD_BITS; + + for (code_chunk, word_chunk) in codes[..simd_code_len] + .chunks_exact(simd_code_block_len) + .zip(dst[..simd_word_len].chunks_exact_mut(simd_word_block_len)) + { + // SAFETY: The selected kernel reads only initialized bytes from this + // chunk, and the backend is compiled with AVX2 enabled. + unsafe { + match BITS { + 1 => pack_64_one_bit(code_chunk, word_chunk), + 3 => pack_64_three_bit(code_chunk, word_chunk), + 4 => pack_32_four_bit(code_chunk, word_chunk), + 5 => pack_64_five_bit(code_chunk, word_chunk), + 6 => pack_32_six_bit(code_chunk, word_chunk), + 7 => pack_64_seven_bit(code_chunk, word_chunk), + _ => unreachable!(), + } + } + } + + if simd_code_len < codes.len() { + scalar::pack_codes::(&mut dst[simd_word_len..], &codes[simd_code_len..]); + } +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_64_one_bit(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), 64); + debug_assert_eq!(words.len(), 1); + + // SAFETY: Both loads read exactly 32 initialized bytes. + let (low, high) = unsafe { + ( + _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()), + _mm256_loadu_si256(codes.as_ptr().add(32).cast::<__m256i>()), + ) + }; + let low = _mm256_movemask_epi8(_mm256_slli_epi16(low, 7)) as u32; + let high = _mm256_movemask_epi8(_mm256_slli_epi16(high, 7)) as u32; + words[0] = u64::from(low) | (u64::from(high) << 32); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_64_three_bit(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), 64); + debug_assert_eq!(words.len(), 3); + + let mut groups = [0u64; 8]; + // SAFETY: Each half contains exactly 32 initialized codes and has space + // for the four 24-bit groups produced by the kernel. + unsafe { + pack_32_three_bit_groups(&codes[..32], &mut groups[..4]); + pack_32_three_bit_groups(&codes[32..], &mut groups[4..]); + } + + write_groups::(words, &groups); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_three_bit_groups(codes: &[u8], groups: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); + debug_assert_eq!(groups.len(), 4); + + let pair_weights = _mm256_setr_epi8( + 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, // + 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, + ); + let group_weights = _mm256_setr_epi16( + 1, 64, 1, 64, 1, 64, 1, 64, // + 1, 64, 1, 64, 1, 64, 1, 64, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_maddubs_epi16(input, pair_weights); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let packed = _mm256_or_si256(even, _mm256_slli_epi64(odd, 12)); + + // SAFETY: `groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_four_bit(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); + debug_assert_eq!(words.len(), 2); + + let even_indices = _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, // + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, + ); + let odd_indices = _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, // + 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let even = _mm256_shuffle_epi8(input, even_indices); + let odd = _mm256_shuffle_epi8(input, odd_indices); + let packed = _mm256_or_si256(even, _mm256_slli_epi16(odd, 4)); + let compacted = _mm256_permute4x64_epi64(packed, 0x88); + + // SAFETY: `compacted` contains two output words in its low 128 bits, and + // `words` has space for exactly those two words. + unsafe { + _mm_storeu_si128( + words.as_mut_ptr().cast::<__m128i>(), + _mm256_castsi256_si128(compacted), + ); + } +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_64_five_bit(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), 64); + debug_assert_eq!(words.len(), 5); + + let mut groups = [0u64; 8]; + // SAFETY: Each half contains exactly 32 initialized codes and has space + // for the four 40-bit groups produced by the kernel. + unsafe { + pack_32_five_bit_groups(&codes[..32], &mut groups[..4]); + pack_32_five_bit_groups(&codes[32..], &mut groups[4..]); + } + + write_groups::(words, &groups); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_five_bit_groups(codes: &[u8], groups: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); + debug_assert_eq!(groups.len(), 4); + + let pair_weights = _mm256_setr_epi8( + 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, // + 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, + ); + let group_weights = _mm256_setr_epi16( + 1, 1024, 1, 1024, 1, 1024, 1, 1024, // + 1, 1024, 1, 1024, 1, 1024, 1, 1024, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_maddubs_epi16(input, pair_weights); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let packed = _mm256_or_si256(even, _mm256_slli_epi64(odd, 20)); + + // SAFETY: `groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_six_bit(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); + debug_assert_eq!(words.len(), 3); + + // First combine adjacent codes into 12-bit pairs, then combine adjacent + // pairs into 24-bit values. Two values make one complete 48-bit group. + let pair_weights = _mm256_setr_epi8( + 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, // + 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, + ); + let group_weights = _mm256_setr_epi16( + 1, 4096, 1, 4096, 1, 4096, 1, 4096, // + 1, 4096, 1, 4096, 1, 4096, 1, 4096, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_maddubs_epi16(input, pair_weights); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let groups = _mm256_or_si256(even, _mm256_slli_epi64(odd, 24)); + + let mut packed_groups = [0u64; 4]; + // SAFETY: `packed_groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(packed_groups.as_mut_ptr().cast::<__m256i>(), groups) }; + + words[0] = packed_groups[0] | (packed_groups[1] << 48); + words[1] = (packed_groups[1] >> 16) | (packed_groups[2] << 32); + words[2] = (packed_groups[2] >> 32) | (packed_groups[3] << 16); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_64_seven_bit(codes: &[u8], words: &mut [u64]) { + debug_assert_eq!(codes.len(), 64); + debug_assert_eq!(words.len(), 7); + + let mut groups = [0u64; 8]; + // SAFETY: Each half contains exactly 32 initialized codes and has space + // for the four 56-bit groups produced by the kernel. + unsafe { + pack_32_seven_bit_groups(&codes[..32], &mut groups[..4]); + pack_32_seven_bit_groups(&codes[32..], &mut groups[4..]); + } + + write_groups::(words, &groups); +} + +#[target_feature(enable = "avx2")] +unsafe fn pack_32_seven_bit_groups(codes: &[u8], groups: &mut [u64]) { + debug_assert_eq!(codes.len(), CODES_PER_VECTOR); + debug_assert_eq!(groups.len(), 4); + + // `maddubs` uses signed byte weights, so 128 cannot be represented + // directly. The first product uses 127 and the second adds the odd code. + let pair_weights = _mm256_setr_epi8( + 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, // + 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, + ); + let odd_weights = _mm256_setr_epi8( + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, // + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + ); + let group_weights = _mm256_setr_epi16( + 1, 16384, 1, 16384, 1, 16384, 1, 16384, // + 1, 16384, 1, 16384, 1, 16384, 1, 16384, + ); + + // SAFETY: `codes` contains exactly 32 initialized bytes. + let input = unsafe { _mm256_loadu_si256(codes.as_ptr().cast::<__m256i>()) }; + let pairs = _mm256_add_epi16( + _mm256_maddubs_epi16(input, pair_weights), + _mm256_maddubs_epi16(input, odd_weights), + ); + let values = _mm256_madd_epi16(pairs, group_weights); + + let even_indices = _mm256_setr_epi32(0, 2, 4, 6, 0, 2, 4, 6); + let odd_indices = _mm256_setr_epi32(1, 3, 5, 7, 1, 3, 5, 7); + let even = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + even_indices, + ))); + let odd = _mm256_cvtepu32_epi64(_mm256_castsi256_si128(_mm256_permutevar8x32_epi32( + values, + odd_indices, + ))); + let packed = _mm256_or_si256(even, _mm256_slli_epi64(odd, 28)); + + // SAFETY: `groups` has space for exactly four u64 values. + unsafe { _mm256_storeu_si256(groups.as_mut_ptr().cast::<__m256i>(), packed) }; +} + +#[inline] +fn write_groups(words: &mut [u64], groups: &[u64]) { + let mut accumulator = 0u64; + let mut accumulated_bits = 0; + let mut word_index = 0; + + for &group in groups { + let next_bits = accumulated_bits + GROUP_BITS; + if next_bits < WORD_BITS { + accumulator |= group << accumulated_bits; + accumulated_bits = next_bits; + } else if next_bits == WORD_BITS { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = 0; + accumulated_bits = 0; + } else { + words[word_index] = accumulator | (group << accumulated_bits); + word_index += 1; + accumulator = group >> (WORD_BITS - accumulated_bits); + accumulated_bits = next_bits - WORD_BITS; + } + } + + debug_assert_eq!(accumulated_bits, 0); + debug_assert_eq!(word_index, words.len()); +} diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs new file mode 100644 index 0000000..bc91414 --- /dev/null +++ b/src/traits/words_pack/funcs_for_pack_codes_core/scalar.rs @@ -0,0 +1,98 @@ +/// Packs complete layout blocks into complete little-endian words. +pub(super) fn pack_codes(dst: &mut [u64], codes: &[u8]) { + match BITS { + 8 => pack_bytes(dst, codes), + 4 => pack_nibbles(dst, codes), + 2 => pack_pairs(dst, codes), + 1 | 3 | 5 | 6 | 7 => pack_groups::(dst, codes), + _ => unreachable!("BITS validated by WordsPack core"), + } +} + +#[inline] +fn pack_bytes(dst: &mut [u64], codes: &[u8]) { + debug_assert_eq!(codes.len(), dst.len() * 8); + + for (word, chunk) in dst.iter_mut().zip(codes.chunks_exact(8)) { + *word = u64::from_le_bytes(chunk.try_into().expect("chunk has eight bytes")); + } +} + +#[inline] +fn pack_nibbles(dst: &mut [u64], codes: &[u8]) { + debug_assert_eq!(codes.len(), dst.len() * 16); + + for (word, chunk) in dst.iter_mut().zip(codes.chunks_exact(16)) { + let mut packed = 0u64; + for (index, &code) in chunk.iter().enumerate() { + packed |= u64::from(code) << (index * 4); + } + *word = packed; + } +} + +#[inline] +fn pack_pairs(dst: &mut [u64], codes: &[u8]) { + debug_assert_eq!(codes.len(), dst.len() * 32); + + for (word, chunk) in dst.iter_mut().zip(codes.chunks_exact(32)) { + let mut packed = 0u64; + for (index, &code) in chunk.iter().enumerate() { + packed |= u64::from(code) << (index * 2); + } + *word = packed; + } +} + +#[inline] +fn pack_groups(dst: &mut [u64], codes: &[u8]) { + let width = usize::from(BITS); + let group_bits = width * 8; + let mut accumulator = 0u64; + let mut accumulated_bits = 0; + let mut word_index = 0; + + for group in codes.chunks_exact(8) { + let packed_group = pack_group::(group); + + let next_bits = accumulated_bits + group_bits; + if next_bits < 64 { + accumulator |= packed_group << accumulated_bits; + accumulated_bits = next_bits; + } else if next_bits == 64 { + dst[word_index] = accumulator | (packed_group << accumulated_bits); + word_index += 1; + accumulator = 0; + accumulated_bits = 0; + } else { + dst[word_index] = accumulator | (packed_group << accumulated_bits); + word_index += 1; + accumulator = packed_group >> (64 - accumulated_bits); + accumulated_bits = next_bits - 64; + } + } + + debug_assert_eq!(accumulated_bits, 0); + debug_assert_eq!(word_index, dst.len()); +} + +#[inline] +fn pack_group(group: &[u8]) -> u64 { + if BITS == 6 { + return u64::from(group[0]) + | (u64::from(group[1]) << 6) + | (u64::from(group[2]) << 12) + | (u64::from(group[3]) << 18) + | (u64::from(group[4]) << 24) + | (u64::from(group[5]) << 30) + | (u64::from(group[6]) << 36) + | (u64::from(group[7]) << 42); + } + + let width = usize::from(BITS); + let mut packed = 0u64; + for (index, &code) in group.iter().enumerate() { + packed |= u64::from(code) << (index * width); + } + packed +} diff --git a/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs new file mode 100644 index 0000000..e2afb37 --- /dev/null +++ b/src/traits/words_pack/funcs_for_pack_codes_core/tests_for_backend_equivalence.rs @@ -0,0 +1,183 @@ +use alloc::vec; +use alloc::vec::Vec; + +use super::scalar; +use crate::traits::layout_block_len; + +fn codes(len: usize) -> Vec { + let mask = crate::code_mask::(); + (0..len) + .map(|index| ((index.wrapping_mul(29) + 7) as u8) & mask) + .collect() +} + +fn run(codes: &[u8]) -> Vec { + let words = codes.len() * usize::from(BITS) / 64; + let mut dst = vec![0; words]; + scalar::pack_codes::(&mut dst, codes); + dst +} + +#[test] +fn scalar_packs_complete_layout_blocks_for_all_widths() { + for bits in 1..=8 { + match bits { + 1 => assert_width::<1>(), + 2 => assert_width::<2>(), + 3 => assert_width::<3>(), + 4 => assert_width::<4>(), + 5 => assert_width::<5>(), + 6 => assert_width::<6>(), + 7 => assert_width::<7>(), + 8 => assert_width::<8>(), + _ => unreachable!(), + } + } +} + +fn assert_width() { + let block_len = layout_block_len::(); + for blocks in 0..=3 { + let input = codes::(blocks * block_len); + let actual = run::(&input); + + let mut expected = vec![0; actual.len()]; + let mut bit = 0; + for &code in &input { + let word = bit / 64; + let offset = bit % 64; + expected[word] |= u64::from(code) << offset; + if offset + usize::from(BITS) > 64 { + expected[word + 1] |= u64::from(code) >> (64 - offset); + } + bit += usize::from(BITS); + } + + assert_eq!(actual, expected, "BITS={BITS}, blocks={blocks}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +mod tests_for_avx2 { + use super::super::avx2; + use super::*; + + #[test] + fn bits_four_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 16, 32, 48, 64, 96, 128] { + for input in [ + vec![0; code_len], + vec![0x0f; code_len], + codes::<4>(code_len), + ] { + let expected = run::<4>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<4>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } + + #[test] + fn bits_three_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 64, 128, 192, 256] { + for input in [ + vec![0; code_len], + vec![0x07; code_len], + codes::<3>(code_len), + ] { + let expected = run::<3>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<3>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } + + #[test] + fn bits_five_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 64, 128, 192, 256] { + for input in [ + vec![0; code_len], + vec![0x1f; code_len], + codes::<5>(code_len), + ] { + let expected = run::<5>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<5>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } + + #[test] + fn bits_seven_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 64, 128, 192, 256] { + for input in [ + vec![0; code_len], + vec![0x7f; code_len], + codes::<7>(code_len), + ] { + let expected = run::<7>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<7>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } + + #[test] + fn bits_one_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 64, 128, 192, 256] { + for input in [vec![0; code_len], vec![1; code_len], codes::<1>(code_len)] { + let expected = run::<1>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<1>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } + + #[test] + fn bits_six_matches_scalar_across_simd_prefix_and_tail() { + for code_len in [0, 32, 64, 96, 128] { + for input in [ + vec![0; code_len], + vec![0x3f; code_len], + codes::<6>(code_len), + ] { + let expected = run::<6>(&input); + let mut actual = vec![u64::MAX; expected.len()]; + + // SAFETY: This test is compiled only when AVX2 is enabled, + // and the input satisfies the WordsPack contract. + unsafe { avx2::pack_codes::<6>(&mut actual, &input) }; + + assert_eq!(actual, expected, "code_len={code_len}, input={input:?}"); + } + } + } +} diff --git a/src/traits/words_pack/impls_for_u64_slice.rs b/src/traits/words_pack/impls_for_u64_slice.rs new file mode 100644 index 0000000..fef45fc --- /dev/null +++ b/src/traits/words_pack/impls_for_u64_slice.rs @@ -0,0 +1,9 @@ +use super::WordsPack; +use super::funcs_for_pack_codes_core; + +impl WordsPack for [u64] { + #[inline] + fn pack_codes(&mut self, codes: &[u8]) { + funcs_for_pack_codes_core::pack_codes::(self, codes); + } +} diff --git a/src/traits/words_unpack.rs b/src/traits/words_unpack.rs new file mode 100644 index 0000000..1359ca1 --- /dev/null +++ b/src/traits/words_unpack.rs @@ -0,0 +1,43 @@ +/// Fixed-width code unpacking operations on `[u64]` backing storage. +/// +/// The input must contain complete layout blocks and the output length must +/// equal the number of codes represented by those words. Every output code is +/// fully overwritten and contains only its low `BITS` bits. +pub(crate) trait WordsUnpack { + fn unpack_codes(&self, codes: &mut [u8]); +} + +pub(crate) mod funcs_for_unpack_codes_core; +mod impls_for_u64_slice; + +pub(crate) use funcs_for_unpack_codes_core::has_accelerated_bulk_unpack; + +#[cfg(test)] +mod tests_for_words_unpack; + +#[cfg(test)] +mod tests_for_bulk_backend_availability { + use super::has_accelerated_bulk_unpack; + + #[test] + fn availability_matches_compiled_avx2_widths() { + let has_avx2 = cfg!(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + )); + + for (bits, has_backend) in [ + (1, has_accelerated_bulk_unpack::<1>()), + (2, has_accelerated_bulk_unpack::<2>()), + (3, has_accelerated_bulk_unpack::<3>()), + (4, has_accelerated_bulk_unpack::<4>()), + (5, has_accelerated_bulk_unpack::<5>()), + (6, has_accelerated_bulk_unpack::<6>()), + (7, has_accelerated_bulk_unpack::<7>()), + (8, has_accelerated_bulk_unpack::<8>()), + ] { + let has_kernel = matches!(bits, 1 | 2 | 4 | 6 | 8); + assert_eq!(has_backend, has_avx2 && has_kernel, "BITS={bits}"); + } + } +} diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs new file mode 100644 index 0000000..eec2be2 --- /dev/null +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core.rs @@ -0,0 +1,107 @@ +use crate::{WORD_BITS, assert_valid_width, traits::layout_block_len}; + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_1_THRESHOLD: usize = 64; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_2_THRESHOLD: usize = 64; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_4_THRESHOLD: usize = 64; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_6_THRESHOLD: usize = 32; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const AVX2_BITS_8_THRESHOLD: usize = 64; + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +const fn avx2_threshold() -> Option { + match BITS { + 1 => Some(AVX2_BITS_1_THRESHOLD), + 2 => Some(AVX2_BITS_2_THRESHOLD), + 4 => Some(AVX2_BITS_4_THRESHOLD), + 6 => Some(AVX2_BITS_6_THRESHOLD), + 8 => Some(AVX2_BITS_8_THRESHOLD), + _ => None, + } +} + +pub(crate) const fn has_accelerated_bulk_unpack() -> bool { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + return avx2_threshold::().is_some(); + } + + #[cfg(not(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + )))] + { + false + } +} + +#[inline] +pub(super) fn unpack_codes(words: &[u64], codes: &mut [u8]) { + assert_valid_width::(); + let expected_words = codes.len() * usize::from(BITS) / WORD_BITS; + + assert!( + codes.len().is_multiple_of(layout_block_len::()), + "code count must contain complete packed blocks" + ); + assert_eq!( + words.len(), + expected_words, + "input must contain exactly the words represented by the codes" + ); + + dispatch::(words, codes); +} + +#[inline] +fn dispatch(words: &[u64], codes: &mut [u8]) { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // Thresholds match complete AVX2 kernel blocks. BITS=6 uses its + // 32-code block; the other retained kernels process 64 codes. + if avx2_threshold::().is_some_and(|threshold| codes.len() >= threshold) { + // SAFETY: This branch is compiled only when AVX2 is enabled, and + // the core has validated the complete-block input/output contract. + unsafe { avx2::unpack_codes::(words, codes) }; + return; + } + } + + scalar::unpack_codes::(words, codes); +} + +mod scalar; + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2; + +#[cfg(test)] +mod tests_for_backend_equivalence; diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs new file mode 100644 index 0000000..6f1f2f4 --- /dev/null +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/avx2.rs @@ -0,0 +1,254 @@ +//! AVX2 kernels are retained for BITS=1, BITS=2, BITS=4, BITS=6, and BITS=8. +//! BITS=3, BITS=5, and BITS=7 intentionally remain scalar: a tested generic +//! AVX2 gather regressed bulk conversion. Re-benchmark before adding kernels. + +use super::scalar; + +#[cfg(target_arch = "x86")] +use core::arch::x86::{ + __m128i, __m256i, _mm_and_si128, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_set1_epi8, + _mm_srli_epi16, _mm_srli_si128, _mm_storeu_si128, _mm_unpackhi_epi8, _mm_unpackhi_epi16, + _mm_unpacklo_epi8, _mm_unpacklo_epi16, _mm256_and_si256, _mm256_castsi256_si128, + _mm256_cvtepu8_epi64, _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_or_si256, + _mm256_packus_epi16, _mm256_packus_epi32, _mm256_set1_epi8, _mm256_set1_epi32, + _mm256_set1_epi64x, _mm256_setr_epi8, _mm256_shuffle_epi8, _mm256_slli_epi64, + _mm256_srli_epi16, _mm256_srli_epi32, _mm256_storeu_si256, _mm256_unpackhi_epi8, + _mm256_unpacklo_epi8, +}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{ + __m128i, __m256i, _mm_and_si128, _mm_cvtsi64_si128, _mm_loadu_si128, _mm_set1_epi8, + _mm_srli_epi16, _mm_srli_si128, _mm_storeu_si128, _mm_unpackhi_epi8, _mm_unpackhi_epi16, + _mm_unpacklo_epi8, _mm_unpacklo_epi16, _mm256_and_si256, _mm256_castsi256_si128, + _mm256_cvtepu8_epi64, _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_or_si256, + _mm256_packus_epi16, _mm256_packus_epi32, _mm256_set1_epi8, _mm256_set1_epi32, + _mm256_set1_epi64x, _mm256_setr_epi8, _mm256_shuffle_epi8, _mm256_slli_epi64, + _mm256_srli_epi16, _mm256_srli_epi32, _mm256_storeu_si256, _mm256_unpackhi_epi8, + _mm256_unpacklo_epi8, +}; + +const CODES_PER_BLOCK: usize = 64; +const BITS_1_WORDS_PER_BLOCK: usize = 1; +const BITS_2_WORDS_PER_BLOCK: usize = 2; +const BITS_4_WORDS_PER_BLOCK: usize = 4; +const BITS_6_CODES_PER_BLOCK: usize = 32; +const BITS_6_WORDS_PER_BLOCK: usize = 3; +const BITS_8_WORDS_PER_BLOCK: usize = 8; + +/// AVX2 backend entry for BITS=1, BITS=2, BITS=4, BITS=6, and BITS=8. +/// It handles its SIMD prefix and scalar tail. +/// +/// # Safety +/// +/// The caller must ensure AVX2 is available and that the slices satisfy the +/// complete-layout-block `WordsUnpack` contract for `BITS=1`, `BITS=2`, `BITS=4`, `BITS=6`, or `BITS=8`. +#[target_feature(enable = "avx2")] +pub(super) unsafe fn unpack_codes(words: &[u64], codes: &mut [u8]) { + if BITS == 6 { + // BITS=6 has a 32-code AVX2 block, two blocks per 64-code batch. + let simd_code_len = codes.len() / BITS_6_CODES_PER_BLOCK * BITS_6_CODES_PER_BLOCK; + let simd_word_len = simd_code_len / BITS_6_CODES_PER_BLOCK * BITS_6_WORDS_PER_BLOCK; + for (word_block, code_block) in words[..simd_word_len] + .chunks_exact(BITS_6_WORDS_PER_BLOCK) + .zip(codes[..simd_code_len].chunks_exact_mut(BITS_6_CODES_PER_BLOCK)) + { + // SAFETY: Each block has exactly three initialized words and 32 + // output bytes, as required by the BITS=6 kernel. + unsafe { unpack_32_six_bit(word_block, code_block) }; + } + + scalar::unpack_codes::(&words[simd_word_len..], &mut codes[simd_code_len..]); + return; + } + + debug_assert!(matches!(BITS, 1 | 2 | 4 | 8)); + let words_per_block = match BITS { + 1 => BITS_1_WORDS_PER_BLOCK, + 2 => BITS_2_WORDS_PER_BLOCK, + 4 => BITS_4_WORDS_PER_BLOCK, + 8 => BITS_8_WORDS_PER_BLOCK, + _ => unreachable!(), + }; + + let simd_code_len = codes.len() / CODES_PER_BLOCK * CODES_PER_BLOCK; + let simd_word_len = simd_code_len / CODES_PER_BLOCK * words_per_block; + + for (word_block, code_block) in words[..simd_word_len] + .chunks_exact(words_per_block) + .zip(codes[..simd_code_len].chunks_exact_mut(CODES_PER_BLOCK)) + { + // SAFETY: Each paired block has the width-specific initialized input + // words and 64 initialized output bytes required by its kernel. + unsafe { + match BITS { + 1 => unpack_64_one_bit(word_block, code_block), + 2 => unpack_64_two_bit(word_block, code_block), + 4 => unpack_64_four_bit(word_block, code_block), + 8 => unpack_64_eight_bit(word_block, code_block), + _ => unreachable!(), + } + }; + } + + scalar::unpack_codes::(&words[simd_word_len..], &mut codes[simd_code_len..]); +} + +#[target_feature(enable = "avx2")] +unsafe fn unpack_32_six_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), 3); + debug_assert_eq!(codes.len(), 32); + + let mut groups = [0u32; 8]; + for (group_index, group) in groups.iter_mut().enumerate() { + let bit_position = group_index * 24; + let word_index = bit_position / 64; + let bit_offset = bit_position % 64; + let mut packed = words[word_index] >> bit_offset; + if bit_offset + 24 > 64 { + packed |= words[word_index + 1] << (64 - bit_offset); + } + *group = packed as u32 & 0x00ff_ffff; + } + + // SAFETY: `groups` contains eight initialized 24-bit groups. + let packed = unsafe { _mm256_loadu_si256(groups.as_ptr().cast::<__m256i>()) }; + let code_mask = _mm256_set1_epi32(0x3f); + let first = _mm256_and_si256(packed, code_mask); + let second = _mm256_and_si256(_mm256_srli_epi32::<6>(packed), code_mask); + let third = _mm256_and_si256(_mm256_srli_epi32::<12>(packed), code_mask); + let fourth = _mm256_and_si256(_mm256_srli_epi32::<18>(packed), code_mask); + + let first_pair = _mm256_packus_epi32(first, second); + let second_pair = _mm256_packus_epi32(third, fourth); + let column_major = _mm256_packus_epi16(first_pair, second_pair); + let transpose = _mm256_setr_epi8( + 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, // + 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, + ); + let output = _mm256_shuffle_epi8(column_major, transpose); + + // SAFETY: `output` contains exactly 32 codes and the output block has + // exactly 32 initialized bytes. + unsafe { _mm256_storeu_si256(codes.as_mut_ptr().cast::<__m256i>(), output) }; +} + +#[target_feature(enable = "avx2")] +unsafe fn unpack_64_one_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), BITS_1_WORDS_PER_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_BLOCK); + + let packed = _mm_cvtsi64_si128(words[0] as i64); + let low = _mm256_cvtepu8_epi64(packed); + let high = _mm256_cvtepu8_epi64(_mm_srli_si128::<4>(packed)); + let spread_mask = _mm256_set1_epi64x(0x0101_0101_0101_0101); + let low = spread_one_bit_bytes(low, spread_mask); + let high = spread_one_bit_bytes(high, spread_mask); + + // SAFETY: Each vector contains exactly 32 output bytes, and the output + // slice contains exactly 64 initialized bytes. + unsafe { + _mm256_storeu_si256(codes.as_mut_ptr().cast::<__m256i>(), low); + _mm256_storeu_si256(codes.as_mut_ptr().add(32).cast::<__m256i>(), high); + } +} + +#[target_feature(enable = "avx2")] +fn spread_one_bit_bytes(mut bits: __m256i, output_mask: __m256i) -> __m256i { + bits = _mm256_and_si256( + _mm256_or_si256(bits, _mm256_slli_epi64::<28>(bits)), + _mm256_set1_epi64x(0x0000_000f_0000_000f), + ); + bits = _mm256_and_si256( + _mm256_or_si256(bits, _mm256_slli_epi64::<14>(bits)), + _mm256_set1_epi64x(0x0003_0003_0003_0003), + ); + _mm256_and_si256( + _mm256_or_si256(bits, _mm256_slli_epi64::<7>(bits)), + output_mask, + ) +} + +#[target_feature(enable = "avx2")] +unsafe fn unpack_64_two_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), BITS_2_WORDS_PER_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_BLOCK); + + let code_mask = _mm_set1_epi8(0x03); + // SAFETY: Two words contain exactly 16 initialized packed bytes. + let packed = unsafe { _mm_loadu_si128(words.as_ptr().cast::<__m128i>()) }; + let first = _mm_and_si128(packed, code_mask); + let second = _mm_and_si128(_mm_srli_epi16::<2>(packed), code_mask); + let third = _mm_and_si128(_mm_srli_epi16::<4>(packed), code_mask); + let fourth = _mm_and_si128(_mm_srli_epi16::<6>(packed), code_mask); + + let first_pair = _mm_unpacklo_epi8(first, second); + let second_pair = _mm_unpacklo_epi8(third, fourth); + let third_pair = _mm_unpackhi_epi8(first, second); + let fourth_pair = _mm_unpackhi_epi8(third, fourth); + + // Interleave four 2-bit codes from each packed byte into sequential order. + let output0 = _mm_unpacklo_epi16(first_pair, second_pair); + let output1 = _mm_unpackhi_epi16(first_pair, second_pair); + let output2 = _mm_unpacklo_epi16(third_pair, fourth_pair); + let output3 = _mm_unpackhi_epi16(third_pair, fourth_pair); + + // SAFETY: Four stores write exactly 64 bytes to the output slice. + unsafe { + _mm_storeu_si128(codes.as_mut_ptr().cast::<__m128i>(), output0); + _mm_storeu_si128(codes.as_mut_ptr().add(16).cast::<__m128i>(), output1); + _mm_storeu_si128(codes.as_mut_ptr().add(32).cast::<__m128i>(), output2); + _mm_storeu_si128(codes.as_mut_ptr().add(48).cast::<__m128i>(), output3); + } +} + +#[target_feature(enable = "avx2")] +unsafe fn unpack_64_four_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), BITS_4_WORDS_PER_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_BLOCK); + + let nibble_mask = _mm256_set1_epi8(0x0f); + // SAFETY: This block contains exactly 32 initialized bytes. + let packed = unsafe { _mm256_loadu_si256(words.as_ptr().cast::<__m256i>()) }; + let low = _mm256_and_si256(packed, nibble_mask); + let high = _mm256_and_si256(_mm256_srli_epi16(packed, 4), nibble_mask); + let first = _mm256_unpacklo_epi8(low, high); + let second = _mm256_unpackhi_epi8(low, high); + + // AVX2 byte unpack is lane-local. Store its four 128-bit halves in code order. + unsafe { + _mm_storeu_si128( + codes.as_mut_ptr().cast::<__m128i>(), + _mm256_castsi256_si128(first), + ); + _mm_storeu_si128( + codes.as_mut_ptr().add(16).cast::<__m128i>(), + _mm256_castsi256_si128(second), + ); + _mm_storeu_si128( + codes.as_mut_ptr().add(32).cast::<__m128i>(), + _mm256_extracti128_si256::<1>(first), + ); + _mm_storeu_si128( + codes.as_mut_ptr().add(48).cast::<__m128i>(), + _mm256_extracti128_si256::<1>(second), + ); + } +} + +#[target_feature(enable = "avx2")] +unsafe fn unpack_64_eight_bit(words: &[u64], codes: &mut [u8]) { + debug_assert_eq!(words.len(), BITS_8_WORDS_PER_BLOCK); + debug_assert_eq!(codes.len(), CODES_PER_BLOCK); + + // SAFETY: Eight words and 64 output bytes are initialized, so each load + // and store accesses exactly 32 bytes within its slice. + unsafe { + let input = words.as_ptr().cast::<__m256i>(); + let output = codes.as_mut_ptr().cast::<__m256i>(); + let first = _mm256_loadu_si256(input); + let second = _mm256_loadu_si256(input.add(1)); + _mm256_storeu_si256(output, first); + _mm256_storeu_si256(output.add(1), second); + } +} diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/scalar.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/scalar.rs new file mode 100644 index 0000000..a3d19f5 --- /dev/null +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/scalar.rs @@ -0,0 +1,30 @@ +use crate::WORD_BITS; + +pub(super) fn unpack_codes(words: &[u64], codes: &mut [u8]) { + let width = usize::from(BITS); + let mask = (1u64 << BITS) - 1; + let mut next_word_index = 0; + let mut pending_bits = 0u64; + let mut pending_bit_count = 0; + + for code in codes { + let value = if pending_bit_count >= width { + let value = pending_bits; + pending_bits >>= width; + pending_bit_count -= width; + value + } else { + let old_fill = pending_bit_count; + let next_word = words[next_word_index]; + next_word_index += 1; + + let value = pending_bits | (next_word << old_fill); + let used_from_next = width - old_fill; + pending_bits = next_word >> used_from_next; + pending_bit_count = WORD_BITS - used_from_next; + value + }; + + *code = (value & mask) as u8; + } +} diff --git a/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs b/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs new file mode 100644 index 0000000..abd56f5 --- /dev/null +++ b/src/traits/words_unpack/funcs_for_unpack_codes_core/tests_for_backend_equivalence.rs @@ -0,0 +1,192 @@ +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +use super::scalar; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +use crate::traits::WordsPack; +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +use alloc::vec; + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +fn assert_avx2_matches_scalar_for_each_code_position(code_len: usize) { + let code_mask = crate::code_mask::(); + + for position in 0..code_len { + let mut input = vec![0; code_len]; + input[position] = code_mask; + + let words_len = code_len * usize::from(BITS) / crate::WORD_BITS; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::(&words, &mut expected); + assert_eq!(expected, input, "scalar decode at position={position}"); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::(&words, &mut actual) }; + + assert_eq!(actual, input, "AVX2 decode at position={position}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_four_matches_scalar_across_prefix_and_tail() { + fn codes(len: usize) -> alloc::vec::Vec { + let mask = crate::code_mask::(); + (0..len) + .map(|index| ((index.wrapping_mul(29) + 7) as u8) & mask) + .collect() + } + + for code_len in [0, 16, 32, 48, 64, 80, 96, 112, 128, 144] { + let input = codes::<4>(code_len); + let words_len = code_len / 16; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<4>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<4>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<4>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_one_matches_scalar_across_blocks() { + for code_len in [0, 64, 128, 192, 256] { + let input = (0..code_len) + .map(|index: usize| ((index.wrapping_mul(29) + 7) as u8) & 0x01) + .collect::>(); + let words_len = code_len / 64; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<1>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<1>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<1>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_two_matches_scalar_across_prefix_and_tail() { + for code_len in [0, 32, 64, 96, 128, 160, 192] { + let input = (0..code_len) + .map(|index: usize| ((index.wrapping_mul(29) + 7) as u8) & 0x03) + .collect::>(); + let words_len = code_len / 32; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<2>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<2>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<2>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_eight_matches_scalar_across_prefix_and_tail() { + for code_len in [0, 8, 16, 32, 56, 64, 72, 96, 120, 128, 136] { + let input = (0..code_len) + .map(|index: usize| (index.wrapping_mul(29) + 7) as u8) + .collect::>(); + let words_len = code_len / 8; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<8>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<8>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<8>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_bits_six_matches_scalar_across_blocks() { + let mask = crate::code_mask::<6>(); + for code_len in [0, 32, 64, 96, 128, 160] { + let input = (0..code_len) + .map(|index: usize| (index.wrapping_mul(29) + 7) as u8 & mask) + .collect::>(); + let words_len = code_len * 6 / crate::WORD_BITS; + let mut words = vec![0; words_len]; + words.as_mut_slice().pack_codes::<6>(&input); + + let mut expected = vec![0; code_len]; + scalar::unpack_codes::<6>(&words, &mut expected); + + let mut actual = vec![u8::MAX; code_len]; + // SAFETY: This test only compiles when AVX2 is enabled, and the packed + // words and output satisfy the complete-layout-block contract. + unsafe { super::avx2::unpack_codes::<6>(&words, &mut actual) }; + + assert_eq!(actual, expected, "code_len={code_len}"); + } +} + +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" +))] +#[test] +fn avx2_unpack_preserves_each_code_position() { + assert_avx2_matches_scalar_for_each_code_position::<1>(64); + assert_avx2_matches_scalar_for_each_code_position::<2>(64); + assert_avx2_matches_scalar_for_each_code_position::<4>(64); + assert_avx2_matches_scalar_for_each_code_position::<6>(32); + assert_avx2_matches_scalar_for_each_code_position::<8>(64); +} diff --git a/src/traits/words_unpack/impls_for_u64_slice.rs b/src/traits/words_unpack/impls_for_u64_slice.rs new file mode 100644 index 0000000..8d3344f --- /dev/null +++ b/src/traits/words_unpack/impls_for_u64_slice.rs @@ -0,0 +1,9 @@ +use super::WordsUnpack; +use super::funcs_for_unpack_codes_core; + +impl WordsUnpack for [u64] { + #[inline] + fn unpack_codes(&self, codes: &mut [u8]) { + funcs_for_unpack_codes_core::unpack_codes::(self, codes); + } +} diff --git a/src/traits/words_unpack/tests_for_words_unpack.rs b/src/traits/words_unpack/tests_for_words_unpack.rs new file mode 100644 index 0000000..bf291f1 --- /dev/null +++ b/src/traits/words_unpack/tests_for_words_unpack.rs @@ -0,0 +1,37 @@ +use alloc::vec; + +use super::WordsUnpack; +use crate::traits::{WordsPack, layout_block_len}; + +#[test] +fn unpack_matches_codes_for_all_widths_and_layout_tails() { + fn check() { + let mask = crate::code_mask::(); + let codes = (0usize..257) + .map(|index| ((index.wrapping_mul(37) + 11) as u8) & mask) + .collect::>(); + let block_len = layout_block_len::(); + + for requested_len in [ + 0, 1, 15, 16, 31, 32, 63, 64, 65, 79, 80, 95, 96, 127, 128, 129, 257, + ] { + let code_len = requested_len / block_len * block_len; + let word_len = code_len * usize::from(BITS) / crate::WORD_BITS; + let mut words = vec![0; word_len]; + words.as_mut_slice().pack_codes::(&codes[..code_len]); + + let mut actual = vec![u8::MAX; code_len]; + words.as_slice().unpack_codes::(&mut actual); + assert_eq!(actual, codes[..code_len], "BITS={BITS}, len={code_len}"); + } + } + + check::<1>(); + check::<2>(); + check::<3>(); + check::<4>(); + check::<5>(); + check::<6>(); + check::<7>(); + check::<8>(); +}