From 7f851471f76b58b7d4dd37cc4c2560e8bfcf1711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D1=91=D0=BC=20=D0=9F=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=5BArtyom=20Pavlov=5D?= Date: Fri, 31 Jul 2026 18:01:57 +0300 Subject: [PATCH 1/3] aes: enable VAES backends by default, bump MSRV to 1.89 --- .github/workflows/aes.yml | 22 +-- aes/Cargo.toml | 4 +- aes/src/backends.rs | 33 ++-- aes/src/backends/aarch64_aes.rs | 50 +++--- aes/src/backends/aarch64_aes/encdec.rs | 215 +++++++++++-------------- aes/src/backends/aarch64_aes/expand.rs | 41 +++-- aes/src/backends/aarch64_aes/hazmat.rs | 77 +++++---- aes/src/backends/aarch64_aes/utils.rs | 14 ++ aes/src/backends/x86_aes.rs | 50 +++--- aes/src/backends/x86_aes/encdec.rs | 114 ++++++------- aes/src/backends/x86_aes/expand.rs | 52 +++--- aes/src/backends/x86_aes/hazmat.rs | 82 ++++------ aes/src/backends/x86_aes/utils.rs | 36 +++++ aes/src/backends/x86_vaes256.rs | 24 +-- aes/src/backends/x86_vaes256/encdec.rs | 26 +-- aes/src/backends/x86_vaes512.rs | 30 ++-- aes/src/backends/x86_vaes512/encdec.rs | 22 +-- aes/src/hazmat.rs | 1 + aes/src/lib.rs | 10 -- 19 files changed, 411 insertions(+), 492 deletions(-) create mode 100644 aes/src/backends/aarch64_aes/utils.rs create mode 100644 aes/src/backends/x86_aes/utils.rs diff --git a/.github/workflows/aes.yml b/.github/workflows/aes.yml index fe0d08dd..1b57281d 100644 --- a/.github/workflows/aes.yml +++ b/.github/workflows/aes.yml @@ -29,7 +29,7 @@ jobs: strategy: matrix: rust: - - 1.85.0 # MSRV + - 1.89.0 # MSRV - stable target: - thumbv7em-none-eabi @@ -78,7 +78,7 @@ jobs: include: # 32-bit Linux - target: i686-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV deps: sudo apt update && sudo apt install gcc-multilib - target: i686-unknown-linux-gnu rust: stable @@ -86,7 +86,7 @@ jobs: # 64-bit Linux - target: x86_64-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV - target: x86_64-unknown-linux-gnu rust: stable steps: @@ -177,7 +177,7 @@ jobs: include: # 32-bit Linux - target: i686-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV deps: sudo apt update && sudo apt install gcc-multilib - target: i686-unknown-linux-gnu rust: stable @@ -185,7 +185,7 @@ jobs: # 64-bit Linux - target: x86_64-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV - target: x86_64-unknown-linux-gnu rust: stable steps: @@ -210,7 +210,7 @@ jobs: include: # 32-bit Linux - target: i686-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV deps: sudo apt update && sudo apt install gcc-multilib - target: i686-unknown-linux-gnu rust: stable @@ -218,7 +218,7 @@ jobs: # 64-bit Linux - target: x86_64-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV - target: x86_64-unknown-linux-gnu rust: stable steps: @@ -239,13 +239,13 @@ jobs: include: # ARM64 - target: aarch64-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV - target: aarch64-unknown-linux-gnu rust: stable # PPC32 - target: powerpc-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV - target: powerpc-unknown-linux-gnu rust: stable runs-on: ubuntu-latest @@ -287,7 +287,7 @@ jobs: matrix: include: - target: aarch64-unknown-linux-gnu - rust: 1.85.0 # MSRV + rust: 1.89.0 # MSRV runs-on: ubuntu-latest # Cross mounts only current package, i.e. by default it ignores workspace's Cargo.toml defaults: @@ -315,6 +315,6 @@ jobs: - uses: RustCrypto/actions/cargo-cache@master - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.85.0 # MSRV + toolchain: 1.89.0 # MSRV components: clippy - run: cargo clippy --features hazmat -- -D warnings diff --git a/aes/Cargo.toml b/aes/Cargo.toml index dfd94b60..c8027f03 100644 --- a/aes/Cargo.toml +++ b/aes/Cargo.toml @@ -5,7 +5,7 @@ description = "Pure Rust implementation of the Advanced Encryption Standard (a.k authors = ["RustCrypto Developers"] license = "MIT OR Apache-2.0" edition = "2024" -rust-version = "1.85" +rust-version = "1.89" readme = "README.md" documentation = "https://docs.rs/aes" repository = "https://github.com/RustCrypto/block-ciphers" @@ -31,7 +31,7 @@ hazmat = [] # Expose cryptographically hazardous APIs level = "warn" check-cfg = [ 'cfg(aes_backend_soft, values("compact"))', - 'cfg(aes_backend, values("soft", "avx256", "avx512"))', + 'cfg(aes_backend, values("soft"))', 'cfg(cpubits, values("16", "32", "64"))' ] diff --git a/aes/src/backends.rs b/aes/src/backends.rs index 59761d51..4f687c8f 100644 --- a/aes/src/backends.rs +++ b/aes/src/backends.rs @@ -1,24 +1,13 @@ pub(crate) mod soft; -#[cfg(all(target_arch = "aarch64", not(aes_backend = "soft")))] -pub(crate) mod aarch64_aes; - -#[cfg(all( - any(target_arch = "x86_64", target_arch = "x86"), - not(aes_backend = "soft") -))] -pub(crate) mod x86_aes; - -#[cfg(all( - any(aes_backend = "avx256", aes_backend = "avx512"), - any(target_arch = "x86_64", target_arch = "x86"), - not(aes_backend = "soft"), -))] -pub(crate) mod x86_vaes256; - -#[cfg(all( - aes_backend = "avx512", - any(target_arch = "x86_64", target_arch = "x86"), - not(aes_backend = "soft"), -))] -pub(crate) mod x86_vaes512; +cpubits::cfg_if! { + if #[cfg(aes_backend = "soft")] { + // Do not add other backends if software backend is forced + } else if #[cfg(target_arch = "aarch64")] { + pub(crate) mod aarch64_aes; + } else if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { + pub(crate) mod x86_aes; + pub(crate) mod x86_vaes256; + pub(crate) mod x86_vaes512; + } +} diff --git a/aes/src/backends/aarch64_aes.rs b/aes/src/backends/aarch64_aes.rs index 8e8ac254..592d2fff 100644 --- a/aes/src/backends/aarch64_aes.rs +++ b/aes/src/backends/aarch64_aes.rs @@ -7,6 +7,7 @@ use cipher::{ mod encdec; mod expand; +mod utils; #[cfg(feature = "hazmat")] pub(crate) mod hazmat; @@ -36,15 +37,13 @@ pub(crate) struct Aes { impl Aes { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn encrypt(&self, f: impl BlockCipherEncClosure) { + pub(crate) fn encrypt(&self, f: impl BlockCipherEncClosure) { f.call(self); } #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn decrypt(&self, f: impl BlockCipherDecClosure) { + pub(crate) fn decrypt(&self, f: impl BlockCipherDecClosure) { f.call(self); } } @@ -69,7 +68,7 @@ impl BlockCipherEncBackend for Aes { fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with // `#[target_feature(enable = "aes")]` - unsafe { encdec::encrypt_par(&self.enc_rk, blocks) }; + unsafe { encdec::batch_encrypt(&self.enc_rk, blocks) }; } } @@ -85,7 +84,7 @@ impl BlockCipherDecBackend for Aes { fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with // `#[target_feature(enable = "aes")]` - unsafe { encdec::decrypt_par(&self.dec_rk, blocks) }; + unsafe { encdec::batch_decrypt(&self.dec_rk, blocks) }; } } @@ -97,25 +96,22 @@ pub(crate) struct AesEnc { impl AesEnc { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn as_encdec(&self) -> Aes { + pub(crate) fn as_encdec(&self) -> Aes { let enc_rk = self.enc_rk; - let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + let dec_rk = expand::inv_expanded_keys(&enc_rk); Aes { enc_rk, dec_rk } } #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn as_dec(&self) -> AesDec { - let dec_rk = unsafe { expand::inv_expanded_keys(&self.enc_rk) }; + pub(crate) fn as_dec(&self) -> AesDec { + let dec_rk = expand::inv_expanded_keys(&self.enc_rk); AesDec { dec_rk } } #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn encrypt(&self, f: impl BlockCipherEncClosure) { + pub(crate) fn encrypt(&self, f: impl BlockCipherEncClosure) { f.call(self) } } @@ -140,7 +136,7 @@ impl BlockCipherEncBackend for AesEnc { fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with // `#[target_feature(enable = "aes")]` - unsafe { encdec::encrypt_par(&self.enc_rk, blocks) }; + unsafe { encdec::batch_encrypt(&self.enc_rk, blocks) }; } } @@ -152,8 +148,7 @@ pub(crate) struct AesDec { impl AesDec { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn decrypt(&self, f: impl BlockCipherDecClosure) { + pub(crate) fn decrypt(&self, f: impl BlockCipherDecClosure) { f.call(self); } } @@ -178,7 +173,7 @@ impl BlockCipherDecBackend for AesDec { fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with // `#[target_feature(enable = "aes")]` - unsafe { encdec::decrypt_par(&self.dec_rk, blocks) }; + unsafe { encdec::batch_decrypt(&self.dec_rk, blocks) }; } } @@ -187,10 +182,9 @@ macro_rules! impl_key_init { impl $name { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { - let enc_rk = unsafe { expand::expand_key(key) }; - let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + pub(crate) fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = expand::expand_key(key); + let dec_rk = expand::inv_expanded_keys(&enc_rk); Self { enc_rk, dec_rk } } } @@ -198,9 +192,8 @@ macro_rules! impl_key_init { impl $name_enc { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { - let enc_rk = unsafe { expand::expand_key(key) }; + pub(crate) fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = expand::expand_key(key); Self { enc_rk } } } @@ -208,10 +201,9 @@ macro_rules! impl_key_init { impl $name_dec { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { - let enc_rk = unsafe { expand::expand_key(key) }; - let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + pub(crate) fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = expand::expand_key(key); + let dec_rk = expand::inv_expanded_keys(&enc_rk); Self { dec_rk } } } diff --git a/aes/src/backends/aarch64_aes/encdec.rs b/aes/src/backends/aarch64_aes/encdec.rs index d8175c9b..7f5c290b 100644 --- a/aes/src/backends/aarch64_aes/encdec.rs +++ b/aes/src/backends/aarch64_aes/encdec.rs @@ -1,183 +1,152 @@ //! AES encryption support //! //! Note that `aes` target feature implicitly enables `neon`, see: -//! https://doc.rust-lang.org/reference/attributes/codegen.html#aarch64 -#![allow(unsafe_op_in_unsafe_fn)] - +//! https://doc.rust-lang.org/reference/attributes/codegen.html#r-attributes.codegen.target_feature.aarch64 +use super::utils::{load_block, store_block}; use crate::Block; use cipher::{ array::{Array, ArraySize}, inout::InOut, }; -use core::{arch::aarch64::*, mem}; +use core::arch::aarch64::*; -/// Perform AES encryption using the given expanded keys. #[target_feature(enable = "aes")] #[inline] -pub(super) unsafe fn encrypt( - keys: &[uint8x16_t; KEYS], - block: InOut<'_, '_, Block>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); - let (in_ptr, out_ptr) = block.into_raw(); - let mut block = vld1q_u8(in_ptr.cast()); +pub(super) fn encrypt(keys: &[uint8x16_t; RK], mut block: InOut<'_, '_, Block>) { + const { assert!(matches!(RK, 11 | 13 | 15)) } - for &key in &keys[..KEYS - 2] { + let mut b = load_block(block.get_in()); + + for &key in &keys[..RK - 2] { // AES single round encryption - block = vaeseq_u8(block, key); + b = vaeseq_u8(b, key); // Mix columns - block = vaesmcq_u8(block); + b = vaesmcq_u8(b); } // AES single round encryption - block = vaeseq_u8(block, keys[KEYS - 2]); + b = vaeseq_u8(b, keys[RK - 2]); // Final add (bitwise XOR) - block = veorq_u8(block, keys[KEYS - 1]); + b = veorq_u8(b, keys[RK - 1]); - vst1q_u8(out_ptr.cast(), block); + store_block(block.get_out(), b); } -/// Perform AES decryption using the given expanded keys. #[target_feature(enable = "aes")] #[inline] -pub(super) unsafe fn decrypt( - keys: &[uint8x16_t; KEYS], - block: InOut<'_, '_, Block>, -) { - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); +pub(super) fn decrypt(keys: &[uint8x16_t; RK], mut block: InOut<'_, '_, Block>) { + const { assert!(matches!(RK, 11 | 13 | 15)) } - let (in_ptr, out_ptr) = block.into_raw(); - let mut block = vld1q_u8(in_ptr.cast()); + let mut b = load_block(block.get_in()); - for &key in &keys[..KEYS - 2] { + for &key in &keys[..RK - 2] { // AES single round decryption - block = vaesdq_u8(block, key); + b = vaesdq_u8(b, key); // Inverse mix columns - block = vaesimcq_u8(block); + b = vaesimcq_u8(b); } // AES single round decryption - block = vaesdq_u8(block, keys[KEYS - 2]); + b = vaesdq_u8(b, keys[RK - 2]); // Final add (bitwise XOR) - block = veorq_u8(block, keys[KEYS - 1]); + b = veorq_u8(b, keys[RK - 1]); - vst1q_u8(out_ptr.cast(), block); + store_block(block.get_out(), b); } -/// Perform parallel AES encryption 8-blocks-at-a-time using the given expanded keys. #[target_feature(enable = "aes")] #[inline] -pub(super) unsafe fn encrypt_par( - keys: &[uint8x16_t; KEYS], - blocks: InOut<'_, '_, Array>, +pub(super) fn batch_encrypt( + keys: &[uint8x16_t; RK], + mut blocks: InOut<'_, '_, Array>, ) { - #[target_feature(enable = "aes")] - unsafe fn par_round( - key: uint8x16_t, - blocks: &mut Array, - ) { - for block in blocks { - // AES single round encryption and mix columns - *block = vaesmcq_u8(vaeseq_u8(*block, key)); - } - } - - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); + const { assert!(matches!(RK, 11 | 13 | 15)) } - let (in_ptr, out_ptr) = blocks.into_raw(); - let in_ptr: *const Block = in_ptr.cast(); - let out_ptr: *mut Block = out_ptr.cast(); - - // Load plaintext blocks - let mut blocks: Array = mem::zeroed(); - for i in 0..ParBlocks::USIZE { - blocks[i] = vld1q_u8(in_ptr.add(i).cast()); - } + let mut b = load_batch_blocks(blocks.get_in()); // Loop is intentionally not used here to enforce inlining - par_round(keys[0], &mut blocks); - par_round(keys[1], &mut blocks); - par_round(keys[2], &mut blocks); - par_round(keys[3], &mut blocks); - par_round(keys[4], &mut blocks); - par_round(keys[5], &mut blocks); - par_round(keys[6], &mut blocks); - par_round(keys[7], &mut blocks); - par_round(keys[8], &mut blocks); - if KEYS >= 13 { - par_round(keys[9], &mut blocks); - par_round(keys[10], &mut blocks); + batch_enc_round(keys[0], &mut b); + batch_enc_round(keys[1], &mut b); + batch_enc_round(keys[2], &mut b); + batch_enc_round(keys[3], &mut b); + batch_enc_round(keys[4], &mut b); + batch_enc_round(keys[5], &mut b); + batch_enc_round(keys[6], &mut b); + batch_enc_round(keys[7], &mut b); + batch_enc_round(keys[8], &mut b); + if RK >= 13 { + batch_enc_round(keys[9], &mut b); + batch_enc_round(keys[10], &mut b); } - if KEYS == 15 { - par_round(keys[11], &mut blocks); - par_round(keys[12], &mut blocks); + if RK == 15 { + batch_enc_round(keys[11], &mut b); + batch_enc_round(keys[12], &mut b); } for i in 0..ParBlocks::USIZE { - // AES single round encryption - blocks[i] = vaeseq_u8(blocks[i], keys[KEYS - 2]); - // Final add (bitwise XOR) - blocks[i] = veorq_u8(blocks[i], keys[KEYS - 1]); - // Save encrypted blocks - vst1q_u8(out_ptr.add(i).cast(), blocks[i]); + b[i] = vaeseq_u8(b[i], keys[RK - 2]); + b[i] = veorq_u8(b[i], keys[RK - 1]); + store_block(&mut blocks.get_out()[i], b[i]); } } -/// Perform parallel AES decryption 8-blocks-at-a-time using the given expanded keys. #[target_feature(enable = "aes")] #[inline] -pub(super) unsafe fn decrypt_par( - keys: &[uint8x16_t; KEYS], - blocks: InOut<'_, '_, Array>, +pub(super) fn batch_decrypt( + keys: &[uint8x16_t; RK], + mut blocks: InOut<'_, '_, Array>, ) { - #[target_feature(enable = "aes")] - unsafe fn par_round( - key: uint8x16_t, - blocks: &mut Array, - ) { - for block in blocks { - // AES single round decryption and inverse mix columns - *block = vaesimcq_u8(vaesdq_u8(*block, key)); - } - } + const { assert!(matches!(RK, 11 | 13 | 15)) } - assert!(KEYS == 11 || KEYS == 13 || KEYS == 15); + let mut b = load_batch_blocks(blocks.get_in()); - let (in_ptr, out_ptr) = blocks.into_raw(); - let in_ptr: *const Block = in_ptr.cast(); - let out_ptr: *mut Block = out_ptr.cast(); + // Loop is intentionally not used here to enforce inlining + batch_dec_round(keys[0], &mut b); + batch_dec_round(keys[1], &mut b); + batch_dec_round(keys[2], &mut b); + batch_dec_round(keys[3], &mut b); + batch_dec_round(keys[4], &mut b); + batch_dec_round(keys[5], &mut b); + batch_dec_round(keys[6], &mut b); + batch_dec_round(keys[7], &mut b); + batch_dec_round(keys[8], &mut b); + if RK >= 13 { + batch_dec_round(keys[9], &mut b); + batch_dec_round(keys[10], &mut b); + } + if RK == 15 { + batch_dec_round(keys[11], &mut b); + batch_dec_round(keys[12], &mut b); + } - // Load encrypted blocks - let mut blocks: Array = mem::zeroed(); for i in 0..ParBlocks::USIZE { - blocks[i] = vld1q_u8(in_ptr.add(i).cast()); + b[i] = vaesdq_u8(b[i], keys[RK - 2]); + b[i] = veorq_u8(b[i], keys[RK - 1]); + store_block(&mut blocks.get_out()[i], b[i]); } +} - // Loop is intentionally not used here to enforce inlining - par_round(keys[0], &mut blocks); - par_round(keys[1], &mut blocks); - par_round(keys[2], &mut blocks); - par_round(keys[3], &mut blocks); - par_round(keys[4], &mut blocks); - par_round(keys[5], &mut blocks); - par_round(keys[6], &mut blocks); - par_round(keys[7], &mut blocks); - par_round(keys[8], &mut blocks); - if KEYS >= 13 { - par_round(keys[9], &mut blocks); - par_round(keys[10], &mut blocks); - } - if KEYS == 15 { - par_round(keys[11], &mut blocks); - par_round(keys[12], &mut blocks); +#[target_feature(enable = "aes")] +fn batch_enc_round( + key: uint8x16_t, + blocks: &mut Array, +) { + for block in blocks { + *block = vaesmcq_u8(vaeseq_u8(*block, key)); } +} - for i in 0..ParBlocks::USIZE { - // AES single round decryption - blocks[i] = vaesdq_u8(blocks[i], keys[KEYS - 2]); - // Final add (bitwise XOR) - blocks[i] = veorq_u8(blocks[i], keys[KEYS - 1]); - // Save plaintext blocks - vst1q_u8(out_ptr.add(i) as *mut u8, blocks[i]); +#[target_feature(enable = "aes")] +fn batch_dec_round( + key: uint8x16_t, + blocks: &mut Array, +) { + for block in blocks { + *block = vaesimcq_u8(vaesdq_u8(*block, key)); } } + +#[target_feature(enable = "neon")] +fn load_batch_blocks(blocks: &Array) -> Array { + Array::from_fn(|i| load_block(&blocks[i])) +} diff --git a/aes/src/backends/aarch64_aes/expand.rs b/aes/src/backends/aarch64_aes/expand.rs index 1623c9eb..1e6104d7 100644 --- a/aes/src/backends/aarch64_aes/expand.rs +++ b/aes/src/backends/aarch64_aes/expand.rs @@ -1,7 +1,5 @@ //! AES key expansion support. -#![allow(unsafe_op_in_unsafe_fn)] - -use core::{arch::aarch64::*, mem, slice}; +use core::arch::aarch64::*; pub(super) type RoundKeys = [uint8x16_t; RK]; @@ -15,17 +13,18 @@ const WORD_SIZE: usize = 4; const ROUND_CONSTS: [u32; 10] = [0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36]; /// AES key expansion. -#[inline] #[target_feature(enable = "aes")] -pub unsafe fn expand_key(key: &[u8; N]) -> [uint8x16_t; RK] { +#[inline] +pub(super) fn expand_key(key: &[u8; N]) -> [uint8x16_t; RK] { const { assert!(matches!((N, RK), (16, 11) | (24, 13) | (32, 15))) } - let mut expanded_keys: [uint8x16_t; RK] = mem::zeroed(); + let mut expanded_keys: [uint8x16_t; RK] = [vdupq_n_u8(0); RK]; // Sanity check, as this is required in order for the subsequent conversion to be sound. - const _: () = assert!(mem::align_of::() >= mem::align_of::()); + const { assert!(align_of::() >= align_of::()) } + let keys_ptr: *mut u32 = expanded_keys.as_mut_ptr().cast(); - let columns = slice::from_raw_parts_mut(keys_ptr, RK * BLOCK_WORDS); + let columns = unsafe { core::slice::from_raw_parts_mut(keys_ptr, RK * BLOCK_WORDS) }; for (i, chunk) in key.chunks_exact(WORD_SIZE).enumerate() { columns[i] = u32::from_ne_bytes(chunk.try_into().unwrap()); @@ -55,27 +54,25 @@ pub unsafe fn expand_key(key: &[u8; N]) -> [uin /// /// This is the reverse of the encryption keys, with the Inverse Mix Columns /// operation applied to all but the first and last expanded key. -#[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn inv_expanded_keys( - keys: &[uint8x16_t; RK], -) -> [uint8x16_t; RK] { +#[inline] +pub(super) fn inv_expanded_keys(keys: &[uint8x16_t; RK]) -> [uint8x16_t; RK] { const { assert!(matches!(RK, 11 | 13 | 15)) } - let mut inv_keys: [uint8x16_t; RK] = core::mem::zeroed(); - inv_keys[0] = keys[RK - 1]; - for i in 1..RK - 1 { - inv_keys[i] = vaesimcq_u8(keys[RK - 1 - i]); - } - inv_keys[RK - 1] = keys[0]; - - inv_keys + core::array::from_fn(|i| { + let k = keys[RK - 1 - i]; + if i == 0 || i == RK - 1 { + k + } else { + vaesimcq_u8(k) + } + }) } /// Sub bytes for a single AES word: used for key expansion. -#[inline] #[target_feature(enable = "aes")] -unsafe fn sub_word(input: u32) -> u32 { +#[inline] +fn sub_word(input: u32) -> u32 { let input = vreinterpretq_u8_u32(vdupq_n_u32(input)); // AES single round encryption (with a "round" key of all zeros) diff --git a/aes/src/backends/aarch64_aes/hazmat.rs b/aes/src/backends/aarch64_aes/hazmat.rs index 053704c1..bec5c38d 100644 --- a/aes/src/backends/aarch64_aes/hazmat.rs +++ b/aes/src/backends/aarch64_aes/hazmat.rs @@ -1,105 +1,100 @@ //! Low-level "hazmat" AES functions: ARMv8 Cryptography Extensions support. //! //! Note: this isn't actually used in the `Aes128`/`Aes192`/`Aes256` -//! implementations in this crate, but instead provides raw AES-NI accelerated +//! implementations in this crate, but instead provides raw accelerated //! access to the AES round function gated under the `hazmat` crate feature. -#![allow(unsafe_op_in_unsafe_fn)] - +use super::utils::{load_block, store_block}; use crate::hazmat::{Block, Block8}; use core::arch::aarch64::*; /// AES cipher (encrypt) round function. -#[allow(clippy::cast_ptr_alignment)] #[target_feature(enable = "aes")] -pub(crate) unsafe fn cipher_round(block: &mut Block, round_key: &Block) { - let b = vld1q_u8(block.as_ptr()); - let k = vld1q_u8(round_key.as_ptr()); +pub(crate) fn cipher_round(block: &mut Block, round_key: &Block) { + let mut b = load_block(block); + let k = load_block(round_key); // AES single round encryption (all-zero round key, deferred until the end) - let mut state = vaeseq_u8(b, vdupq_n_u8(0)); + b = vaeseq_u8(b, vdupq_n_u8(0)); // AES mix columns (the `vaeseq_u8` instruction otherwise omits this step) - state = vaesmcq_u8(state); + b = vaesmcq_u8(b); // AES add round key (bitwise XOR) - state = veorq_u8(state, k); + b = veorq_u8(b, k); - vst1q_u8(block.as_mut_ptr(), state); + store_block(block, b); } /// AES cipher (encrypt) round function: parallel version. -#[allow(clippy::cast_ptr_alignment)] #[target_feature(enable = "aes")] -pub(crate) unsafe fn cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { +pub(crate) fn cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { for i in 0..8 { - let mut state = vld1q_u8(blocks[i].as_ptr()); + let mut b = load_block(&blocks[i]); // AES single round encryption - state = vaeseq_u8(state, vdupq_n_u8(0)); + b = vaeseq_u8(b, vdupq_n_u8(0)); // AES mix columns - state = vaesmcq_u8(state); + b = vaesmcq_u8(b); // AES add round key (bitwise XOR) - state = veorq_u8(state, vld1q_u8(round_keys[i].as_ptr())); + let rk = load_block(&round_keys[i]); + b = veorq_u8(b, rk); - vst1q_u8(blocks[i].as_mut_ptr(), state); + store_block(&mut blocks[i], b); } } /// AES equivalent inverse cipher (decrypt) round function. -#[allow(clippy::cast_ptr_alignment)] #[target_feature(enable = "aes")] -pub(crate) unsafe fn equiv_inv_cipher_round(block: &mut Block, round_key: &Block) { - let b = vld1q_u8(block.as_ptr()); - let k = vld1q_u8(round_key.as_ptr()); +pub(crate) fn equiv_inv_cipher_round(block: &mut Block, round_key: &Block) { + let mut b = load_block(block); + let k = load_block(round_key); // AES single round decryption (all-zero round key, deferred until the end) - let mut state = vaesdq_u8(b, vdupq_n_u8(0)); + b = vaesdq_u8(b, vdupq_n_u8(0)); // AES inverse mix columns (the `vaesdq_u8` instruction otherwise omits this step) - state = vaesimcq_u8(state); + b = vaesimcq_u8(b); // AES add round key (bitwise XOR) - state = veorq_u8(state, k); + b = veorq_u8(b, k); - vst1q_u8(block.as_mut_ptr(), state); + store_block(block, b); } /// AES equivalent inverse cipher (decrypt) round function: parallel version. -#[allow(clippy::cast_ptr_alignment)] #[target_feature(enable = "aes")] -pub(crate) unsafe fn equiv_inv_cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { +pub(crate) fn equiv_inv_cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { for i in 0..8 { - let mut state = vld1q_u8(blocks[i].as_ptr()); + let mut b = load_block(&blocks[i]); // AES single round decryption (all-zero round key, deferred until the end) - state = vaesdq_u8(state, vdupq_n_u8(0)); + b = vaesdq_u8(b, vdupq_n_u8(0)); // AES inverse mix columns (the `vaesdq_u8` instruction otherwise omits this step) - state = vaesimcq_u8(state); + b = vaesimcq_u8(b); // AES add round key (bitwise XOR) - state = veorq_u8(state, vld1q_u8(round_keys[i].as_ptr())); + let rk = load_block(&round_keys[i]); + b = veorq_u8(b, rk); - vst1q_u8(blocks[i].as_mut_ptr(), state); + store_block(&mut blocks[i], b); } } /// AES mix columns function. -#[allow(clippy::cast_ptr_alignment)] #[target_feature(enable = "aes")] -pub(crate) unsafe fn mix_columns(block: &mut Block) { - let b = vld1q_u8(block.as_ptr()); +pub(crate) fn mix_columns(block: &mut Block) { + let b = load_block(block); let out = vaesmcq_u8(b); - vst1q_u8(block.as_mut_ptr(), out); + store_block(block, out); } /// AES inverse mix columns function. -#[allow(clippy::cast_ptr_alignment)] #[target_feature(enable = "aes")] -pub(crate) unsafe fn inv_mix_columns(block: &mut Block) { - let b = vld1q_u8(block.as_ptr()); +pub(crate) fn inv_mix_columns(block: &mut Block) { + let b = load_block(block); let out = vaesimcq_u8(b); - vst1q_u8(block.as_mut_ptr(), out); + store_block(block, out); } diff --git a/aes/src/backends/aarch64_aes/utils.rs b/aes/src/backends/aarch64_aes/utils.rs new file mode 100644 index 00000000..d669cd4f --- /dev/null +++ b/aes/src/backends/aarch64_aes/utils.rs @@ -0,0 +1,14 @@ +use crate::Block; +use core::arch::aarch64::{uint8x16_t, vld1q_u8, vst1q_u8}; + +#[target_feature(enable = "neon")] +pub(super) fn load_block(block: &Block) -> uint8x16_t { + // SAFETY: `vld1q_u8` supports unaligned loads + unsafe { vld1q_u8(block.as_ptr()) } +} + +#[target_feature(enable = "neon")] +pub(super) fn store_block(dst: &mut Block, block: uint8x16_t) { + // SAFETY: `vst1q_u8` supports unaligned stores + unsafe { vst1q_u8(dst.as_mut_ptr(), block) } +} diff --git a/aes/src/backends/x86_aes.rs b/aes/src/backends/x86_aes.rs index ac0290b0..a329db32 100644 --- a/aes/src/backends/x86_aes.rs +++ b/aes/src/backends/x86_aes.rs @@ -7,11 +7,11 @@ use cipher::{ mod encdec; mod expand; +mod utils; #[cfg(feature = "hazmat")] pub(crate) mod hazmat; -#[cfg(any(aes_backend = "avx512", aes_backend = "avx256"))] pub(crate) use encdec::{decrypt, encrypt}; pub(crate) use expand::RoundKeys; @@ -38,15 +38,13 @@ pub(crate) struct Aes { impl Aes { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn encrypt(&self, f: impl BlockCipherEncClosure) { + pub(crate) fn encrypt(&self, f: impl BlockCipherEncClosure) { f.call(self); } #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn decrypt(&self, f: impl BlockCipherDecClosure) { + pub(crate) fn decrypt(&self, f: impl BlockCipherDecClosure) { f.call(self); } } @@ -71,7 +69,7 @@ impl BlockCipherEncBackend for Aes { fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with // `#[target_feature(enable = "aes")]` - unsafe { encdec::encrypt_par(&self.enc_rk, blocks) }; + unsafe { encdec::batch_encrypt(&self.enc_rk, blocks) }; } } @@ -99,27 +97,22 @@ pub(crate) struct AesEnc { impl AesEnc { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn as_encdec(&self) -> Aes { + pub(crate) fn as_encdec(&self) -> Aes { let enc_rk = self.enc_rk; - // SAFETY: the is method marked with `#[target_feature(enable = "aes")]` - let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + let dec_rk = expand::inv_expanded_keys(&enc_rk); Aes { enc_rk, dec_rk } } #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn as_dec(&self) -> AesDec { - // SAFETY: the is method marked with `#[target_feature(enable = "aes")]` - let dec_rk = unsafe { expand::inv_expanded_keys(&self.enc_rk) }; + pub(crate) fn as_dec(&self) -> AesDec { + let dec_rk = expand::inv_expanded_keys(&self.enc_rk); AesDec { dec_rk } } #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn encrypt(&self, f: impl BlockCipherEncClosure) { + pub(crate) fn encrypt(&self, f: impl BlockCipherEncClosure) { f.call(self) } } @@ -144,7 +137,7 @@ impl BlockCipherEncBackend for AesEnc { fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with // `#[target_feature(enable = "aes")]` - unsafe { encdec::encrypt_par(&self.enc_rk, blocks) }; + unsafe { encdec::batch_encrypt(&self.enc_rk, blocks) }; } } @@ -156,8 +149,7 @@ pub(crate) struct AesDec { impl AesDec { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn decrypt(&self, f: impl BlockCipherDecClosure) { + pub(crate) fn decrypt(&self, f: impl BlockCipherDecClosure) { f.call(self); } } @@ -191,10 +183,9 @@ macro_rules! impl_key_init { impl $name { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { - let enc_rk = unsafe { expand::$expand_fn(key) }; - let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + pub(crate) fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = expand::$expand_fn(key); + let dec_rk = expand::inv_expanded_keys(&enc_rk); Self { enc_rk, dec_rk } } } @@ -202,10 +193,8 @@ macro_rules! impl_key_init { impl $name_enc { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { - // SAFETY: the is method marked with `#[target_feature(enable = "aes")]` - let enc_rk = unsafe { expand::$expand_fn(key) }; + pub(crate) fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = expand::$expand_fn(key); Self { enc_rk } } } @@ -213,10 +202,9 @@ macro_rules! impl_key_init { impl $name_dec { #[inline] #[target_feature(enable = "aes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn new(key: &[u8; $key_size]) -> Self { - let enc_rk = unsafe { expand::$expand_fn(key) }; - let dec_rk = unsafe { expand::inv_expanded_keys(&enc_rk) }; + pub(crate) fn new(key: &[u8; $key_size]) -> Self { + let enc_rk = expand::$expand_fn(key); + let dec_rk = expand::inv_expanded_keys(&enc_rk); Self { dec_rk } } } diff --git a/aes/src/backends/x86_aes/encdec.rs b/aes/src/backends/x86_aes/encdec.rs index fa4a245e..458b8012 100644 --- a/aes/src/backends/x86_aes/encdec.rs +++ b/aes/src/backends/x86_aes/encdec.rs @@ -1,6 +1,4 @@ -#![allow(unsafe_op_in_unsafe_fn)] - -use super::RoundKeys; +use super::{RoundKeys, utils}; use crate::Block; use cipher::{ array::{Array, ArraySize}, @@ -14,144 +12,128 @@ use core::arch::x86_64::*; #[inline] #[target_feature(enable = "aes")] -pub(crate) unsafe fn encrypt(keys: &RoundKeys, block: InOut<'_, '_, Block>) { +pub(crate) fn encrypt(keys: &RoundKeys, mut block: InOut<'_, '_, Block>) { const { assert!(matches!(RK, 11 | 13 | 15)) } - let (block_in, block_out) = block.into_raw(); - let mut b = _mm_loadu_si128(block_in.cast()); + let mut b = utils::load_block(block.get_in()); b = _mm_xor_si128(b, keys[0]); for &key in &keys[1..RK - 1] { b = _mm_aesenc_si128(b, key); } b = _mm_aesenclast_si128(b, keys[RK - 1]); - _mm_storeu_si128(block_out.cast(), b); + utils::store_block(block.get_out(), b); } #[inline] #[target_feature(enable = "aes")] -pub(crate) unsafe fn decrypt(keys: &RoundKeys, block: InOut<'_, '_, Block>) { +pub(crate) fn decrypt(keys: &RoundKeys, mut block: InOut<'_, '_, Block>) { const { assert!(matches!(RK, 11 | 13 | 15)) } - let (block_in, block_out) = block.into_raw(); - let mut b = _mm_loadu_si128(block_in.cast()); + let mut b = utils::load_block(block.get_in()); b = _mm_xor_si128(b, keys[0]); for &key in &keys[1..RK - 1] { b = _mm_aesdec_si128(b, key); } b = _mm_aesdeclast_si128(b, keys[RK - 1]); - _mm_storeu_si128(block_out.cast(), b); + utils::store_block(block.get_out(), b); } #[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn encrypt_par( +pub(super) fn batch_encrypt( keys: &RoundKeys, mut blocks: InOut<'_, '_, Array>, ) { const { assert!(matches!(RK, 11 | 13 | 15)) } - let mut b = load(blocks.get_in()); + let mut b = utils::load_batch_blocks(blocks.get_in()); // Loop over keys is intentionally not used here to force inlining - xor(&mut b, keys[0]); - aesenc(&mut b, keys[1]); - aesenc(&mut b, keys[2]); - aesenc(&mut b, keys[3]); - aesenc(&mut b, keys[4]); - aesenc(&mut b, keys[5]); - aesenc(&mut b, keys[6]); - aesenc(&mut b, keys[7]); - aesenc(&mut b, keys[8]); - aesenc(&mut b, keys[9]); + batch_xor(&mut b, keys[0]); + batch_aesenc(&mut b, keys[1]); + batch_aesenc(&mut b, keys[2]); + batch_aesenc(&mut b, keys[3]); + batch_aesenc(&mut b, keys[4]); + batch_aesenc(&mut b, keys[5]); + batch_aesenc(&mut b, keys[6]); + batch_aesenc(&mut b, keys[7]); + batch_aesenc(&mut b, keys[8]); + batch_aesenc(&mut b, keys[9]); if RK >= 13 { - aesenc(&mut b, keys[10]); - aesenc(&mut b, keys[11]); + batch_aesenc(&mut b, keys[10]); + batch_aesenc(&mut b, keys[11]); } if RK == 15 { - aesenc(&mut b, keys[12]); - aesenc(&mut b, keys[13]); + batch_aesenc(&mut b, keys[12]); + batch_aesenc(&mut b, keys[13]); } - aesenclast(&mut b, keys[RK - 1]); - store(blocks.get_out(), b); + batch_aesenclast(&mut b, keys[RK - 1]); + utils::store_batch_blocks(blocks.get_out(), b); } #[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn decrypt_par( +pub(super) fn decrypt_par( keys: &RoundKeys, mut blocks: InOut<'_, '_, Array>, ) { const { assert!(matches!(RK, 11 | 13 | 15)) }; - let mut b = load(blocks.get_in()); + let mut b = utils::load_batch_blocks(blocks.get_in()); // Loop over keys is intentionally not used here to force inlining - xor(&mut b, keys[0]); - aesdec(&mut b, keys[1]); - aesdec(&mut b, keys[2]); - aesdec(&mut b, keys[3]); - aesdec(&mut b, keys[4]); - aesdec(&mut b, keys[5]); - aesdec(&mut b, keys[6]); - aesdec(&mut b, keys[7]); - aesdec(&mut b, keys[8]); - aesdec(&mut b, keys[9]); + batch_xor(&mut b, keys[0]); + batch_aesdec(&mut b, keys[1]); + batch_aesdec(&mut b, keys[2]); + batch_aesdec(&mut b, keys[3]); + batch_aesdec(&mut b, keys[4]); + batch_aesdec(&mut b, keys[5]); + batch_aesdec(&mut b, keys[6]); + batch_aesdec(&mut b, keys[7]); + batch_aesdec(&mut b, keys[8]); + batch_aesdec(&mut b, keys[9]); if RK >= 13 { - aesdec(&mut b, keys[10]); - aesdec(&mut b, keys[11]); + batch_aesdec(&mut b, keys[10]); + batch_aesdec(&mut b, keys[11]); } if RK == 15 { - aesdec(&mut b, keys[12]); - aesdec(&mut b, keys[13]); - } - aesdeclast(&mut b, keys[RK - 1]); - store(blocks.get_out(), b); -} - -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn load(blocks: &Array) -> Array<__m128i, N> { - let p: *const __m128i = blocks.as_ptr().cast(); - Array::from_fn(|i| unsafe { _mm_loadu_si128(p.add(i)) }) -} - -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn store(dst: &mut Array, blocks: Array<__m128i, N>) { - let p: *mut __m128i = dst.as_mut_ptr().cast(); - for (i, block) in blocks.into_iter().enumerate() { - unsafe { _mm_storeu_si128(p.add(i), block) } + batch_aesdec(&mut b, keys[12]); + batch_aesdec(&mut b, keys[13]); } + batch_aesdeclast(&mut b, keys[RK - 1]); + utils::store_batch_blocks(blocks.get_out(), b); } #[target_feature(enable = "sse2")] -pub(crate) unsafe fn xor(blocks: &mut Array<__m128i, N>, key: __m128i) { +fn batch_xor(blocks: &mut Array<__m128i, N>, key: __m128i) { for block in blocks { *block = _mm_xor_si128(*block, key); } } #[target_feature(enable = "aes")] -pub(crate) unsafe fn aesenc(blocks: &mut Array<__m128i, N>, key: __m128i) { +fn batch_aesenc(blocks: &mut Array<__m128i, N>, key: __m128i) { for block in blocks { *block = _mm_aesenc_si128(*block, key); } } #[target_feature(enable = "aes")] -pub(crate) unsafe fn aesenclast(blocks: &mut Array<__m128i, N>, key: __m128i) { +fn batch_aesenclast(blocks: &mut Array<__m128i, N>, key: __m128i) { for block in blocks { *block = _mm_aesenclast_si128(*block, key); } } #[target_feature(enable = "aes")] -pub(crate) unsafe fn aesdec(blocks: &mut Array<__m128i, N>, key: __m128i) { +fn batch_aesdec(blocks: &mut Array<__m128i, N>, key: __m128i) { for block in blocks { *block = _mm_aesdec_si128(*block, key); } } #[target_feature(enable = "aes")] -pub(crate) unsafe fn aesdeclast(blocks: &mut Array<__m128i, N>, key: __m128i) { +fn batch_aesdeclast(blocks: &mut Array<__m128i, N>, key: __m128i) { for block in blocks { *block = _mm_aesdeclast_si128(*block, key); } diff --git a/aes/src/backends/x86_aes/expand.rs b/aes/src/backends/x86_aes/expand.rs index 3f07f43f..9eeac1eb 100644 --- a/aes/src/backends/x86_aes/expand.rs +++ b/aes/src/backends/x86_aes/expand.rs @@ -1,5 +1,3 @@ -#![allow(unsafe_op_in_unsafe_fn)] - #[cfg(target_arch = "x86")] use core::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -9,8 +7,9 @@ pub(crate) type RoundKeys = [__m128i; RK]; #[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn aes128_expand_key(key: &[u8; 16]) -> RoundKeys<11> { - unsafe fn expand_round(keys: &mut RoundKeys<11>, pos: usize) { +pub(super) fn aes128_expand_key(key: &[u8; 16]) -> RoundKeys<11> { + #[target_feature(enable = "aes")] + fn expand_round(keys: &mut RoundKeys<11>, pos: usize) { let mut t1 = keys[pos - 1]; let mut t2; let mut t3; @@ -29,8 +28,7 @@ pub(super) unsafe fn aes128_expand_key(key: &[u8; 16]) -> RoundKeys<11> { } let mut keys = [_mm_setzero_si128(); 11]; - let k = _mm_loadu_si128(key.as_ptr().cast()); - keys[0] = k; + keys[0] = load(key); let kr = &mut keys; expand_round::<0x01>(kr, 1); @@ -49,14 +47,15 @@ pub(super) unsafe fn aes128_expand_key(key: &[u8; 16]) -> RoundKeys<11> { #[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> RoundKeys<13> { - unsafe fn unpack_hilo(a: __m128i, b: __m128i) -> __m128i { +pub(super) fn aes192_expand_key(key: &[u8; 24]) -> RoundKeys<13> { + #[target_feature(enable = "aes")] + fn unpack_hilo(a: __m128i, b: __m128i) -> __m128i { let a = _mm_shuffle_epi32(a, 0b01_00_11_10); _mm_unpacklo_epi64(a, b) } #[target_feature(enable = "aes")] - unsafe fn expand_round(mut t1: __m128i, mut t3: __m128i) -> (__m128i, __m128i) { + fn expand_round(mut t1: __m128i, mut t3: __m128i) -> (__m128i, __m128i) { let (mut t2, mut t4); t2 = _mm_aeskeygenassist_si128(t3, R); @@ -77,17 +76,9 @@ pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> RoundKeys<13> { } let mut keys = [_mm_setzero_si128(); 13]; - // We are being extra pedantic here to remove out-of-bound access. - // This should be optimized into movups, movsd sequence. - let (k0, k1l) = { - let mut t = [0u8; 32]; - t[..key.len()].copy_from_slice(key); - ( - _mm_loadu_si128(t.as_ptr().cast()), - _mm_loadu_si128(t.as_ptr().offset(16).cast()), - ) - }; + let k0 = load(&key[..16]); + let k1l = load(&key[16..]); keys[0] = k0; let (k1_2, k2r) = expand_round::<0x01>(k0, k1l); @@ -125,8 +116,9 @@ pub(super) unsafe fn aes192_expand_key(key: &[u8; 24]) -> RoundKeys<13> { #[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> RoundKeys<15> { - unsafe fn expand_round(keys: &mut RoundKeys<15>, pos: usize) { +pub(super) fn aes256_expand_key(key: &[u8; 32]) -> RoundKeys<15> { + #[target_feature(enable = "aes")] + fn expand_round(keys: &mut RoundKeys<15>, pos: usize) { let mut t1 = keys[pos - 2]; let mut t2; let mut t3 = keys[pos - 1]; @@ -157,7 +149,8 @@ pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> RoundKeys<15> { keys[pos + 1] = t3; } - unsafe fn expand_round_last(keys: &mut RoundKeys<15>, pos: usize) { + #[target_feature(enable = "aes")] + fn expand_round_last(keys: &mut RoundKeys<15>, pos: usize) { let mut t1 = keys[pos - 2]; let mut t2; let t3 = keys[pos - 1]; @@ -178,9 +171,8 @@ pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> RoundKeys<15> { let mut keys = [_mm_setzero_si128(); 15]; - let kp = key.as_ptr().cast::<__m128i>(); - keys[0] = _mm_loadu_si128(kp); - keys[1] = _mm_loadu_si128(kp.add(1)); + keys[0] = load(&key[..16]); + keys[1] = load(&key[16..]); let k = &mut keys; expand_round::<0x01>(k, 2); @@ -196,7 +188,7 @@ pub(super) unsafe fn aes256_expand_key(key: &[u8; 32]) -> RoundKeys<15> { #[inline] #[target_feature(enable = "aes")] -pub(super) unsafe fn inv_expanded_keys(keys: &[__m128i; N]) -> [__m128i; N] { +pub(super) fn inv_expanded_keys(keys: &[__m128i; N]) -> [__m128i; N] { let mut inv_keys: [__m128i; N] = [_mm_setzero_si128(); N]; inv_keys[0] = keys[N - 1]; for i in 1..N - 1 { @@ -205,3 +197,11 @@ pub(super) unsafe fn inv_expanded_keys(keys: &[__m128i; N]) -> [ inv_keys[N - 1] = keys[0]; inv_keys } + +#[target_feature(enable = "sse2")] +fn load(bytes: &[u8]) -> __m128i { + assert!(size_of_val(bytes) <= size_of::<__m128i>()); + let mut t = crate::Block::default(); + t[..bytes.len()].copy_from_slice(bytes); + super::utils::load_block(&t) +} diff --git a/aes/src/backends/x86_aes/hazmat.rs b/aes/src/backends/x86_aes/hazmat.rs index b2f8041b..07c5485e 100644 --- a/aes/src/backends/x86_aes/hazmat.rs +++ b/aes/src/backends/x86_aes/hazmat.rs @@ -3,99 +3,77 @@ //! Note: this isn't actually used in the `Aes128`/`Aes192`/`Aes256` //! implementations in this crate, but instead provides raw AES-NI accelerated //! access to the AES round function gated under the `hazmat` crate feature. -#![allow(unsafe_op_in_unsafe_fn)] - +use super::utils; use crate::hazmat::{Block, Block8}; -use cipher::array::{Array, ArraySize}; #[cfg(target_arch = "x86")] use core::arch::x86::*; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn load(blocks: *const Array) -> Array<__m128i, N> { - let p = blocks.cast::<__m128i>(); - let mut res: Array<__m128i, N> = core::mem::zeroed(); - for i in 0..N::USIZE { - res[i] = _mm_loadu_si128(p.add(i)); - } - res -} - -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn store(blocks: *mut Array, b: Array<__m128i, N>) { - let p = blocks.cast::<__m128i>(); - for i in 0..N::USIZE { - _mm_storeu_si128(p.add(i), b[i]); - } -} - /// AES cipher (encrypt) round function. #[target_feature(enable = "aes")] -pub(crate) unsafe fn cipher_round(block: &mut Block, round_key: &Block) { +pub(crate) fn cipher_round(block: &mut Block, round_key: &Block) { // Safety: `loadu` and `storeu` support unaligned access - let b = _mm_loadu_si128(block.as_ptr() as *const __m128i); - let k = _mm_loadu_si128(round_key.as_ptr() as *const __m128i); - let out = _mm_aesenc_si128(b, k); - _mm_storeu_si128(block.as_mut_ptr() as *mut __m128i, out); + let b = utils::load_block(block); + let k = utils::load_block(round_key); + let b = _mm_aesenc_si128(b, k); + utils::store_block(block, b); } /// AES cipher (encrypt) round function: parallel version. #[target_feature(enable = "aes")] -pub(crate) unsafe fn cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { - let xmm_keys = load(round_keys); - let mut xmm_blocks = load(blocks); +pub(crate) fn cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { + let rks = utils::load_batch_blocks(round_keys); + let mut bb = utils::load_batch_blocks(blocks); - for i in 0..8 { - xmm_blocks[i] = _mm_aesenc_si128(xmm_blocks[i], xmm_keys[i]); + for i in 0..bb.len() { + bb[i] = _mm_aesenc_si128(bb[i], rks[i]); } - store(blocks, xmm_blocks); + utils::store_batch_blocks(blocks, bb); } /// AES cipher (encrypt) round function. #[target_feature(enable = "aes")] -pub(crate) unsafe fn equiv_inv_cipher_round(block: &mut Block, round_key: &Block) { - // Safety: `loadu` and `storeu` support unaligned access - let b = _mm_loadu_si128(block.as_ptr() as *const __m128i); - let k = _mm_loadu_si128(round_key.as_ptr() as *const __m128i); - let out = _mm_aesdec_si128(b, k); - _mm_storeu_si128(block.as_mut_ptr() as *mut __m128i, out); +pub(crate) fn equiv_inv_cipher_round(block: &mut Block, round_key: &Block) { + let b = utils::load_block(block); + let rk = utils::load_block(round_key); + let b = _mm_aesdec_si128(b, rk); + utils::store_block(block, b); } /// AES cipher (encrypt) round function: parallel version. #[target_feature(enable = "aes")] -pub(crate) unsafe fn equiv_inv_cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { - let xmm_keys = load(round_keys); - let mut xmm_blocks = load(blocks); +pub(crate) fn equiv_inv_cipher_round_par(blocks: &mut Block8, round_keys: &Block8) { + let rks = utils::load_batch_blocks(round_keys); + let mut bb = utils::load_batch_blocks(blocks); - for i in 0..8 { - xmm_blocks[i] = _mm_aesdec_si128(xmm_blocks[i], xmm_keys[i]); + for i in 0..bb.len() { + bb[i] = _mm_aesdec_si128(bb[i], rks[i]); } - store(blocks, xmm_blocks); + utils::store_batch_blocks(blocks, bb); } /// AES mix columns function. #[target_feature(enable = "aes")] -pub(crate) unsafe fn mix_columns(block: &mut Block) { +pub(crate) fn mix_columns(block: &mut Block) { // Safety: `loadu` and `storeu` support unaligned access - let mut state = _mm_loadu_si128(block.as_ptr() as *const __m128i); + let mut state = utils::load_block(block); // Emulate mix columns by performing three inverse mix columns operations state = _mm_aesimc_si128(state); state = _mm_aesimc_si128(state); state = _mm_aesimc_si128(state); - _mm_storeu_si128(block.as_mut_ptr() as *mut __m128i, state); + utils::store_block(block, state); } /// AES inverse mix columns function. #[target_feature(enable = "aes")] -pub(crate) unsafe fn inv_mix_columns(block: &mut Block) { - // Safety: `loadu` and `storeu` support unaligned access - let b = _mm_loadu_si128(block.as_ptr() as *const __m128i); - let out = _mm_aesimc_si128(b); - _mm_storeu_si128(block.as_mut_ptr() as *mut __m128i, out); +pub(crate) fn inv_mix_columns(block: &mut Block) { + let b = utils::load_block(block); + let b = _mm_aesimc_si128(b); + utils::store_block(block, b); } diff --git a/aes/src/backends/x86_aes/utils.rs b/aes/src/backends/x86_aes/utils.rs new file mode 100644 index 00000000..04b845d2 --- /dev/null +++ b/aes/src/backends/x86_aes/utils.rs @@ -0,0 +1,36 @@ +use crate::Block; +use cipher::{Array, array::ArraySize}; + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +#[target_feature(enable = "sse2")] +pub(super) fn load_block(block: &Block) -> __m128i { + let p: *const __m128i = block.as_ptr().cast(); + // SAFETY: sizes of `Block` and `__m128i` are equal and we use unaligned load instruction + unsafe { _mm_loadu_si128(p) } +} + +#[target_feature(enable = "sse2")] +pub(super) fn store_block(dst: &mut Block, block: __m128i) { + let p: *mut __m128i = dst.as_mut_ptr().cast(); + // SAFETY: sizes of `Block` and `__m128i` are equal and we use unaligned store instruction + unsafe { _mm_storeu_si128(p, block) } +} + +#[target_feature(enable = "sse2")] +pub(super) fn load_batch_blocks(blocks: &Array) -> Array<__m128i, N> { + Array::from_fn(|i| load_block(&blocks[i])) +} + +#[target_feature(enable = "sse2")] +pub(super) fn store_batch_blocks( + dst: &mut Array, + blocks: Array<__m128i, N>, +) { + for i in 0..N::USIZE { + store_block(&mut dst[i], blocks[i]); + } +} diff --git a/aes/src/backends/x86_vaes256.rs b/aes/src/backends/x86_vaes256.rs index ae2dbcc9..5c40a3df 100644 --- a/aes/src/backends/x86_vaes256.rs +++ b/aes/src/backends/x86_vaes256.rs @@ -21,24 +21,16 @@ pub(crate) struct Aes<'a, const RK: usize> { impl<'a, const RK: usize> Aes<'a, RK> { #[inline] #[target_feature(enable = "vaes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn encrypt( - rk: &'a RoundKeys, - f: impl BlockCipherEncClosure, - ) { - let rk2 = unsafe { encdec::broadcast_keys(rk) }; + pub(crate) fn encrypt(rk: &'a RoundKeys, f: impl BlockCipherEncClosure) { + let rk2 = encdec::broadcast_keys(rk); let backend = Self { rk, rk2 }; f.call(&backend) } #[inline] #[target_feature(enable = "vaes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn decrypt( - rk: &'a RoundKeys, - f: impl BlockCipherDecClosure, - ) { - let rk2 = unsafe { encdec::broadcast_keys(rk) }; + pub(crate) fn decrypt(rk: &'a RoundKeys, f: impl BlockCipherDecClosure) { + let rk2 = encdec::broadcast_keys(rk); let backend = Self { rk, rk2 }; f.call(&backend) } @@ -61,14 +53,14 @@ impl BlockCipherEncBackend for Aes<'_, RK> { fn encrypt_block(&self, block: InOut<'_, '_, Block>) { // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with // `#[target_feature(enable = "vaes")]` - unsafe { super::x86_aes::encrypt(&self.rk, block) }; + unsafe { super::x86_aes::encrypt(self.rk, block) }; } #[inline(always)] fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with // `#[target_feature(enable = "vaes")]` - unsafe { encdec::encrypt_par(&self.rk2, blocks) }; + unsafe { encdec::batch_encrypt(&self.rk2, blocks) }; } } @@ -77,13 +69,13 @@ impl BlockCipherDecBackend for Aes<'_, RK> { fn decrypt_block(&self, block: InOut<'_, '_, Block>) { // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with // `#[target_feature(enable = "vaes")]` - unsafe { super::x86_aes::decrypt(&self.rk, block) }; + unsafe { super::x86_aes::decrypt(self.rk, block) }; } #[inline(always)] fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with // `#[target_feature(enable = "vaes")]` - unsafe { encdec::decrypt_par(&self.rk2, blocks) }; + unsafe { encdec::batch_decrypt(&self.rk2, blocks) }; } } diff --git a/aes/src/backends/x86_vaes256/encdec.rs b/aes/src/backends/x86_vaes256/encdec.rs index 93280a52..3fa9b79e 100644 --- a/aes/src/backends/x86_vaes256/encdec.rs +++ b/aes/src/backends/x86_vaes256/encdec.rs @@ -10,17 +10,17 @@ use core::arch::x86_64::*; pub(super) type RoundKeys2 = [__m256i; ROUNDS]; -type SimdBlocks = Array<__m256i, Quot>; +type BatchBlocks = Array<__m256i, Quot>; #[inline] #[target_feature(enable = "avx2")] -pub(crate) unsafe fn broadcast_keys(keys: &RoundKeys) -> RoundKeys2 { +pub(super) fn broadcast_keys(keys: &RoundKeys) -> RoundKeys2 { keys.map(|key| _mm256_broadcastsi128_si256(key)) } #[inline] #[target_feature(enable = "vaes")] -pub(crate) unsafe fn encrypt_par( +pub(super) fn batch_encrypt( keys: &RoundKeys2, mut blocks: InOut<'_, '_, Array>, ) where @@ -32,7 +32,7 @@ pub(crate) unsafe fn encrypt_par( assert!(ParBlocks::USIZE % 2 == 0); } - let mut blocks2 = load(blocks.get_in()); + let mut blocks2 = batch_load(blocks.get_in()); for block2 in &mut blocks2 { *block2 = _mm256_xor_si256(*block2, keys[0]); @@ -46,12 +46,12 @@ pub(crate) unsafe fn encrypt_par( *block2 = _mm256_aesenclast_epi128(*block2, keys[RK - 1]); } - store(blocks.get_out(), blocks2); + batch_store(blocks.get_out(), blocks2); } #[inline] #[target_feature(enable = "vaes")] -pub(crate) unsafe fn decrypt_par( +pub(super) fn batch_decrypt( keys: &RoundKeys2, mut blocks: InOut<'_, '_, Array>, ) where @@ -63,7 +63,7 @@ pub(crate) unsafe fn decrypt_par( assert!(ParBlocks::USIZE % 2 == 0); } - let mut blocks2 = load(blocks.get_in()); + let mut blocks2 = batch_load(blocks.get_in()); for block2 in &mut blocks2 { *block2 = _mm256_xor_si256(*block2, keys[0]); @@ -77,12 +77,12 @@ pub(crate) unsafe fn decrypt_par( *block2 = _mm256_aesdeclast_epi128(*block2, keys[RK - 1]); } - store(blocks.get_out(), blocks2); + batch_store(blocks.get_out(), blocks2); } #[inline] #[target_feature(enable = "avx")] -fn load(blocks: &Array) -> SimdBlocks +fn batch_load(blocks: &Array) -> BatchBlocks where ParBlocks: ArraySize + Div, Quot: ArraySize, @@ -90,20 +90,22 @@ where const { assert!(ParBlocks::USIZE % 2 == 0) } let in_ptr: *const __m256i = blocks.as_ptr().cast(); + // SAFETY: we use unaligned load instruction Array::from_fn(|i| unsafe { _mm256_loadu_si256(in_ptr.add(i)) }) } #[inline] #[target_feature(enable = "avx")] -fn store(dst: &mut Array, blocks: SimdBlocks) +fn batch_store(dst: &mut Array, blocks: BatchBlocks) where ParBlocks: ArraySize + Div, Quot: ArraySize, { const { assert!(ParBlocks::USIZE % 2 == 0) } - let out_ptr: *mut __m256i = dst.as_mut_ptr().cast(); + let dst_ptr: *mut __m256i = dst.as_mut_ptr().cast(); for (i, block) in blocks.into_iter().enumerate() { - unsafe { _mm256_storeu_si256(out_ptr.add(i), block) } + // SAFETY: we use unaligned store instruction + unsafe { _mm256_storeu_si256(dst_ptr.add(i), block) } } } diff --git a/aes/src/backends/x86_vaes512.rs b/aes/src/backends/x86_vaes512.rs index f443e03d..bf0974e9 100644 --- a/aes/src/backends/x86_vaes512.rs +++ b/aes/src/backends/x86_vaes512.rs @@ -15,31 +15,23 @@ pub(crate) type Aes256<'a> = Aes<'a, 15>; #[derive(Clone, Copy)] pub(crate) struct Aes<'a, const RK: usize> { rk: &'a RoundKeys, - rk2: encdec::RoundKeys4, + rk4: encdec::RoundKeys4, } impl<'a, const RK: usize> Aes<'a, RK> { #[inline] #[target_feature(enable = "avx512f,vaes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn encrypt( - rk: &'a RoundKeys, - f: impl BlockCipherEncClosure, - ) { - let rk2 = unsafe { encdec::broadcast_keys(rk) }; - let backend = Self { rk, rk2 }; + pub(crate) fn encrypt(rk: &'a RoundKeys, f: impl BlockCipherEncClosure) { + let rk4 = encdec::broadcast_keys(rk); + let backend = Self { rk, rk4 }; f.call(&backend) } #[inline] #[target_feature(enable = "avx512f,vaes")] - // TODO(MSRV-1.86): remove `unsafe` - pub(crate) unsafe fn decrypt( - rk: &'a RoundKeys, - f: impl BlockCipherDecClosure, - ) { - let rk2 = unsafe { encdec::broadcast_keys(rk) }; - let backend = Self { rk, rk2 }; + pub(crate) fn decrypt(rk: &'a RoundKeys, f: impl BlockCipherDecClosure) { + let rk4 = encdec::broadcast_keys(rk); + let backend = Self { rk, rk4 }; f.call(&backend) } } @@ -63,14 +55,14 @@ impl BlockCipherEncBackend for Aes<'_, RK> { fn encrypt_block(&self, block: InOut<'_, '_, Block>) { // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with // `#[target_feature(enable = "vaavx512f,vaeses")]` - unsafe { super::x86_aes::encrypt(&self.rk, block) }; + unsafe { super::x86_aes::encrypt(self.rk, block) }; } #[inline(always)] fn encrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::encrypt` method marked with // `#[target_feature(enable = "avx512f,vaes")]` - unsafe { encdec::encrypt_par(&self.rk2, blocks) }; + unsafe { encdec::batch_encrypt(&self.rk4, blocks) }; } } @@ -79,13 +71,13 @@ impl BlockCipherDecBackend for Aes<'_, RK> { fn decrypt_block(&self, block: InOut<'_, '_, Block>) { // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with // `#[target_feature(enable = "avx512f,vaes")]` - unsafe { super::x86_aes::decrypt(&self.rk, block) }; + unsafe { super::x86_aes::decrypt(self.rk, block) }; } #[inline(always)] fn decrypt_par_blocks(&self, blocks: InOut<'_, '_, ParBlocks>) { // SAFETY: this trait impl is used only by the `Self::decrypt` method marked with // `#[target_feature(enable = "avx512f,vaes")]` - unsafe { encdec::decrypt_par(&self.rk2, blocks) }; + unsafe { encdec::batch_decrypt(&self.rk4, blocks) }; } } diff --git a/aes/src/backends/x86_vaes512/encdec.rs b/aes/src/backends/x86_vaes512/encdec.rs index 696286bb..b2d6bcdc 100644 --- a/aes/src/backends/x86_vaes512/encdec.rs +++ b/aes/src/backends/x86_vaes512/encdec.rs @@ -10,17 +10,17 @@ use core::arch::x86_64::*; pub(super) type RoundKeys4 = [__m512i; ROUNDS]; -type SimdBlocks = Array<__m512i, Quot>; +type BatchBlocks = Array<__m512i, Quot>; #[inline] #[target_feature(enable = "avx512f")] -pub(crate) unsafe fn broadcast_keys(keys: &RoundKeys) -> RoundKeys4 { +pub(super) fn broadcast_keys(keys: &RoundKeys) -> RoundKeys4 { keys.map(|key| _mm512_broadcast_i32x4(key)) } #[inline] #[target_feature(enable = "avx512f,vaes")] -pub(crate) unsafe fn encrypt_par( +pub(super) fn batch_encrypt( keys: &RoundKeys4, mut blocks: InOut<'_, '_, Array>, ) where @@ -51,7 +51,7 @@ pub(crate) unsafe fn encrypt_par( #[inline] #[target_feature(enable = "avx512f,vaes")] -pub(crate) unsafe fn decrypt_par( +pub(super) fn batch_decrypt( keys: &RoundKeys4, mut blocks: InOut<'_, '_, Array>, ) where @@ -82,28 +82,30 @@ pub(crate) unsafe fn decrypt_par( #[inline] #[target_feature(enable = "avx512f")] -fn load(blocks: &Array) -> SimdBlocks +fn load(blocks: &Array) -> BatchBlocks where ParBlocks: ArraySize + Div, Quot: ArraySize, { const { assert!(ParBlocks::USIZE % 4 == 0) } - let in_ptr: *const __m512i = blocks.as_ptr().cast(); - Array::from_fn(|i| unsafe { _mm512_loadu_si512(in_ptr.add(i)) }) + let src_ptr: *const __m512i = blocks.as_ptr().cast(); + // SAFETY: we use unaligned load instruction + Array::from_fn(|i| unsafe { _mm512_loadu_si512(src_ptr.add(i)) }) } #[inline] #[target_feature(enable = "avx512f")] -fn store(dst: &mut Array, blocks: SimdBlocks) +fn store(dst: &mut Array, blocks: BatchBlocks) where ParBlocks: ArraySize + Div, Quot: ArraySize, { const { assert!(ParBlocks::USIZE % 4 == 0) } - let out_ptr: *mut __m512i = dst.as_mut_ptr().cast(); + let dst_ptr: *mut __m512i = dst.as_mut_ptr().cast(); for (i, block) in blocks.into_iter().enumerate() { - unsafe { _mm512_storeu_si512(out_ptr.add(i), block) } + // SAFETY: we use unaligned store instruction + unsafe { _mm512_storeu_si512(dst_ptr.add(i), block) } } } diff --git a/aes/src/hazmat.rs b/aes/src/hazmat.rs index 6a3eeaa5..46cf658e 100644 --- a/aes/src/hazmat.rs +++ b/aes/src/hazmat.rs @@ -31,6 +31,7 @@ macro_rules! if_intrinsics_available { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use crate::backends::x86_aes::hazmat as intrinsics; + // SAFETY: we checked target feature availability unsafe { $body } return; } diff --git a/aes/src/lib.rs b/aes/src/lib.rs index 12ae4ced..25cd0109 100644 --- a/aes/src/lib.rs +++ b/aes/src/lib.rs @@ -150,17 +150,13 @@ cfg_if! { type Token = (); } else if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { cpufeatures::new!(features_aes, "aes"); - #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] cpufeatures::new!(features_vaes256, "vaes"); - #[cfg(aes_backend = "avx512")] cpufeatures::new!(features_vaes512, "avx512f", "vaes"); #[derive(Clone, Copy)] struct Token { aes: features_aes::InitToken, - #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] vaes256: features_vaes256::InitToken, - #[cfg(aes_backend = "avx512")] vaes512: features_vaes512::InitToken, } @@ -168,9 +164,7 @@ cfg_if! { fn default() -> Self { Token { aes: features_aes::InitToken::init(), - #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] vaes256: features_vaes256::InitToken::init(), - #[cfg(aes_backend = "avx512")] vaes512: features_vaes512::InitToken::init(), } } @@ -269,7 +263,6 @@ macro_rules! impl_encrypt { #[cfg(not(aes_backend = "soft"))] cfg_if! { if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { - #[cfg(aes_backend = "avx512")] if self.token.vaes512.get() { // SAFETY: we access correct union variant let enc_rk = unsafe { &self.inner.aes.enc_rk }; @@ -278,7 +271,6 @@ macro_rules! impl_encrypt { return; } - #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] if self.token.vaes256.get() { // SAFETY: we access correct union variant let enc_rk = unsafe { &self.inner.aes.enc_rk }; @@ -321,7 +313,6 @@ macro_rules! impl_decrypt { #[cfg(not(aes_backend = "soft"))] cfg_if! { if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { - #[cfg(aes_backend = "avx512")] if self.token.vaes512.get() { // SAFETY: we access correct union variant let dec_rk = unsafe { &self.inner.aes.dec_rk }; @@ -330,7 +321,6 @@ macro_rules! impl_decrypt { return; } - #[cfg(any(aes_backend = "avx256", aes_backend = "avx512"))] if self.token.vaes256.get() { // SAFETY: we access correct union variant let dec_rk = unsafe { &self.inner.aes.dec_rk }; From 03b123eda76dac7c20698a77e78238be5c6dd674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D1=91=D0=BC=20=D0=9F=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=5BArtyom=20Pavlov=5D?= Date: Fri, 31 Jul 2026 18:10:22 +0300 Subject: [PATCH 2/3] Add changelog entries --- aes/CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/aes/CHANGELOG.md b/aes/CHANGELOG.md index 8d0770d9..e0406c5d 100644 --- a/aes/CHANGELOG.md +++ b/aes/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.9.3 (UNRELEASED) +### Changed +- MSRV bumped to 1.89 ([#580]) +- VAES-256 and VAES-512 backends on x86 targets are now supported by default ([#580]) + +### Removed +- `aes_backend = "avx256"` and `aes_backend = "avx512"` configuration flags ([#580]) + +[#580]: https://github.com/RustCrypto/block-ciphers/pull/580 + ## 0.9.2 (2026-07-27) ### Added - `hardware_accelerated` function ([#579]) From 2678c3712ff6901003bcbc58c061386e7c530276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D1=91=D0=BC=20=D0=9F=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=5BArtyom=20Pavlov=5D?= Date: Fri, 31 Jul 2026 18:29:07 +0300 Subject: [PATCH 3/3] use `as_chunks` --- aes/src/backends/aarch64_aes/expand.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/aes/src/backends/aarch64_aes/expand.rs b/aes/src/backends/aarch64_aes/expand.rs index 1e6104d7..8b01f8d8 100644 --- a/aes/src/backends/aarch64_aes/expand.rs +++ b/aes/src/backends/aarch64_aes/expand.rs @@ -26,8 +26,11 @@ pub(super) fn expand_key(key: &[u8; N]) -> [uin let keys_ptr: *mut u32 = expanded_keys.as_mut_ptr().cast(); let columns = unsafe { core::slice::from_raw_parts_mut(keys_ptr, RK * BLOCK_WORDS) }; - for (i, chunk) in key.chunks_exact(WORD_SIZE).enumerate() { - columns[i] = u32::from_ne_bytes(chunk.try_into().unwrap()); + let (chunks, tail) = key.as_chunks::(); + assert!(tail.is_empty()); + + for (i, chunk) in chunks.iter().enumerate() { + columns[i] = u32::from_le_bytes(*chunk); } // From "The Rijndael Block Cipher" Section 4.1: