diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 02f4e52f550..7592329f496 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -143,6 +143,7 @@ CONFIG_IDF_TARGET_ESP32S3 CONFIG_IDF_TARGET_ESP8266 CONFIG_IDF_TARGET_ESP8684 CONFIG_KASAN +CONFIG_KERNEL_MODE_NEON CONFIG_KMSAN CONFIG_KPROBES CONFIG_MAIN_TASK_STACK_SIZE diff --git a/configure.ac b/configure.ac index 0bdb4dfa56b..9d797f50b21 100644 --- a/configure.ac +++ b/configure.ac @@ -12213,6 +12213,39 @@ then fi AC_SUBST([ENABLED_LINUXKM_LKCAPI_REGISTER]) +# The aarch64 ChaCha20 and Poly1305 assembly takes no vector register claim, +# so a kernel module must not carry it. Outside FIPS these algorithms are on +# by default, so back them off with a warning when the user did not ask for +# them, and only fail the build when they did. +if test "$ENABLED_LINUXKM" = "yes" && test "$ENABLED_ARMASM" = "yes" +then + case $host_cpu in + *aarch64*|*arm64*) + if test "$ENABLED_CHACHA" != "no" && test "$ENABLED_CHACHA" != "noasm" \ + && test "$ENABLED_ASM" != "no" + then + if test "x$enable_chacha" = "x" + then + AC_MSG_WARN([ChaCha20 assembly holds no vector register claim on aarch64; building ChaCha20 without assembly for the kernel module.]) + ENABLED_CHACHA=noasm + else + AC_MSG_ERROR([ChaCha20 assembly is unusable in a kernel module on aarch64: it holds no vector register claim. Use --enable-chacha=noasm or --disable-chacha.]) + fi + fi + if test "$ENABLED_POLY1305" != "no" && test "$ENABLED_ASM" != "no" + then + if test "x$enable_poly1305" = "x" + then + AC_MSG_WARN([Poly1305 assembly holds no vector register claim on aarch64 and has no assembly-free build; disabling Poly1305 for the kernel module.]) + ENABLED_POLY1305=no + else + AC_MSG_ERROR([Poly1305 assembly is unusable in a kernel module on aarch64: it holds no vector register claim. Use --disable-poly1305.]) + fi + fi + ;; + esac +fi + # Library Suffix LIBSUFFIX="" AC_ARG_WITH([libsuffix], diff --git a/linuxkm/Kbuild b/linuxkm/Kbuild index 3e8deb9628e..bd90f9477bf 100644 --- a/linuxkm/Kbuild +++ b/linuxkm/Kbuild @@ -264,6 +264,17 @@ $(obj)/wolfcrypt/src/wc_mldsa_asm.o: OBJECT_FILES_NON_STANDARD := y $(obj)/wolfcrypt/src/wc_falcon_fpr_x86_64_asm.o: asflags-y := $(WOLFSSL_ASFLAGS) $(ASFLAGS_FPU_DISABLE_SIMD_ENABLE) $(obj)/wolfcrypt/src/wc_falcon_fpr_x86_64_asm.o: OBJECT_FILES_NON_STANDARD := y +# The arm64 kernel builds with a baseline -march that rejects the crypto and +# NEON instructions in port/arm/*.S, and with -mgeneral-regs-only, which clang +# also applies to inline asm. Widen both for these objects. An -march only +# widens what the assembler accepts; it does not change what is emitted. +ifeq ($(CONFIG_ARM64),y) +$(obj)/wolfcrypt/src/port/arm/%.o: asflags-y := $(WOLFSSL_ASFLAGS) -march=armv8.2-a+crypto+sha3 +$(obj)/wolfcrypt/src/port/arm/%-asm_c.o: ccflags-y += -march=armv8.2-a+crypto+sha3 +$(obj)/wolfcrypt/src/port/arm/%-asm_c.o: ccflags-remove-y += -mgeneral-regs-only +$(obj)/wolfcrypt/src/port/arm/%.o: OBJECT_FILES_NON_STANDARD := y +endif + ifndef READELF READELF := readelf endif diff --git a/linuxkm/Makefile b/linuxkm/Makefile index 00fc70df6a0..43c04ba69f8 100644 --- a/linuxkm/Makefile +++ b/linuxkm/Makefile @@ -62,7 +62,10 @@ endif WOLFSSL_ASFLAGS=-DHAVE_CONFIG_H -I$(SRC_TOP) -DBUILDING_WOLFSSL $(AM_CCASFLAGS) $(CCASFLAGS) -WOLFSSL_OBJ_FILES=$(patsubst %.lo, %.o, $(patsubst src/src_libwolfssl_la-%, src/%, $(patsubst src/libwolfssl_la-%, src/%, $(patsubst wolfcrypt/src/src_libwolfssl_la-%, wolfcrypt/src/%, $(src_libwolfssl_la_OBJECTS))))) +# Strip libtool's per-target object prefix so Kbuild sees the real object +# names. The innermost patsubst covers the wolfcrypt/src/port/arm/ assembly, +# which the others do not reach. +WOLFSSL_OBJ_FILES=$(patsubst %.lo, %.o, $(patsubst src/src_libwolfssl_la-%, src/%, $(patsubst src/libwolfssl_la-%, src/%, $(patsubst wolfcrypt/src/src_libwolfssl_la-%, wolfcrypt/src/%, $(patsubst wolfcrypt/src/port/arm/src_libwolfssl_la-%, wolfcrypt/src/port/arm/%, $(src_libwolfssl_la_OBJECTS)))))) ifeq "$(ENABLED_CRYPT_TESTS)" "yes" WOLFSSL_OBJ_FILES+=wolfcrypt/test/test.o diff --git a/linuxkm/arm64_vector_register_glue.c b/linuxkm/arm64_vector_register_glue.c new file mode 100644 index 00000000000..ff86c6df345 --- /dev/null +++ b/linuxkm/arm64_vector_register_glue.c @@ -0,0 +1,244 @@ +/* arm64_vector_register_glue.c: glue logic to claim and release the FPSIMD + * and NEON registers on arm64 + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* included by linuxkm/module_hooks.c */ +#ifndef WC_SKIP_INCLUDED_C_FILES + +#if !defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) || !defined(CONFIG_ARM64) + #error arm64 vector register glue included in non-vectorized or non-arm64 project. +#endif + +#ifndef CONFIG_KERNEL_MODE_NEON + /* Without this option the kernel exports no kernel_neon_begin() and + * may_use_simd() is always false (linux-6.6.99 fpsimd.c:1904, simd.h:46). */ + #error wolfSSL linuxkm on arm64 requires CONFIG_KERNEL_MODE_NEON. +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 19, 0) + /* Written against the void kernel_neon_begin() of linux-6.6.99. 6.19 + * changes that signature and has not been read here. */ + #error arm64 vector register glue does not yet support kernel_neon_begin() with a state buffer (6.19+). +#endif + +#ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING + #error DEBUG_VECTOR_REGISTER_ACCESS_FUZZING is not implemented by the arm64 vector register glue. +#endif + +/* kernel_neon_begin() BUGs unless may_use_simd(), then takes this CPU's FPSIMD + * context with bottom halves off (linux-6.6.99 fpsimd.c:1904 and :239, + * simd.h:26). One record per CPU per context counts claim and inhibit depth. */ + +struct wc_svr_arm64_ctx_state { + unsigned int depth; /* open claims in this context on this CPU */ + unsigned int inhibit_at; /* depth of the open inhibit claim, 0 when none */ + unsigned int neon_held; /* 1 between kernel_neon_begin() and _end() */ + unsigned int bh_held; /* 1 while a pin-only section holds local_bh_disable() */ +}; + +struct wc_svr_arm64_cpu_state { + struct wc_svr_arm64_ctx_state ctx[2]; /* [0] task, [1] softirq */ +}; + +static DEFINE_PER_CPU(struct wc_svr_arm64_cpu_state, wc_svr_arm64_state); +static atomic64_t wc_svr_disallowed_count = ATOMIC64_INIT(0); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) + #define wc_svr_arm64_in_hardirq() in_irq() +#else + #define wc_svr_arm64_in_hardirq() in_hardirq() +#endif + +/* Keep softirqs off a section that holds no registers, as the x86 glue does. + * With interrupts already off, nothing can arrive anyway. */ +static inline void wc_svr_arm64_pin_bh(struct wc_svr_arm64_ctx_state *st) +{ + if (! irqs_disabled()) { + local_bh_disable(); + st->bh_held = 1; + } +} + +/* The record for this context on this CPU. Every caller has the CPU held, + * by preempt_disable() here or by the open claim's bottom-half disable. */ +static inline struct wc_svr_arm64_ctx_state *wc_svr_arm64_here(void) +{ + return &this_cpu_ptr(&wc_svr_arm64_state)->ctx[in_serving_softirq() ? 1 : 0]; +} + +/* The records are static. These entry points exist because wc_port.c and + * the redirect table expect them. */ +__must_check int wc_linuxkm_allocate_svr_states(void) +{ + return 0; +} + +void wc_linuxkm_free_svr_states(void) +{ +} + +void wc_svr_disallowed_count_reset(void) +{ + atomic64_set(&wc_svr_disallowed_count, 0); +} + +__must_check unsigned long long int wc_svr_disallowed_count_current(void) +{ + return (unsigned long long int)atomic64_read(&wc_svr_disallowed_count); +} + +/* Nonzero when a claim made now would succeed. */ +__must_check int wc_can_save_vector_registers_x86(void) +{ + struct wc_svr_arm64_ctx_state *st; + int ret; + + if (in_nmi() || wc_svr_arm64_in_hardirq()) + return 0; + + preempt_disable(); + st = wc_svr_arm64_here(); + if (st->depth > 0) + ret = (st->inhibit_at == 0); + else + ret = may_use_simd() ? 1 : 0; + preempt_enable(); + + return ret; +} + +__must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags) +{ + struct wc_svr_arm64_ctx_state *st; + + if (in_nmi() || wc_svr_arm64_in_hardirq()) { + /* may_use_simd() is false here, and any open record belongs to the + * context this interrupt landed on. */ + atomic64_inc(&wc_svr_disallowed_count); + return WC_ACCEL_INHIBIT_E; + } + + preempt_disable(); + st = wc_svr_arm64_here(); + + if (st->depth > 0) { + /* Nested in this context's own section, which already holds the + * CPU, so the preempt_disable() above is balanced, not carried. */ + if (flags & WC_SVR_FLAG_MAYBE_INHIBIT) { + /* Pin-only claims are outermost only, as in the x86 glue. */ + preempt_enable(); + atomic64_inc(&wc_svr_disallowed_count); + return BAD_STATE_E; + } + if (st->inhibit_at != 0) { + preempt_enable(); + atomic64_inc(&wc_svr_disallowed_count); + return WC_ACCEL_INHIBIT_E; + } + ++st->depth; + if (flags & WC_SVR_FLAG_INHIBIT) + st->inhibit_at = st->depth; + preempt_enable(); + return 0; + } + + /* Outermost claim. The bottom-half disable below is what holds the CPU, + * taken by kernel_neon_begin() or wc_svr_arm64_pin_bh(). */ + if (flags & WC_SVR_FLAG_INHIBIT) { + wc_svr_arm64_pin_bh(st); + st->depth = 1; + st->inhibit_at = 1; + /* Counted as the x86 glue counts it: a span in which claims are + * refused, opened on request. */ + atomic64_inc(&wc_svr_disallowed_count); + return 0; + } + + if (! may_use_simd()) { + if (flags & WC_SVR_FLAG_MAYBE_INHIBIT) { + /* Held without the registers: nested claims are refused. */ + wc_svr_arm64_pin_bh(st); + st->depth = 1; + st->inhibit_at = 1; + atomic64_inc(&wc_svr_disallowed_count); + return 0; + } + preempt_enable(); + atomic64_inc(&wc_svr_disallowed_count); + return WC_ACCEL_INHIBIT_E; + } + + kernel_neon_begin(); + st->depth = 1; + st->neon_held = 1; + return 0; +} + +void wc_restore_vector_registers_x86(enum wc_svr_flags flags) +{ + struct wc_svr_arm64_ctx_state *st; + + if (in_nmi() || wc_svr_arm64_in_hardirq()) { + wc_linuxkm_pr_err_ratelimited("BUG: wc_restore_vector_registers_x86() " + "called from %s on CPU %d, where no claim can be open.\n", + in_nmi() ? "NMI" : "hardirq", raw_smp_processor_id()); + return; + } + + preempt_disable(); + st = wc_svr_arm64_here(); + if (st->depth == 0) { + preempt_enable(); + wc_linuxkm_pr_err_ratelimited("BUG: wc_restore_vector_registers_x86() " + "with no open claim on CPU %d.\n", raw_smp_processor_id()); + return; + } + /* The open claim already holds the CPU; balance the disable above. */ + preempt_enable(); + + if (st->inhibit_at == st->depth) { + if (! (flags & (WC_SVR_FLAG_INHIBIT | WC_SVR_FLAG_MAYBE_INHIBIT))) { + wc_linuxkm_pr_err_ratelimited("BUG: wc_restore_vector_registers_x86() " + "closing an inhibit claim without an inhibit flag on CPU %d.\n", + raw_smp_processor_id()); + } + st->inhibit_at = 0; + } + else if (flags & WC_SVR_FLAG_INHIBIT) { + wc_linuxkm_pr_err_ratelimited("BUG: wc_restore_vector_registers_x86() " + "with the inhibit flag but no matching inhibit claim on CPU %d.\n", + raw_smp_processor_id()); + } + + if (--st->depth == 0) { + if (st->neon_held) { + st->neon_held = 0; + kernel_neon_end(); + } + if (st->bh_held) { + st->bh_held = 0; + local_bh_enable(); + } + preempt_enable(); + } +} + +#endif /* !WC_SKIP_INCLUDED_C_FILES */ diff --git a/linuxkm/include.am b/linuxkm/include.am index e99954af615..07be6125fd0 100644 --- a/linuxkm/include.am +++ b/linuxkm/include.am @@ -16,6 +16,7 @@ EXTRA_DIST += m4/ax_linuxkm.m4 \ linuxkm/linuxkm_memory.h \ linuxkm/linuxkm_wc_port.h \ linuxkm/x86_vector_register_glue.c \ + linuxkm/arm64_vector_register_glue.c \ linuxkm/lkcapi_glue.c \ linuxkm/lkcapi_aes_glue.c \ linuxkm/lkcapi_sha_glue.c \ diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 051c37ed012..aaf07d17147 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -807,8 +807,10 @@ #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS #endif + /* x86 and arm64 share one interface: both glue files keep the wc_*_x86 + * names, so callers and the PIE redirect table are the same. */ #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ - defined(CONFIG_X86) + (defined(CONFIG_X86) || defined(CONFIG_ARM64)) extern __must_check int wc_linuxkm_allocate_svr_states(void); extern void wc_linuxkm_free_svr_states(void); @@ -821,6 +823,7 @@ WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags); WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags); + #ifdef CONFIG_X86 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) #include #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) @@ -834,6 +837,16 @@ #include #endif #endif + #else /* CONFIG_ARM64 */ + /* arch/arm64/include/asm/simd.h, and may_use_simd() with it, + * arrived in 4.14; the module otherwise accepts 3.16 and up. */ + #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) && \ + !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) + #error arm64 vector registers need may_use_simd(), added in 4.14. + #endif + #include /* may_use_simd() */ + #include /* kernel_neon_begin(), kernel_neon_end() */ + #endif #ifndef CAN_SAVE_VECTOR_REGISTERS #if defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF) && \ defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) @@ -943,6 +956,8 @@ #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT) #endif + /* A 0 return means a section is open on both glues, whether or not + * the registers came with it, so the caller must always release it. */ #ifndef SAVE_VECTOR_REGISTERS_MAYBE_INHIBIT #if (defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) || \ defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF)) @@ -957,41 +972,9 @@ #define RESTORE_VECTOR_REGISTERS_MAYBE_INHIBITED() wc_restore_vector_registers_x86(WC_SVR_FLAG_MAYBE_INHIBIT) #endif - #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64)) + #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_ARM) - #error kernel module ARM SIMD is not yet tested or usable. - - #include - - static WARN_UNUSED_RESULT inline int save_vector_registers_arm(void) - { - preempt_disable(); - if (! may_use_simd()) { - preempt_enable(); - return BAD_STATE_E; - } else { - fpsimd_preserve_current_state(); - return 0; - } - } - static inline void restore_vector_registers_arm(void) - { - fpsimd_restore_current_state(); - preempt_enable(); - } - - #ifndef SAVE_VECTOR_REGISTERS - #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_arm(); if (_svr_ret != 0) { fail_clause } } - #endif - #ifndef SAVE_VECTOR_REGISTERS2 - #define SAVE_VECTOR_REGISTERS2() save_vector_registers_arm() - #endif - #ifndef CAN_SAVE_VECTOR_REGISTERS - #define CAN_SAVE_VECTOR_REGISTERS() can_save_vector_registers_arm() - #endif - #ifndef RESTORE_VECTOR_REGISTERS - #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm() - #endif + #error kernel module 32-bit ARM SIMD is not yet tested or usable. #elif (defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ (!defined(SAVE_VECTOR_REGISTERS) || \ @@ -1350,13 +1333,13 @@ #ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS - #ifdef CONFIG_X86 + #if defined(CONFIG_X86) || defined(CONFIG_ARM64) typeof(wc_linuxkm_allocate_svr_states) *wc_linuxkm_allocate_svr_states; typeof(wc_can_save_vector_registers_x86) *wc_can_save_vector_registers_x86; typeof(wc_linuxkm_free_svr_states) *wc_linuxkm_free_svr_states; typeof(wc_restore_vector_registers_x86) *wc_restore_vector_registers_x86; typeof(wc_save_vector_registers_x86) *wc_save_vector_registers_x86; - #elif !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) /* !CONFIG_X86 */ + #elif !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) /* !CONFIG_X86 && !CONFIG_ARM64 */ #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture. #endif /* arch */ @@ -1709,7 +1692,8 @@ #undef get_current #define get_current WC_PIE_INDIRECT_SYM(get_current) - #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) + #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ + (defined(CONFIG_X86) || defined(CONFIG_ARM64)) #define wc_linuxkm_allocate_svr_states WC_PIE_INDIRECT_SYM(wc_linuxkm_allocate_svr_states) #define wc_can_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_can_save_vector_registers_x86) #define wc_linuxkm_free_svr_states WC_PIE_INDIRECT_SYM(wc_linuxkm_free_svr_states) @@ -2056,7 +2040,7 @@ #if !defined(BUILDING_WOLFSSL) /* some caller code needs these. */ #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) - #if defined(CONFIG_X86) + #if defined(CONFIG_X86) || defined(CONFIG_ARM64) WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void); WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags); WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags); @@ -2066,9 +2050,9 @@ #ifndef REENABLE_VECTOR_REGISTERS #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT) #endif - #elif !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) /* !CONFIG_X86 */ + #elif !defined(WC_DEBUG_FORCE_KERNEL_SETTINGS) /* !CONFIG_X86 && !CONFIG_ARM64 */ #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture. - #endif /* !CONFIG_X86 */ + #endif /* !CONFIG_X86 && !CONFIG_ARM64 */ #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */ #ifdef WC_LINUXKM_USE_HEAP_WRAPPERS WOLFSSL_API extern void *wc_linuxkm_malloc(size_t size); diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index ae44c2ef9c1..22defa69d5c 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -654,6 +654,8 @@ int wc_linuxkm_GenerateSeed_IntelRD(struct OS_Seed* os, byte* output, word32 sz) #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) #include "linuxkm/x86_vector_register_glue.c" +#elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_ARM64) + #include "linuxkm/arm64_vector_register_glue.c" #endif #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(WC_C_DYNAMIC_FALLBACK) && \ @@ -1702,11 +1704,21 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { #endif #ifndef CONFIG_FORTIFY_SOURCE #ifndef __ARCH_MEMCPY_NO_REDIRECT +#ifdef CONFIG_ARM64 + /* The plain names resolve to the module's own definitions here, so name + * the kernel's implementations (arch/arm64/lib/memcpy.S:243, memset.S:206). */ + wolfssl_linuxkm_pie_redirect_table.memcpy = __memcpy; +#else wolfssl_linuxkm_pie_redirect_table.memcpy = memcpy; #endif +#endif #ifndef __ARCH_MEMSET_NO_REDIRECT +#ifdef CONFIG_ARM64 + wolfssl_linuxkm_pie_redirect_table.memset = __memset; +#else wolfssl_linuxkm_pie_redirect_table.memset = memset; #endif +#endif #ifndef __ARCH_MEMMOVE_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.memmove = memmove; #endif @@ -1835,7 +1847,8 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread; -#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) +#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ + (defined(CONFIG_X86) || defined(CONFIG_ARM64)) wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_allocate_svr_states = wc_linuxkm_allocate_svr_states; wolfssl_linuxkm_pie_redirect_table.wc_can_save_vector_registers_x86 = wc_can_save_vector_registers_x86; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_free_svr_states = wc_linuxkm_free_svr_states; diff --git a/linuxkm/pie_redirect_table.c b/linuxkm/pie_redirect_table.c index 03be2e04fa0..d3c0999284b 100644 --- a/linuxkm/pie_redirect_table.c +++ b/linuxkm/pie_redirect_table.c @@ -53,13 +53,19 @@ const struct wolfssl_linuxkm_pie_redirect_table return &wolfssl_linuxkm_pie_redirect_table; } -/* placeholder implementations for missing functions. */ -#if defined(CONFIG_MIPS) +/* The container may hold no undefined symbol (linuxkm/Kbuild:301), so define + * these here. arm64 forwards to the kernel's __memcpy()/__memset() + * (arch/arm64/lib/memcpy.S:243, memset.S:206); the loops run before that. */ +#if defined(CONFIG_MIPS) || defined(CONFIG_ARM64) #undef memcpy void *memcpy(void *dest, const void *src, size_t n) { char *dest_i = (char *)dest; char *dest_end = dest_i + n; char *src_i = (char *)src; +#if defined(CONFIG_ARM64) && !defined(__ARCH_MEMCPY_NO_REDIRECT) + if (wolfssl_linuxkm_pie_redirect_table.memcpy != NULL) + return wolfssl_linuxkm_pie_redirect_table.memcpy(dest, src, n); +#endif while (dest_i < dest_end) *dest_i++ = *src_i++; return dest; @@ -69,6 +75,10 @@ const struct wolfssl_linuxkm_pie_redirect_table void *memset(void *dest, int c, size_t n) { char *dest_i = (char *)dest; char *dest_end = dest_i + n; +#if defined(CONFIG_ARM64) && !defined(__ARCH_MEMSET_NO_REDIRECT) + if (wolfssl_linuxkm_pie_redirect_table.memset != NULL) + return wolfssl_linuxkm_pie_redirect_table.memset(dest, c, n); +#endif while (dest_i < dest_end) *dest_i++ = c; return dest; diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c index 3e6b8b304cf..1ea6a80e64c 100644 --- a/wolfcrypt/benchmark/benchmark.c +++ b/wolfcrypt/benchmark/benchmark.c @@ -6932,6 +6932,20 @@ void bench_aesxts(void) WC_DECLARE_VAR(aes, XtsAes, 1, HEAP_HINT); double start; int i, count, ret; +#ifdef WOLFSSL_AESXTS_STREAM + /* Chunk the one-shot buffer so the two numbers compare directly. Must + * be a block multiple: only the final Update() may be short. */ + #ifndef BENCH_XTS_CHUNK + #ifdef BENCH_EMBEDDED + #define BENCH_XTS_CHUNK 256 + #else + #define BENCH_XTS_CHUNK (64*1024) + #endif + #endif + struct XtsAesStreamData stream; + word32 off; + word32 chunk; +#endif DECLARE_MULTI_VALUE_STATS_VARS() static const unsigned char k1[] = { @@ -6979,6 +6993,61 @@ void bench_aesxts(void) #ifdef MULTI_VALUE_STATISTICS bench_multi_value_stats(max, min, sum, squareSum, runs); #endif + +#ifdef WOLFSSL_AESXTS_STREAM + /* bench_size drops with -blocks and BENCH_EMBEDDED is 1 KiB, so keep two + * chunks or this times one Update() and calls it streaming. */ + chunk = (word32)BENCH_XTS_CHUNK; + if (chunk > (word32)bench_size / 2U) { + chunk = ((word32)bench_size / 2U) & ~(word32)(WC_AES_BLOCK_SIZE - 1); + } + if (chunk < (word32)WC_AES_BLOCK_SIZE) { + chunk = (word32)WC_AES_BLOCK_SIZE; + } + + if ((word32)bench_size >= (word32)WC_AES_BLOCK_SIZE) { + RESET_MULTI_VALUE_STATS_VARS(); + bench_stats_start(&count, &start); + do { + for (i = 0; i < numBlocks; i++) { + if ((ret = wc_AesXtsEncryptInit(aes, i1, sizeof(i1), + &stream)) != 0) { + printf("wc_AesXtsEncryptInit failed, ret = %d\n", ret); + goto exit; + } + for (off = 0; + off + chunk < (word32)bench_size; + off += chunk) { + if ((ret = wc_AesXtsEncryptUpdate(aes, bench_cipher + off, + bench_plain + off, chunk, + &stream)) != 0) { + printf("wc_AesXtsEncryptUpdate failed, ret = %d\n", + ret); + goto exit; + } + } + if ((ret = wc_AesXtsEncryptFinal(aes, bench_cipher + off, + bench_plain + off, + (word32)bench_size - off, &stream)) != 0) { + printf("wc_AesXtsEncryptFinal failed, ret = %d\n", ret); + goto exit; + } + RECORD_MULTI_VALUE_STATS(); + } + count += i; + } while (bench_stats_check(start) +#ifdef MULTI_VALUE_STATISTICS + || runs < minimum_runs +#endif + ); + + bench_stats_sym_finish("AES-XTS-stream-enc", 0, count, bench_size, + start, ret); +#ifdef MULTI_VALUE_STATISTICS + bench_multi_value_stats(max, min, sum, squareSum, runs); +#endif + } +#endif /* WOLFSSL_AESXTS_STREAM */ wc_AesXtsFree(aes); /* decryption benchmark */ @@ -7013,6 +7082,51 @@ void bench_aesxts(void) #ifdef MULTI_VALUE_STATISTICS bench_multi_value_stats(max, min, sum, squareSum, runs); #endif + +#ifdef WOLFSSL_AESXTS_STREAM + if ((word32)bench_size >= (word32)WC_AES_BLOCK_SIZE) { + RESET_MULTI_VALUE_STATS_VARS(); + bench_stats_start(&count, &start); + do { + for (i = 0; i < numBlocks; i++) { + if ((ret = wc_AesXtsDecryptInit(aes, i1, sizeof(i1), + &stream)) != 0) { + printf("wc_AesXtsDecryptInit failed, ret = %d\n", ret); + goto exit; + } + for (off = 0; + off + chunk < (word32)bench_size; + off += chunk) { + if ((ret = wc_AesXtsDecryptUpdate(aes, bench_plain + off, + bench_cipher + off, chunk, + &stream)) != 0) { + printf("wc_AesXtsDecryptUpdate failed, ret = %d\n", + ret); + goto exit; + } + } + if ((ret = wc_AesXtsDecryptFinal(aes, bench_plain + off, + bench_cipher + off, + (word32)bench_size - off, &stream)) != 0) { + printf("wc_AesXtsDecryptFinal failed, ret = %d\n", ret); + goto exit; + } + RECORD_MULTI_VALUE_STATS(); + } + count += i; + } while (bench_stats_check(start) +#ifdef MULTI_VALUE_STATISTICS + || runs < minimum_runs +#endif + ); + + bench_stats_sym_finish("AES-XTS-stream-dec", 0, count, bench_size, + start, ret); +#ifdef MULTI_VALUE_STATISTICS + bench_multi_value_stats(max, min, sum, squareSum, runs); +#endif + } +#endif /* WOLFSSL_AESXTS_STREAM */ #endif exit: diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 562e69c5974..a3c9cad9ca3 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -148,6 +148,24 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits #include +/* aarch64 claim for the NEON GHASH and the XTS block routine. Other builds + * expand these to nothing. */ +/* gcmKeySet gates streaming GCM. A failed claim leaves H underived, so the + * flag must go back off or streaming would run with a stale H. */ +#ifdef WOLFSSL_AESGCM_STREAM + #define WC_AES_GCM_UNKEY(aes) do { (aes)->gcmKeySet = 0; } while (0) +#else + #define WC_AES_GCM_UNKEY(aes) WC_DO_NOTHING +#endif + +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + #define WC_AES_ARM64_SVR_BEGIN() SAVE_VECTOR_REGISTERS(return _svr_ret;) + #define WC_AES_ARM64_SVR_END() RESTORE_VECTOR_REGISTERS() +#else + #define WC_AES_ARM64_SVR_BEGIN() WC_DO_NOTHING + #define WC_AES_ARM64_SVR_END() WC_DO_NOTHING +#endif + #ifdef WOLF_CRYPTO_CB #include #endif @@ -5980,7 +5998,9 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #else Check_CPU_support_HwCrypto(aes); if (aes->use_aes_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_set_key_AARCH64(userKey, keylen, (byte*)aes->key, dir); + RESTORE_VECTOR_REGISTERS(); } else #endif /* __aarch64__ */ @@ -5988,6 +6008,7 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \ defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_set_encrypt_key_NEON(userKey, keylen * 8, (byte*)aes->key); #ifdef HAVE_AES_DECRYPT if (dir == AES_DECRYPTION) { @@ -5996,6 +6017,7 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir) #else (void)dir; #endif + RESTORE_VECTOR_REGISTERS(); } #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ defined(WOLFSSL_ARM32_AES_DISPATCH) @@ -6298,7 +6320,42 @@ int wc_AesSetIV(Aes* aes, const byte* iv) #endif -#else /* !WOLFSSL_AESNI */ +#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ + defined(SAVE_VECTOR_REGISTERS2_DOES_NOTHING) + +#define VECTOR_REGISTERS_PUSH { \ + WC_DO_NOTHING + +#define VECTOR_REGISTERS_PUSH2(fail_clause) { \ + WC_DO_NOTHING + +#define VECTOR_REGISTERS_POP \ + } \ + WC_DO_NOTHING + +#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) + +/* aarch64: one mode-level claim for CCM, CFB, OFB, CMAC and key wrap. A + * refusal ends the call. */ +#define VECTOR_REGISTERS_PUSH { \ + if ((ret = SAVE_VECTOR_REGISTERS2()) != 0) { \ + return ret; \ + } \ + WC_DO_NOTHING + +#define VECTOR_REGISTERS_PUSH2(fail_clause) { \ + if ((ret = SAVE_VECTOR_REGISTERS2()) != 0) { \ + { fail_clause } \ + return ret; \ + } \ + WC_DO_NOTHING + +#define VECTOR_REGISTERS_POP \ + RESTORE_VECTOR_REGISTERS(); \ + } \ + WC_DO_NOTHING + +#else /* !WOLFSSL_AESNI && !(__aarch64__ && WOLFSSL_ARMASM) */ #define VECTOR_REGISTERS_PUSH WC_DO_NOTHING #define VECTOR_REGISTERS_PUSH2(fail_clause) WC_DO_NOTHING @@ -7350,8 +7407,10 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_CBC_encrypt_AARCH64(in, out, sz, (byte*)aes->reg, (byte*)aes->key, (int)aes->rounds); + RESTORE_VECTOR_REGISTERS(); } else #endif /* __aarch64__ */ @@ -7359,8 +7418,10 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \ defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_CBC_encrypt_NEON(in, out, sz, (const unsigned char*)aes->key, aes->rounds, (unsigned char*)aes->reg); + RESTORE_VECTOR_REGISTERS(); } #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ defined(WOLFSSL_ARM32_AES_DISPATCH) @@ -7604,8 +7665,10 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_CBC_decrypt_AARCH64(in, out, sz, (byte*)aes->reg, (byte*)aes->key, (int)aes->rounds); + RESTORE_VECTOR_REGISTERS(); } else #endif /* !__aarch64__ */ @@ -7615,8 +7678,10 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) if (sz >= 64) #endif { + SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_CBC_decrypt_NEON(in, out, sz, (const unsigned char*)aes->key, aes->rounds, (unsigned char*)aes->reg); + RESTORE_VECTOR_REGISTERS(); } #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP else @@ -8061,6 +8126,15 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) return MISSING_KEY; } + #if defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && \ + defined(__aarch64__) + /* Claimed before the drain below: failing after it would return an + * error with aes->left already decremented and output written. */ + if (aes->use_aes_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); + } + #endif + /* consume any unused bytes left in aes->tmp */ processed = min(aes->left, sz); xorbufout(out, in, (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left, @@ -8099,6 +8173,7 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) if (aes->use_aes_hw_crypto) { AES_CTR_encrypt_AARCH64(in, out, sz, (byte*)aes->reg, (byte*)aes->key, (byte*)aes->tmp, &aes->left, aes->rounds); + RESTORE_VECTOR_REGISTERS(); return 0; } else @@ -8124,9 +8199,11 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) if (sz >= 32) #endif { + SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_CTR_encrypt_NEON(in, out, numBlocks * WC_AES_BLOCK_SIZE, (byte*)aes->key, aes->rounds, (byte*)aes->reg); + RESTORE_VECTOR_REGISTERS(); } #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP else @@ -8156,9 +8233,11 @@ int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz) !defined(WOLFSSL_ARMASM_NO_NEON) && \ defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); AES_CTR_encrypt_NEON(zeros, (byte*)aes->tmp, WC_AES_BLOCK_SIZE, (byte*)aes->key, aes->rounds, (byte*)aes->reg); + RESTORE_VECTOR_REGISTERS(); } #else { @@ -8817,8 +8896,11 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) #endif /* WOLFSSL_ARM32_AES_DISPATCH */ #else if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { + SAVE_VECTOR_REGISTERS(WC_AES_GCM_UNKEY(aes); + return _svr_ret;); AES_GCM_set_key_AARCH64(iv, (byte*)aes->key, aes->gcm.H, aes->rounds); + RESTORE_VECTOR_REGISTERS(); } else #endif /* !__aarch64__ */ @@ -8826,8 +8908,11 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \ defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP) { + SAVE_VECTOR_REGISTERS(WC_AES_GCM_UNKEY(aes); + return _svr_ret;); AES_ECB_encrypt_NEON(iv, aes->gcm.H, WC_AES_BLOCK_SIZE, (const unsigned char*)aes->key, aes->rounds); + RESTORE_VECTOR_REGISTERS(); } #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) || \ defined(WOLFSSL_ARM32_AES_DISPATCH) @@ -14666,7 +14751,9 @@ int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv, #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); ret = AesGcmInit_AARCH64(aes, iv, ivSz); + RESTORE_VECTOR_REGISTERS(); } else #elif defined(WOLFSSL_RISCV_ASM) @@ -14674,7 +14761,9 @@ int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv, if (0) #endif /* WOLFSSL_AESNI */ { + WC_AES_ARM64_SVR_BEGIN(); ret = AesGcmInit_C(aes, iv, ivSz); + WC_AES_ARM64_SVR_END(); } if (ret == 0) @@ -14814,8 +14903,10 @@ int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); ret = AesGcmEncryptUpdate_AARCH64(aes, out, in, sz, authIn, authInSz); + RESTORE_VECTOR_REGISTERS(); } else #elif defined(WOLFSSL_RISCV_ASM) @@ -14823,6 +14914,7 @@ int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, if (0) #endif { + WC_AES_ARM64_SVR_BEGIN(); /* Encrypt the plaintext. */ ret = AesGcmCryptUpdate_C(aes, out, in, sz); if (ret == 0) { @@ -14830,6 +14922,7 @@ int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, * new cipher text. */ GHASH_UPDATE(aes, authIn, authInSz, out, sz); } + WC_AES_ARM64_SVR_END(); } } @@ -14884,7 +14977,9 @@ int wc_AesGcmEncryptFinal(Aes* aes, byte* authTag, word32 authTagSz) #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); ret = AesGcmEncryptFinal_AARCH64(aes, authTag, authTagSz); + RESTORE_VECTOR_REGISTERS(); } else #elif defined(WOLFSSL_RISCV_ASM) @@ -14892,7 +14987,9 @@ int wc_AesGcmEncryptFinal(Aes* aes, byte* authTag, word32 authTagSz) if (0) #endif { + WC_AES_ARM64_SVR_BEGIN(); ret = AesGcmFinal_C(aes, authTag, authTagSz); + WC_AES_ARM64_SVR_END(); } } @@ -14976,8 +15073,10 @@ int wc_AesGcmDecryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); ret = AesGcmDecryptUpdate_AARCH64(aes, out, in, sz, authIn, authInSz); + RESTORE_VECTOR_REGISTERS(); } else #elif defined(WOLFSSL_RISCV_ASM) @@ -14987,9 +15086,11 @@ int wc_AesGcmDecryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz, { /* Update the authentication tag with any authentication data and * cipher text. */ + WC_AES_ARM64_SVR_BEGIN(); GHASH_UPDATE(aes, authIn, authInSz, in, sz); /* Decrypt the cipher text. */ ret = AesGcmCryptUpdate_C(aes, out, in, sz); + WC_AES_ARM64_SVR_END(); } } @@ -15039,7 +15140,9 @@ int wc_AesGcmDecryptFinal(Aes* aes, const byte* authTag, word32 authTagSz) #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); ret = AesGcmDecryptFinal_AARCH64(aes, authTag, authTagSz); + RESTORE_VECTOR_REGISTERS(); } else #elif defined(WOLFSSL_RISCV_ASM) @@ -15049,7 +15152,9 @@ int wc_AesGcmDecryptFinal(Aes* aes, const byte* authTag, word32 authTagSz) { ALIGN32 byte calcTag[WC_AES_BLOCK_SIZE]; /* Calculate authentication tag. */ + WC_AES_ARM64_SVR_BEGIN(); ret = AesGcmFinal_C(aes, calcTag, WC_AES_BLOCK_SIZE); + WC_AES_ARM64_SVR_END(); if (ret == 0) { /* Check calculated tag matches the one passed in. */ if (ConstantCompare(authTag, calcTag, (int)authTagSz) != 0) { @@ -16974,9 +17079,8 @@ static WARN_UNUSED_RESULT int AesCfbDecrypt_C(Aes* aes, byte* out, if (blocks > WC_AES_CFB_DEC_BUF_BLOCKS) blocks = WC_AES_CFB_DEC_BUF_BLOCKS; nbytes = blocks * WC_AES_BLOCK_SIZE; - /* tmp[i] = E(C_i), read directly from the input. Already inside - * VECTOR_REGISTERS_PUSH, so use the inner ECB (no nested - * save/restore or re-dispatch) where available. */ + /* tmp[i] = E(C_i) from the input, already inside + * VECTOR_REGISTERS_PUSH. */ #if defined(WOLFSSL_AESNI) && defined(WOLFSSL_X86_64_BUILD) if (aes->use_aesni) { AesEcbEncryptBlocks(in, tmp, nbytes, (byte*)aes->key, @@ -18551,6 +18655,13 @@ void AES_XTS_decrypt_update_avx512(const unsigned char *in, unsigned char *out, #endif /* WOLFSSL_AESNI */ +#if defined(WOLFSSL_AESXTS_STREAM) && \ + defined(WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING) && FIPS_VERSION3_GE(6,0,0) + /* SP800-38E's per-tweak limit is enforced from this byte count, so + * switching the count off would switch the limit off with it. */ + #error "WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING is not allowed in a FIPS build" +#endif + #ifdef HAVE_AES_ECB #if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ @@ -18629,11 +18740,14 @@ static int AesXtsEncrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, int ret; byte tweak_block[WC_AES_BLOCK_SIZE]; + /* One claim for the tweak and the data below it. */ + WC_AES_ARM64_SVR_BEGIN(); ret = wc_AesEncryptDirect(&xaes->tweak, tweak_block, i); - if (ret != 0) - return ret; + if (ret == 0) + ret = AesXtsEncryptUpdate_sw(xaes, out, in, sz, tweak_block); + WC_AES_ARM64_SVR_END(); - return AesXtsEncryptUpdate_sw(xaes, out, in, sz, tweak_block); + return ret; } #endif /* !WOLFSSL_RISCV_ASM */ #endif @@ -18648,11 +18762,22 @@ static int AesXtsEncrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, * returns 0 on success */ static int AesXtsInitTweak_sw(XtsAes* xaes, byte* i) { - return wc_AesEncryptDirect(&xaes->tweak, i, i); + int ret; + WC_AES_ARM64_SVR_BEGIN(); + ret = wc_AesEncryptDirect(&xaes->tweak, i, i); + WC_AES_ARM64_SVR_END(); + return ret; } #endif /* WOLFSSL_AESXTS_STREAM */ +/* aarch64 parts with the AES crypto extensions run the same asm as the + * one-shot entry; NEON and table parts have no streaming asm yet. */ +#if defined(WOLFSSL_AESXTS_STREAM) && defined(__aarch64__) && \ + defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + #define WC_AES_XTS_STREAM_AARCH64 +#endif + #if !defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || \ defined(WOLFSSL_ARM32_AES_DISPATCH) || defined(WOLFSSL_AESXTS_STREAM) @@ -18677,12 +18802,17 @@ static int AesXtsEncryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, word32 blocks = (sz / WC_AES_BLOCK_SIZE); Aes *aes = &xaes->aes; + /* One claim for the call: the block routine below is vector code. */ + WC_AES_ARM64_SVR_BEGIN(); + #ifdef HAVE_AES_ECB /* encrypt all of buffer at once when possible */ if (in != out) { /* can not handle inline */ XMEMCPY(out, i, WC_AES_BLOCK_SIZE); - if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0) + if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0) { + WC_AES_ARM64_SVR_END(); return ret; + } } #endif @@ -18699,8 +18829,10 @@ static int AesXtsEncryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, XMEMCPY(buf, in, WC_AES_BLOCK_SIZE); xorbuf(buf, i, WC_AES_BLOCK_SIZE); ret = wc_AesEncryptDirect(aes, out, buf); - if (ret != 0) + if (ret != 0) { + WC_AES_ARM64_SVR_END(); return ret; + } } xorbuf(out, i, WC_AES_BLOCK_SIZE); @@ -18728,6 +18860,7 @@ static int AesXtsEncryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, XMEMCPY(buf, out - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE); if (sz >= WC_AES_BLOCK_SIZE) { /* extra sanity check before copy */ + WC_AES_ARM64_SVR_END(); return BUFFER_E; } if (in != out) { @@ -18748,6 +18881,8 @@ static int AesXtsEncryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, xorbuf(out - WC_AES_BLOCK_SIZE, i, WC_AES_BLOCK_SIZE); } + WC_AES_ARM64_SVR_END(); + return ret; } #endif @@ -18873,6 +19008,8 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, ret = AesXtsEncrypt_sw(xaes, out, in, sz, i); } #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) + /* One claim for the whole aarch64 section below. */ + SAVE_VECTOR_REGISTERS(return _svr_ret;); #if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) if (aes->use_aes_hw_crypto) { AES_XTS_encrypt_AARCH64(in, out, sz, i, (byte*)xaes->aes.key, @@ -18901,6 +19038,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, ret = 0; } #endif + RESTORE_VECTOR_REGISTERS(); #elif defined(WOLFSSL_PPC64_ASM) AES_XTS_encrypt(in, out, sz, i, (byte*)xaes->aes.key, (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); @@ -19015,20 +19153,30 @@ int wc_AesXtsEncryptInit(XtsAes* xaes, const byte* i, word32 iSz, * * returns 0 on success */ +/* The byte count is written back only after the work succeeds, so a failed + * claim leaves nothing to undo. */ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 sz, struct XtsAesStreamData *stream) { int ret; +#ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING + word32 newTweakBytes; +#endif -#if defined(WOLFSSL_AESNI) +#if defined(WOLFSSL_AESNI) || defined(WC_AES_XTS_STREAM_AARCH64) Aes *aes; #endif +#ifdef WC_AES_XTS_STREAM_AARCH64 + /* Scratch for the asm's ciphertext-stealing tail. Not xaes->aes.tmp: + * that is shared across concurrent requests on one tfm. */ + ALIGN16 byte xts_tmp[WC_AES_BLOCK_SIZE]; +#endif if (xaes == NULL || out == NULL || in == NULL) { return BAD_FUNC_ARG; } -#if defined(WOLFSSL_AESNI) +#if defined(WOLFSSL_AESNI) || defined(WC_AES_XTS_STREAM_AARCH64) aes = &xaes->aes; #endif @@ -19044,11 +19192,15 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s } #ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING + /* Outside FIPS the cap is only a recommendation (SP800-38E p.2, on IEEE + * 1619 5.1), so a stream may run longer. Refuse once the count can no + * longer advance. */ if (! WC_SAFE_SUM_WORD32(stream->bytes_crypted_with_this_tweak, sz, - stream->bytes_crypted_with_this_tweak)) + newTweakBytes)) { WOLFSSL_MSG("Overflow of stream->bytes_crypted_with_this_tweak " "in AesXtsEncryptUpdate()."); + return BAD_FUNC_ARG; } #endif #if FIPS_VERSION3_GE(6,0,0) @@ -19056,8 +19208,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s * WC_AES_BLOCK_SIZE or 16-bytes (128-bits). So each key may only be used to * protect up to 1,048,576 blocks of WC_AES_BLOCK_SIZE (16,777,216 bytes) */ - if (stream->bytes_crypted_with_this_tweak > - FIPS_AES_XTS_MAX_BYTES_PER_TWEAK) + if (newTweakBytes > FIPS_AES_XTS_MAX_BYTES_PER_TWEAK) { WOLFSSL_MSG("Request exceeds allowed bytes per SP800-38E"); return BAD_FUNC_ARG; @@ -19108,11 +19259,28 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s } else #endif /* WOLFSSL_AESNI */ +#ifdef WC_AES_XTS_STREAM_AARCH64 + if (aes->use_aes_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); + AES_XTS_encrypt_update_AARCH64(in, out, sz, (byte*)aes->key, + stream->tweak_block, xts_tmp, (int)aes->rounds); + ret = 0; + ForceZero(xts_tmp, sizeof(xts_tmp)); + RESTORE_VECTOR_REGISTERS(); + } + else +#endif { ret = AesXtsEncryptUpdate_sw(xaes, out, in, sz, stream->tweak_block); } } +#ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING + if (ret == 0) { + stream->bytes_crypted_with_this_tweak = newTweakBytes; + } +#endif + return ret; } @@ -19176,11 +19344,14 @@ static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz, int ret; byte tweak_block[WC_AES_BLOCK_SIZE]; + /* One claim for the tweak and the data below it. */ + WC_AES_ARM64_SVR_BEGIN(); ret = wc_AesEncryptDirect(&xaes->tweak, tweak_block, i); - if (ret != 0) - return ret; + if (ret == 0) + ret = AesXtsDecryptUpdate_sw(xaes, out, in, sz, tweak_block); + WC_AES_ARM64_SVR_END(); - return AesXtsDecryptUpdate_sw(xaes, out, in, sz, tweak_block); + return ret; } #endif /* !WOLFSSL_RISCV_ASM */ #endif @@ -19224,12 +19395,17 @@ static int AesXtsDecryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, blocks--; } + /* One claim for the call: the block routine below is vector code. */ + WC_AES_ARM64_SVR_BEGIN(); + #ifdef HAVE_AES_ECB /* decrypt all of buffer at once when possible */ if (in != out) { /* can not handle inline */ XMEMCPY(out, i, WC_AES_BLOCK_SIZE); - if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0) + if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0) { + WC_AES_ARM64_SVR_END(); return ret; + } } #endif @@ -19243,8 +19419,10 @@ static int AesXtsDecryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, XMEMCPY(buf, in, WC_AES_BLOCK_SIZE); xorbuf(buf, i, WC_AES_BLOCK_SIZE); ret = wc_AesDecryptDirect(aes, out, buf); - if (ret != 0) + if (ret != 0) { + WC_AES_ARM64_SVR_END(); return ret; + } } xorbuf(out, i, WC_AES_BLOCK_SIZE); @@ -19287,8 +19465,10 @@ static int AesXtsDecryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, XMEMCPY(buf, in, WC_AES_BLOCK_SIZE); xorbuf(buf, tmp2, WC_AES_BLOCK_SIZE); ret = wc_AesDecryptDirect(aes, out, buf); - if (ret != 0) + if (ret != 0) { + WC_AES_ARM64_SVR_END(); return ret; + } xorbuf(out, tmp2, WC_AES_BLOCK_SIZE); /* tmp2 holds partial | last */ @@ -19300,6 +19480,7 @@ static int AesXtsDecryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, /* Make buffer with end of cipher text | last */ XMEMCPY(buf, tmp2, WC_AES_BLOCK_SIZE); if (sz >= WC_AES_BLOCK_SIZE) { /* extra sanity check before copy */ + WC_AES_ARM64_SVR_END(); return BUFFER_E; } XMEMCPY(buf, in, sz); @@ -19307,12 +19488,16 @@ static int AesXtsDecryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in, xorbuf(buf, i, WC_AES_BLOCK_SIZE); ret = wc_AesDecryptDirect(aes, tmp2, buf); - if (ret != 0) + if (ret != 0) { + WC_AES_ARM64_SVR_END(); return ret; + } xorbuf(tmp2, i, WC_AES_BLOCK_SIZE); XMEMCPY(out - WC_AES_BLOCK_SIZE, tmp2, WC_AES_BLOCK_SIZE); } + WC_AES_ARM64_SVR_END(); + return ret; } #endif @@ -19345,13 +19530,16 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, aes = &xaes->aes; #endif -/* FIPS TODO: SP800-38E - Restrict data unit to 2^20 blocks per key. A block is - * WC_AES_BLOCK_SIZE or 16-bytes (128-bits). So each key may only be used to - * protect up to 1,048,576 blocks of WC_AES_BLOCK_SIZE (16,777,216 bytes or - * 134,217,728-bits) Add helpful printout and message along with BAD_FUNC_ARG - * return whenever sz / WC_AES_BLOCK_SIZE > 1,048,576 or equal to that and sz is - * not a sequence of complete blocks. - */ +#if FIPS_VERSION3_GE(6,0,0) + /* SP800-38E - Restrict data unit to 2^20 blocks per key. A block is + * WC_AES_BLOCK_SIZE or 16-bytes (128-bits). So each key may only be used to + * protect up to 1,048,576 blocks of WC_AES_BLOCK_SIZE (16,777,216 bytes) + */ + if (sz > FIPS_AES_XTS_MAX_BYTES_PER_TWEAK) { + WOLFSSL_MSG("Request exceeds allowed bytes per SP800-38E"); + return BAD_FUNC_ARG; + } +#endif /* rounds == 0 means no software key schedule: XTS has no crypto * callback dispatch, so a device-owned key is unusable here. */ @@ -19441,6 +19629,8 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, ret = AesXtsDecrypt_sw(xaes, out, in, sz, i); } #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) + /* One claim for the whole aarch64 section below. */ + SAVE_VECTOR_REGISTERS(return _svr_ret;); #if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) if (aes->use_aes_hw_crypto) { AES_XTS_decrypt_AARCH64(in, out, sz, i, (byte*)aes->key, @@ -19469,6 +19659,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, ret = 0; } #endif + RESTORE_VECTOR_REGISTERS(); #elif defined(WOLFSSL_PPC64_ASM) AES_XTS_decrypt(in, out, sz, i, (byte*)aes->key, (byte*)xaes->tweak.key, (byte*)aes->tmp, aes->rounds); @@ -19585,19 +19776,29 @@ int wc_AesXtsDecryptInit(XtsAes* xaes, const byte* i, word32 iSz, * * returns 0 on success */ +/* The byte count is written back only after the work succeeds, so a failed + * claim leaves nothing to undo. */ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 sz, struct XtsAesStreamData *stream) { int ret; -#if defined(WOLFSSL_AESNI) +#ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING + word32 newTweakBytes; +#endif +#if defined(WOLFSSL_AESNI) || defined(WC_AES_XTS_STREAM_AARCH64) Aes *aes; #endif +#ifdef WC_AES_XTS_STREAM_AARCH64 + /* Scratch for the asm's ciphertext-stealing tail. Not xaes->aes.tmp: + * that is shared across concurrent requests on one tfm. */ + ALIGN16 byte xts_tmp[WC_AES_BLOCK_SIZE]; +#endif if (xaes == NULL || out == NULL || in == NULL) { return BAD_FUNC_ARG; } -#if defined(WOLFSSL_AESNI) +#if defined(WOLFSSL_AESNI) || defined(WC_AES_XTS_STREAM_AARCH64) #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS aes = &xaes->aes_decrypt; #else @@ -19618,11 +19819,26 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s } #ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING + /* Outside FIPS the cap is only a recommendation (SP800-38E p.2, on IEEE + * 1619 5.1), so a stream may run longer. Refuse once the count can no + * longer advance. */ if (! WC_SAFE_SUM_WORD32(stream->bytes_crypted_with_this_tweak, sz, - stream->bytes_crypted_with_this_tweak)) + newTweakBytes)) { WOLFSSL_MSG("Overflow of stream->bytes_crypted_with_this_tweak " "in AesXtsDecryptUpdate()."); + return BAD_FUNC_ARG; + } +#endif +#if FIPS_VERSION3_GE(6,0,0) + /* SP800-38E - Restrict data unit to 2^20 blocks per key. A block is + * WC_AES_BLOCK_SIZE or 16-bytes (128-bits). So each key may only be used to + * protect up to 1,048,576 blocks of WC_AES_BLOCK_SIZE (16,777,216 bytes) + */ + if (newTweakBytes > FIPS_AES_XTS_MAX_BYTES_PER_TWEAK) + { + WOLFSSL_MSG("Request exceeds allowed bytes per SP800-38E"); + return BAD_FUNC_ARG; } #endif @@ -19671,12 +19887,29 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s } else #endif /* WOLFSSL_AESNI */ +#ifdef WC_AES_XTS_STREAM_AARCH64 + if (aes->use_aes_hw_crypto) { + SAVE_VECTOR_REGISTERS(return _svr_ret;); + AES_XTS_decrypt_update_AARCH64(in, out, sz, (byte*)aes->key, + stream->tweak_block, xts_tmp, (int)aes->rounds); + ret = 0; + ForceZero(xts_tmp, sizeof(xts_tmp)); + RESTORE_VECTOR_REGISTERS(); + } + else +#endif { ret = AesXtsDecryptUpdate_sw(xaes, out, in, sz, stream->tweak_block); } } +#ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING + if (ret == 0) { + stream->bytes_crypted_with_this_tweak = newTweakBytes; + } +#endif + return ret; } diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm.S b/wolfcrypt/src/port/arm/armv8-aes-asm.S index 6895ef04e8f..0a63135209f 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm.S +++ b/wolfcrypt/src/port/arm/armv8-aes-asm.S @@ -43193,6 +43193,1959 @@ L_aes_xts_decrypt_arm64_crypto_done: .size AES_XTS_decrypt_AARCH64,.-AES_XTS_decrypt_AARCH64 #endif /* __APPLE__ */ #endif /* HAVE_AES_DECRYPT */ +#ifdef WOLFSSL_AESXTS_STREAM +#ifndef __APPLE__ +.text +.globl AES_XTS_encrypt_update_AARCH64 +WC_ASM_ATT_HIDDEN(AES_XTS_encrypt_update_AARCH64) +.type AES_XTS_encrypt_update_AARCH64,@function +.align 2 +AES_XTS_encrypt_update_AARCH64: +#else +.section __TEXT,__text +.globl _AES_XTS_encrypt_update_AARCH64 +WC_ASM_ATT_HIDDEN(_AES_XTS_encrypt_update_AARCH64) +.p2align 2 +_AES_XTS_encrypt_update_AARCH64: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + str x17, [x29, #16] + ld1 {v4.16b}, [x4] + mov x9, v4.d[0] + mov x10, v4.d[1] + lsr w7, w2, #4 + and w2, w2, #15 + mov x17, #0x87 + cmp x6, #12 + blt L_aes_xts_encrypt_update_arm64_crypto_start_128 + bgt L_aes_xts_encrypt_update_arm64_crypto_start_256 + # AES_XTS_192 +#ifndef NO_AES_192 + ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 + ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x3], #0x40 + ld1 {v28.2d}, [x3] + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + and x8, x17, x12, asr 63 + extr x14, x12, x11, #63 + eor x13, x8, x11, lsl 1 + and x8, x17, x14, asr 63 + extr x16, x14, x13, #63 + eor x15, x8, x13, lsl 1 + cmp w7, #4 + blt L_aes_xts_encrypt_update_arm64_crypto_192_start_2 +L_aes_xts_encrypt_update_arm64_crypto_192_start_4: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 + mov v5.d[0], x11 + mov v5.d[1], x12 + mov v6.d[0], x13 + mov v6.d[1], x14 + mov v7.d[0], x15 + mov v7.d[1], x16 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x16, asr 63 + aese v1.16b, v16.16b + aesmc v1.16b, v1.16b + aese v2.16b, v16.16b + aesmc v2.16b, v2.16b + extr x10, x16, x15, #63 + aese v3.16b, v16.16b + aesmc v3.16b, v3.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + eor x9, x8, x15, lsl 1 + aese v1.16b, v17.16b + aesmc v1.16b, v1.16b + aese v2.16b, v17.16b + aesmc v2.16b, v2.16b + and x8, x17, x10, asr 63 + aese v3.16b, v17.16b + aesmc v3.16b, v3.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + extr x12, x10, x9, #63 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b + eor x11, x8, x9, lsl 1 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b + extr x14, x12, x11, #63 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + eor x13, x8, x11, lsl 1 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b + and x8, x17, x14, asr 63 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + extr x16, x14, x13, #63 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b + eor x15, x8, x13, lsl 1 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b + aese v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + aese v1.16b, v27.16b + eor v1.16b, v1.16b, v28.16b + aese v2.16b, v27.16b + eor v2.16b, v2.16b, v28.16b + aese v3.16b, v27.16b + eor v3.16b, v3.16b, v28.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + cmp w7, #4 + bge L_aes_xts_encrypt_update_arm64_crypto_192_start_4 +L_aes_xts_encrypt_update_arm64_crypto_192_start_2: + cmp w7, #2 + blt L_aes_xts_encrypt_update_arm64_crypto_192_start_1 + ld1 {v0.16b, v1.16b}, [x0], #32 + mov v5.d[0], x11 + mov v5.d[1], x12 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aese v1.16b, v16.16b + aesmc v1.16b, v1.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + extr x10, x12, x11, #63 + aese v1.16b, v17.16b + aesmc v1.16b, v1.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + eor x9, x8, x11, lsl 1 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + extr x12, x10, x9, #63 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + eor x11, x8, x9, lsl 1 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b + aese v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + aese v1.16b, v27.16b + eor v1.16b, v1.16b, v28.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #2 + st1 {v0.16b, v1.16b}, [x1], #32 +L_aes_xts_encrypt_update_arm64_crypto_192_start_1: + cbz w7, L_aes_xts_encrypt_update_arm64_crypto_192_done + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v4.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + extr x10, x10, x9, #63 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + eor x9, x8, x9, lsl 1 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b + aese v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + eor v0.16b, v0.16b, v4.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + st1 {v0.16b}, [x1], #16 +L_aes_xts_encrypt_update_arm64_crypto_192_done: + cbz w2, L_aes_xts_encrypt_update_arm64_crypto_192_partial_done + sub x1, x1, #16 + ld1 {v0.16b}, [x1], #16 + st1 {v0.2d}, [x5] + mov w8, w2 +L_aes_xts_encrypt_update_arm64_crypto_192_start_byte: + ldrb w11, [x5] + ldrb w12, [x0], #1 + strb w11, [x1], #1 + strb w12, [x5], #1 + subs w8, w8, #1 + bgt L_aes_xts_encrypt_update_arm64_crypto_192_start_byte + sub x1, x1, x2 + sub x5, x5, x2 + sub x1, x1, #16 + ld1 {v0.2d}, [x5] + eor v0.16b, v0.16b, v4.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b + aese v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x1] +L_aes_xts_encrypt_update_arm64_crypto_192_partial_done: +#endif /* !NO_AES_192 */ + b L_aes_xts_encrypt_update_arm64_crypto_done + # AES_XTS_256 +L_aes_xts_encrypt_update_arm64_crypto_start_256: +#ifndef NO_AES_256 + ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 + ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x3], #0x40 + ld1 {v28.2d, v29.2d}, [x3], #32 + ld1 {v30.2d}, [x3] + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + and x8, x17, x12, asr 63 + extr x14, x12, x11, #63 + eor x13, x8, x11, lsl 1 + and x8, x17, x14, asr 63 + extr x16, x14, x13, #63 + eor x15, x8, x13, lsl 1 + cmp w7, #4 + blt L_aes_xts_encrypt_update_arm64_crypto_256_start_2 +L_aes_xts_encrypt_update_arm64_crypto_256_start_4: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 + mov v5.d[0], x11 + mov v5.d[1], x12 + mov v6.d[0], x13 + mov v6.d[1], x14 + mov v7.d[0], x15 + mov v7.d[1], x16 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x16, asr 63 + aese v1.16b, v16.16b + aesmc v1.16b, v1.16b + aese v2.16b, v16.16b + aesmc v2.16b, v2.16b + extr x10, x16, x15, #63 + aese v3.16b, v16.16b + aesmc v3.16b, v3.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + eor x9, x8, x15, lsl 1 + aese v1.16b, v17.16b + aesmc v1.16b, v1.16b + aese v2.16b, v17.16b + aesmc v2.16b, v2.16b + and x8, x17, x10, asr 63 + aese v3.16b, v17.16b + aesmc v3.16b, v3.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + extr x12, x10, x9, #63 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b + eor x11, x8, x9, lsl 1 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b + extr x14, x12, x11, #63 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + eor x13, x8, x11, lsl 1 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b + and x8, x17, x14, asr 63 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + extr x16, x14, x13, #63 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b + eor x15, x8, x13, lsl 1 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b + aese v2.16b, v25.16b + aesmc v2.16b, v2.16b + aese v3.16b, v25.16b + aesmc v3.16b, v3.16b + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b + aese v2.16b, v26.16b + aesmc v2.16b, v2.16b + aese v3.16b, v26.16b + aesmc v3.16b, v3.16b + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b + aese v2.16b, v27.16b + aesmc v2.16b, v2.16b + aese v3.16b, v27.16b + aesmc v3.16b, v3.16b + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b + aese v2.16b, v28.16b + aesmc v2.16b, v2.16b + aese v3.16b, v28.16b + aesmc v3.16b, v3.16b + aese v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + aese v1.16b, v29.16b + eor v1.16b, v1.16b, v30.16b + aese v2.16b, v29.16b + eor v2.16b, v2.16b, v30.16b + aese v3.16b, v29.16b + eor v3.16b, v3.16b, v30.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + cmp w7, #4 + bge L_aes_xts_encrypt_update_arm64_crypto_256_start_4 +L_aes_xts_encrypt_update_arm64_crypto_256_start_2: + cmp w7, #2 + blt L_aes_xts_encrypt_update_arm64_crypto_256_start_1 + ld1 {v0.16b, v1.16b}, [x0], #32 + mov v5.d[0], x11 + mov v5.d[1], x12 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aese v1.16b, v16.16b + aesmc v1.16b, v1.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + extr x10, x12, x11, #63 + aese v1.16b, v17.16b + aesmc v1.16b, v1.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + eor x9, x8, x11, lsl 1 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + extr x12, x10, x9, #63 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + eor x11, x8, x9, lsl 1 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b + aese v1.16b, v25.16b + aesmc v1.16b, v1.16b + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b + aese v1.16b, v26.16b + aesmc v1.16b, v1.16b + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b + aese v1.16b, v27.16b + aesmc v1.16b, v1.16b + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b + aese v1.16b, v28.16b + aesmc v1.16b, v1.16b + aese v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + aese v1.16b, v29.16b + eor v1.16b, v1.16b, v30.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #2 + st1 {v0.16b, v1.16b}, [x1], #32 +L_aes_xts_encrypt_update_arm64_crypto_256_start_1: + cbz w7, L_aes_xts_encrypt_update_arm64_crypto_256_done + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v4.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + extr x10, x10, x9, #63 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + eor x9, x8, x9, lsl 1 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b + aese v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + eor v0.16b, v0.16b, v4.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + st1 {v0.16b}, [x1], #16 +L_aes_xts_encrypt_update_arm64_crypto_256_done: + cbz w2, L_aes_xts_encrypt_update_arm64_crypto_256_partial_done + sub x1, x1, #16 + ld1 {v0.16b}, [x1], #16 + st1 {v0.2d}, [x5] + mov w8, w2 +L_aes_xts_encrypt_update_arm64_crypto_256_start_byte: + ldrb w11, [x5] + ldrb w12, [x0], #1 + strb w11, [x1], #1 + strb w12, [x5], #1 + subs w8, w8, #1 + bgt L_aes_xts_encrypt_update_arm64_crypto_256_start_byte + sub x1, x1, x2 + sub x5, x5, x2 + sub x1, x1, #16 + ld1 {v0.2d}, [x5] + eor v0.16b, v0.16b, v4.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v0.16b, v25.16b + aesmc v0.16b, v0.16b + aese v0.16b, v26.16b + aesmc v0.16b, v0.16b + aese v0.16b, v27.16b + aesmc v0.16b, v0.16b + aese v0.16b, v28.16b + aesmc v0.16b, v0.16b + aese v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x1] +L_aes_xts_encrypt_update_arm64_crypto_256_partial_done: +#endif /* !NO_AES_256 */ + b L_aes_xts_encrypt_update_arm64_crypto_done + # AES_XTS_128 +L_aes_xts_encrypt_update_arm64_crypto_start_128: +#ifndef NO_AES_128 + ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 + ld1 {v24.2d, v25.2d}, [x3], #32 + ld1 {v26.2d}, [x3] + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + and x8, x17, x12, asr 63 + extr x14, x12, x11, #63 + eor x13, x8, x11, lsl 1 + and x8, x17, x14, asr 63 + extr x16, x14, x13, #63 + eor x15, x8, x13, lsl 1 + cmp w7, #4 + blt L_aes_xts_encrypt_update_arm64_crypto_128_start_2 +L_aes_xts_encrypt_update_arm64_crypto_128_start_4: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 + mov v5.d[0], x11 + mov v5.d[1], x12 + mov v6.d[0], x13 + mov v6.d[1], x14 + mov v7.d[0], x15 + mov v7.d[1], x16 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x16, asr 63 + aese v1.16b, v16.16b + aesmc v1.16b, v1.16b + aese v2.16b, v16.16b + aesmc v2.16b, v2.16b + extr x10, x16, x15, #63 + aese v3.16b, v16.16b + aesmc v3.16b, v3.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + eor x9, x8, x15, lsl 1 + aese v1.16b, v17.16b + aesmc v1.16b, v1.16b + aese v2.16b, v17.16b + aesmc v2.16b, v2.16b + and x8, x17, x10, asr 63 + aese v3.16b, v17.16b + aesmc v3.16b, v3.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + extr x12, x10, x9, #63 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b + aese v2.16b, v18.16b + aesmc v2.16b, v2.16b + eor x11, x8, x9, lsl 1 + aese v3.16b, v18.16b + aesmc v3.16b, v3.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b + aese v2.16b, v19.16b + aesmc v2.16b, v2.16b + extr x14, x12, x11, #63 + aese v3.16b, v19.16b + aesmc v3.16b, v3.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + eor x13, x8, x11, lsl 1 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b + aese v2.16b, v20.16b + aesmc v2.16b, v2.16b + and x8, x17, x14, asr 63 + aese v3.16b, v20.16b + aesmc v3.16b, v3.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + extr x16, x14, x13, #63 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b + aese v2.16b, v21.16b + aesmc v2.16b, v2.16b + eor x15, x8, x13, lsl 1 + aese v3.16b, v21.16b + aesmc v3.16b, v3.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b + aese v2.16b, v22.16b + aesmc v2.16b, v2.16b + aese v3.16b, v22.16b + aesmc v3.16b, v3.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b + aese v2.16b, v23.16b + aesmc v2.16b, v2.16b + aese v3.16b, v23.16b + aesmc v3.16b, v3.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b + aese v2.16b, v24.16b + aesmc v2.16b, v2.16b + aese v3.16b, v24.16b + aesmc v3.16b, v3.16b + aese v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + aese v1.16b, v25.16b + eor v1.16b, v1.16b, v26.16b + aese v2.16b, v25.16b + eor v2.16b, v2.16b, v26.16b + aese v3.16b, v25.16b + eor v3.16b, v3.16b, v26.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + cmp w7, #4 + bge L_aes_xts_encrypt_update_arm64_crypto_128_start_4 +L_aes_xts_encrypt_update_arm64_crypto_128_start_2: + cmp w7, #2 + blt L_aes_xts_encrypt_update_arm64_crypto_128_start_1 + ld1 {v0.16b, v1.16b}, [x0], #32 + mov v5.d[0], x11 + mov v5.d[1], x12 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aese v1.16b, v16.16b + aesmc v1.16b, v1.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + extr x10, x12, x11, #63 + aese v1.16b, v17.16b + aesmc v1.16b, v1.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + eor x9, x8, x11, lsl 1 + aese v1.16b, v18.16b + aesmc v1.16b, v1.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aese v1.16b, v19.16b + aesmc v1.16b, v1.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + extr x12, x10, x9, #63 + aese v1.16b, v20.16b + aesmc v1.16b, v1.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + eor x11, x8, x9, lsl 1 + aese v1.16b, v21.16b + aesmc v1.16b, v1.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v1.16b, v22.16b + aesmc v1.16b, v1.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v1.16b, v23.16b + aesmc v1.16b, v1.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v1.16b, v24.16b + aesmc v1.16b, v1.16b + aese v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + aese v1.16b, v25.16b + eor v1.16b, v1.16b, v26.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #2 + st1 {v0.16b, v1.16b}, [x1], #32 +L_aes_xts_encrypt_update_arm64_crypto_128_start_1: + cbz w7, L_aes_xts_encrypt_update_arm64_crypto_128_done + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v4.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + extr x10, x10, x9, #63 + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + eor x9, x8, x9, lsl 1 + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + eor v0.16b, v0.16b, v4.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + st1 {v0.16b}, [x1], #16 +L_aes_xts_encrypt_update_arm64_crypto_128_done: + cbz w2, L_aes_xts_encrypt_update_arm64_crypto_128_partial_done + sub x1, x1, #16 + ld1 {v0.16b}, [x1], #16 + st1 {v0.2d}, [x5] + mov w8, w2 +L_aes_xts_encrypt_update_arm64_crypto_128_start_byte: + ldrb w11, [x5] + ldrb w12, [x0], #1 + strb w11, [x1], #1 + strb w12, [x5], #1 + subs w8, w8, #1 + bgt L_aes_xts_encrypt_update_arm64_crypto_128_start_byte + sub x1, x1, x2 + sub x5, x5, x2 + sub x1, x1, #16 + ld1 {v0.2d}, [x5] + eor v0.16b, v0.16b, v4.16b + aese v0.16b, v16.16b + aesmc v0.16b, v0.16b + aese v0.16b, v17.16b + aesmc v0.16b, v0.16b + aese v0.16b, v18.16b + aesmc v0.16b, v0.16b + aese v0.16b, v19.16b + aesmc v0.16b, v0.16b + aese v0.16b, v20.16b + aesmc v0.16b, v0.16b + aese v0.16b, v21.16b + aesmc v0.16b, v0.16b + aese v0.16b, v22.16b + aesmc v0.16b, v0.16b + aese v0.16b, v23.16b + aesmc v0.16b, v0.16b + aese v0.16b, v24.16b + aesmc v0.16b, v0.16b + aese v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x1] +L_aes_xts_encrypt_update_arm64_crypto_128_partial_done: +#endif /* !NO_AES_128 */ +L_aes_xts_encrypt_update_arm64_crypto_done: + st1 {v4.16b}, [x4] + ldr x17, [x29, #16] + ldp x29, x30, [sp], #32 + ret +#ifndef __APPLE__ + .size AES_XTS_encrypt_update_AARCH64,.-AES_XTS_encrypt_update_AARCH64 +#endif /* __APPLE__ */ +#ifdef HAVE_AES_DECRYPT +#ifndef __APPLE__ +.text +.globl AES_XTS_decrypt_update_AARCH64 +WC_ASM_ATT_HIDDEN(AES_XTS_decrypt_update_AARCH64) +.type AES_XTS_decrypt_update_AARCH64,@function +.align 2 +AES_XTS_decrypt_update_AARCH64: +#else +.section __TEXT,__text +.globl _AES_XTS_decrypt_update_AARCH64 +WC_ASM_ATT_HIDDEN(_AES_XTS_decrypt_update_AARCH64) +.p2align 2 +_AES_XTS_decrypt_update_AARCH64: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + str x17, [x29, #16] + ld1 {v4.16b}, [x4] + mov x9, v4.d[0] + mov x10, v4.d[1] + lsr w7, w2, #4 + ands w2, w2, #15 + mov x17, #0x87 + cset w8, ne + sub w7, w7, w8 + cmp x6, #12 + blt L_aes_xts_decrypt_update_arm64_crypto_start_128 + bgt L_aes_xts_decrypt_update_arm64_crypto_start_256 + # AES_XTS_192 +#ifndef NO_AES_192 + ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 + ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x3], #0x40 + ld1 {v28.2d}, [x3] + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + and x8, x17, x12, asr 63 + extr x14, x12, x11, #63 + eor x13, x8, x11, lsl 1 + and x8, x17, x14, asr 63 + extr x16, x14, x13, #63 + eor x15, x8, x13, lsl 1 + cmp w7, #4 + blt L_aes_xts_decrypt_update_arm64_crypto_192_start_2 +L_aes_xts_decrypt_update_arm64_crypto_192_start_4: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 + mov v5.d[0], x11 + mov v5.d[1], x12 + mov v6.d[0], x13 + mov v6.d[1], x14 + mov v7.d[0], x15 + mov v7.d[1], x16 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x16, asr 63 + aesd v1.16b, v16.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v16.16b + aesimc v2.16b, v2.16b + extr x10, x16, x15, #63 + aesd v3.16b, v16.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + eor x9, x8, x15, lsl 1 + aesd v1.16b, v17.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v17.16b + aesimc v2.16b, v2.16b + and x8, x17, x10, asr 63 + aesd v3.16b, v17.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + extr x12, x10, x9, #63 + aesd v1.16b, v18.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v18.16b + aesimc v2.16b, v2.16b + eor x11, x8, x9, lsl 1 + aesd v3.16b, v18.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aesd v1.16b, v19.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v19.16b + aesimc v2.16b, v2.16b + extr x14, x12, x11, #63 + aesd v3.16b, v19.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + eor x13, x8, x11, lsl 1 + aesd v1.16b, v20.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v20.16b + aesimc v2.16b, v2.16b + and x8, x17, x14, asr 63 + aesd v3.16b, v20.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + extr x16, x14, x13, #63 + aesd v1.16b, v21.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v21.16b + aesimc v2.16b, v2.16b + eor x15, x8, x13, lsl 1 + aesd v3.16b, v21.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v22.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v22.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v22.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v23.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v23.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v23.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v24.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v24.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v24.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v25.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v25.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v25.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v26.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v26.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v26.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + aesd v1.16b, v27.16b + eor v1.16b, v1.16b, v28.16b + aesd v2.16b, v27.16b + eor v2.16b, v2.16b, v28.16b + aesd v3.16b, v27.16b + eor v3.16b, v3.16b, v28.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + cmp w7, #4 + bge L_aes_xts_decrypt_update_arm64_crypto_192_start_4 +L_aes_xts_decrypt_update_arm64_crypto_192_start_2: + cmp w7, #2 + blt L_aes_xts_decrypt_update_arm64_crypto_192_start_1 + ld1 {v0.16b, v1.16b}, [x0], #32 + mov v5.d[0], x11 + mov v5.d[1], x12 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aesd v1.16b, v16.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + extr x10, x12, x11, #63 + aesd v1.16b, v17.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + eor x9, x8, x11, lsl 1 + aesd v1.16b, v18.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aesd v1.16b, v19.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + extr x12, x10, x9, #63 + aesd v1.16b, v20.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + eor x11, x8, x9, lsl 1 + aesd v1.16b, v21.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v22.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v23.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v24.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v25.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v26.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + aesd v1.16b, v27.16b + eor v1.16b, v1.16b, v28.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #2 + st1 {v0.16b, v1.16b}, [x1], #32 +L_aes_xts_decrypt_update_arm64_crypto_192_start_1: + cbz w7, L_aes_xts_decrypt_update_arm64_crypto_192_done + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v4.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + extr x10, x10, x9, #63 + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + eor x9, x8, x9, lsl 1 + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + eor v0.16b, v0.16b, v4.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + st1 {v0.16b}, [x1], #16 +L_aes_xts_decrypt_update_arm64_crypto_192_done: + cbz w2, L_aes_xts_decrypt_update_arm64_crypto_192_partial_done + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + mov v5.d[0], x11 + mov v5.d[1], x12 + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v5.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + eor v0.16b, v0.16b, v5.16b + st1 {v0.2d}, [x5] + add x1, x1, #16 + mov w8, w2 +L_aes_xts_decrypt_update_arm64_crypto_192_start_byte: + ldrb w11, [x5] + ldrb w12, [x0], #1 + strb w11, [x1], #1 + strb w12, [x5], #1 + subs w8, w8, #1 + bgt L_aes_xts_decrypt_update_arm64_crypto_192_start_byte + sub x1, x1, x2 + sub x5, x5, x2 + sub x1, x1, #16 + ld1 {v0.2d}, [x5] + eor v0.16b, v0.16b, v4.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v27.16b + eor v0.16b, v0.16b, v28.16b + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x1] +L_aes_xts_decrypt_update_arm64_crypto_192_partial_done: +#endif /* !NO_AES_192 */ + b L_aes_xts_decrypt_update_arm64_crypto_done + # AES_XTS_256 +L_aes_xts_decrypt_update_arm64_crypto_start_256: +#ifndef NO_AES_256 + ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 + ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x3], #0x40 + ld1 {v28.2d, v29.2d}, [x3], #32 + ld1 {v30.2d}, [x3] + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + and x8, x17, x12, asr 63 + extr x14, x12, x11, #63 + eor x13, x8, x11, lsl 1 + and x8, x17, x14, asr 63 + extr x16, x14, x13, #63 + eor x15, x8, x13, lsl 1 + cmp w7, #4 + blt L_aes_xts_decrypt_update_arm64_crypto_256_start_2 +L_aes_xts_decrypt_update_arm64_crypto_256_start_4: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 + mov v5.d[0], x11 + mov v5.d[1], x12 + mov v6.d[0], x13 + mov v6.d[1], x14 + mov v7.d[0], x15 + mov v7.d[1], x16 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x16, asr 63 + aesd v1.16b, v16.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v16.16b + aesimc v2.16b, v2.16b + extr x10, x16, x15, #63 + aesd v3.16b, v16.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + eor x9, x8, x15, lsl 1 + aesd v1.16b, v17.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v17.16b + aesimc v2.16b, v2.16b + and x8, x17, x10, asr 63 + aesd v3.16b, v17.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + extr x12, x10, x9, #63 + aesd v1.16b, v18.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v18.16b + aesimc v2.16b, v2.16b + eor x11, x8, x9, lsl 1 + aesd v3.16b, v18.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aesd v1.16b, v19.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v19.16b + aesimc v2.16b, v2.16b + extr x14, x12, x11, #63 + aesd v3.16b, v19.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + eor x13, x8, x11, lsl 1 + aesd v1.16b, v20.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v20.16b + aesimc v2.16b, v2.16b + and x8, x17, x14, asr 63 + aesd v3.16b, v20.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + extr x16, x14, x13, #63 + aesd v1.16b, v21.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v21.16b + aesimc v2.16b, v2.16b + eor x15, x8, x13, lsl 1 + aesd v3.16b, v21.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v22.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v22.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v22.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v23.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v23.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v23.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v24.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v24.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v24.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v25.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v25.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v25.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v26.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v26.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v26.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v27.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v27.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v27.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v27.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v28.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v28.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v28.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v28.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + aesd v1.16b, v29.16b + eor v1.16b, v1.16b, v30.16b + aesd v2.16b, v29.16b + eor v2.16b, v2.16b, v30.16b + aesd v3.16b, v29.16b + eor v3.16b, v3.16b, v30.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + cmp w7, #4 + bge L_aes_xts_decrypt_update_arm64_crypto_256_start_4 +L_aes_xts_decrypt_update_arm64_crypto_256_start_2: + cmp w7, #2 + blt L_aes_xts_decrypt_update_arm64_crypto_256_start_1 + ld1 {v0.16b, v1.16b}, [x0], #32 + mov v5.d[0], x11 + mov v5.d[1], x12 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aesd v1.16b, v16.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + extr x10, x12, x11, #63 + aesd v1.16b, v17.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + eor x9, x8, x11, lsl 1 + aesd v1.16b, v18.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aesd v1.16b, v19.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + extr x12, x10, x9, #63 + aesd v1.16b, v20.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + eor x11, x8, x9, lsl 1 + aesd v1.16b, v21.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v22.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v23.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v24.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v25.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v26.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v27.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v27.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v28.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v28.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + aesd v1.16b, v29.16b + eor v1.16b, v1.16b, v30.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #2 + st1 {v0.16b, v1.16b}, [x1], #32 +L_aes_xts_decrypt_update_arm64_crypto_256_start_1: + cbz w7, L_aes_xts_decrypt_update_arm64_crypto_256_done + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v4.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + extr x10, x10, x9, #63 + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + eor x9, x8, x9, lsl 1 + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v27.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v28.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + eor v0.16b, v0.16b, v4.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + st1 {v0.16b}, [x1], #16 +L_aes_xts_decrypt_update_arm64_crypto_256_done: + cbz w2, L_aes_xts_decrypt_update_arm64_crypto_256_partial_done + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + mov v5.d[0], x11 + mov v5.d[1], x12 + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v5.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v27.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v28.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + eor v0.16b, v0.16b, v5.16b + st1 {v0.2d}, [x5] + add x1, x1, #16 + mov w8, w2 +L_aes_xts_decrypt_update_arm64_crypto_256_start_byte: + ldrb w11, [x5] + ldrb w12, [x0], #1 + strb w11, [x1], #1 + strb w12, [x5], #1 + subs w8, w8, #1 + bgt L_aes_xts_decrypt_update_arm64_crypto_256_start_byte + sub x1, x1, x2 + sub x5, x5, x2 + sub x1, x1, #16 + ld1 {v0.2d}, [x5] + eor v0.16b, v0.16b, v4.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v26.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v27.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v28.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v29.16b + eor v0.16b, v0.16b, v30.16b + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x1] +L_aes_xts_decrypt_update_arm64_crypto_256_partial_done: +#endif /* !NO_AES_256 */ + b L_aes_xts_decrypt_update_arm64_crypto_done + # AES_XTS_128 +L_aes_xts_decrypt_update_arm64_crypto_start_128: +#ifndef NO_AES_128 + ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 + ld1 {v24.2d, v25.2d}, [x3], #32 + ld1 {v26.2d}, [x3] + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + and x8, x17, x12, asr 63 + extr x14, x12, x11, #63 + eor x13, x8, x11, lsl 1 + and x8, x17, x14, asr 63 + extr x16, x14, x13, #63 + eor x15, x8, x13, lsl 1 + cmp w7, #4 + blt L_aes_xts_decrypt_update_arm64_crypto_128_start_2 +L_aes_xts_decrypt_update_arm64_crypto_128_start_4: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 + mov v5.d[0], x11 + mov v5.d[1], x12 + mov v6.d[0], x13 + mov v6.d[1], x14 + mov v7.d[0], x15 + mov v7.d[1], x16 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x16, asr 63 + aesd v1.16b, v16.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v16.16b + aesimc v2.16b, v2.16b + extr x10, x16, x15, #63 + aesd v3.16b, v16.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + eor x9, x8, x15, lsl 1 + aesd v1.16b, v17.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v17.16b + aesimc v2.16b, v2.16b + and x8, x17, x10, asr 63 + aesd v3.16b, v17.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + extr x12, x10, x9, #63 + aesd v1.16b, v18.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v18.16b + aesimc v2.16b, v2.16b + eor x11, x8, x9, lsl 1 + aesd v3.16b, v18.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aesd v1.16b, v19.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v19.16b + aesimc v2.16b, v2.16b + extr x14, x12, x11, #63 + aesd v3.16b, v19.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + eor x13, x8, x11, lsl 1 + aesd v1.16b, v20.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v20.16b + aesimc v2.16b, v2.16b + and x8, x17, x14, asr 63 + aesd v3.16b, v20.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + extr x16, x14, x13, #63 + aesd v1.16b, v21.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v21.16b + aesimc v2.16b, v2.16b + eor x15, x8, x13, lsl 1 + aesd v3.16b, v21.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v22.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v22.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v22.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v23.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v23.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v23.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v24.16b + aesimc v1.16b, v1.16b + aesd v2.16b, v24.16b + aesimc v2.16b, v2.16b + aesd v3.16b, v24.16b + aesimc v3.16b, v3.16b + aesd v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + aesd v1.16b, v25.16b + eor v1.16b, v1.16b, v26.16b + aesd v2.16b, v25.16b + eor v2.16b, v2.16b, v26.16b + aesd v3.16b, v25.16b + eor v3.16b, v3.16b, v26.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + cmp w7, #4 + bge L_aes_xts_decrypt_update_arm64_crypto_128_start_4 +L_aes_xts_decrypt_update_arm64_crypto_128_start_2: + cmp w7, #2 + blt L_aes_xts_decrypt_update_arm64_crypto_128_start_1 + ld1 {v0.16b, v1.16b}, [x0], #32 + mov v5.d[0], x11 + mov v5.d[1], x12 + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x12, asr 63 + aesd v1.16b, v16.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + extr x10, x12, x11, #63 + aesd v1.16b, v17.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + eor x9, x8, x11, lsl 1 + aesd v1.16b, v18.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aesd v1.16b, v19.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + extr x12, x10, x9, #63 + aesd v1.16b, v20.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + eor x11, x8, x9, lsl 1 + aesd v1.16b, v21.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v22.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v23.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v1.16b, v24.16b + aesimc v1.16b, v1.16b + aesd v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + aesd v1.16b, v25.16b + eor v1.16b, v1.16b, v26.16b + eor v0.16b, v0.16b, v4.16b + eor v1.16b, v1.16b, v5.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + sub w7, w7, #2 + st1 {v0.16b, v1.16b}, [x1], #32 +L_aes_xts_decrypt_update_arm64_crypto_128_start_1: + cbz w7, L_aes_xts_decrypt_update_arm64_crypto_128_done + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v4.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + and x8, x17, x10, asr 63 + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + extr x10, x10, x9, #63 + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + eor x9, x8, x9, lsl 1 + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + eor v0.16b, v0.16b, v4.16b + mov v4.d[0], x9 + mov v4.d[1], x10 + st1 {v0.16b}, [x1], #16 +L_aes_xts_decrypt_update_arm64_crypto_128_done: + cbz w2, L_aes_xts_decrypt_update_arm64_crypto_128_partial_done + and x8, x17, x10, asr 63 + extr x12, x10, x9, #63 + eor x11, x8, x9, lsl 1 + mov v5.d[0], x11 + mov v5.d[1], x12 + ld1 {v0.16b}, [x0], #16 + eor v0.16b, v0.16b, v5.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + eor v0.16b, v0.16b, v5.16b + st1 {v0.2d}, [x5] + add x1, x1, #16 + mov w8, w2 +L_aes_xts_decrypt_update_arm64_crypto_128_start_byte: + ldrb w11, [x5] + ldrb w12, [x0], #1 + strb w11, [x1], #1 + strb w12, [x5], #1 + subs w8, w8, #1 + bgt L_aes_xts_decrypt_update_arm64_crypto_128_start_byte + sub x1, x1, x2 + sub x5, x5, x2 + sub x1, x1, #16 + ld1 {v0.2d}, [x5] + eor v0.16b, v0.16b, v4.16b + aesd v0.16b, v16.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v17.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v18.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v19.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v20.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v21.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v22.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v23.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v24.16b + aesimc v0.16b, v0.16b + aesd v0.16b, v25.16b + eor v0.16b, v0.16b, v26.16b + eor v0.16b, v0.16b, v4.16b + st1 {v0.16b}, [x1] +L_aes_xts_decrypt_update_arm64_crypto_128_partial_done: +#endif /* !NO_AES_128 */ +L_aes_xts_decrypt_update_arm64_crypto_done: + st1 {v4.16b}, [x4] + ldr x17, [x29, #16] + ldp x29, x30, [sp], #32 + ret +#ifndef __APPLE__ + .size AES_XTS_decrypt_update_AARCH64,.-AES_XTS_decrypt_update_AARCH64 +#endif /* __APPLE__ */ +#endif /* HAVE_AES_DECRYPT */ +#endif /* WOLFSSL_AESXTS_STREAM */ #endif /* WOLFSSL_AES_XTS */ #ifdef WOLFSSL_AESGCM_SIV #ifndef __APPLE__ diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm_c.c b/wolfcrypt/src/port/arm/armv8-aes-asm_c.c index a39acf46bae..3fa591ab771 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-aes-asm_c.c @@ -25281,7 +25281,7 @@ void AES_GCM_encrypt_update_AARCH64(const byte* key, int nr, byte* out, "mov w9, v13.s[3]\n\t" "rev w9, w9\n\t" "cmp %w[nbytes], #32\n\t" - "bcc L_aes_gcm_encrypt_update_arm64_crypto_h_done_%=\n\t" + "b.cc L_aes_gcm_encrypt_update_arm64_crypto_h_done_%=\n\t" /* Square H => H^2 */ "pmull2 v31.1q, v22.2d, v22.2d\n\t" "pmull v30.1q, v22.1d, v22.1d\n\t" @@ -25292,7 +25292,7 @@ void AES_GCM_encrypt_update_AARCH64(const byte* key, int nr, byte* out, "mov v30.d[1], v29.d[0]\n\t" "eor v23.16b, v30.16b, v31.16b\n\t" "cmp %w[nbytes], #0x40\n\t" - "bcc L_aes_gcm_encrypt_update_arm64_crypto_h_done_%=\n\t" + "b.cc L_aes_gcm_encrypt_update_arm64_crypto_h_done_%=\n\t" /* Multiply H and H^2 => H^3 */ "pmull v28.1q, v22.1d, v23.1d\n\t" "pmull2 v29.1q, v22.2d, v23.2d\n\t" @@ -25319,7 +25319,7 @@ void AES_GCM_encrypt_update_AARCH64(const byte* key, int nr, byte* out, "eor v25.16b, v30.16b, v31.16b\n\t" /* Done */ "cmp %w[nbytes], #0x200\n\t" - "bcc L_aes_gcm_encrypt_update_arm64_crypto_h_done_%=\n\t" + "b.cc L_aes_gcm_encrypt_update_arm64_crypto_h_done_%=\n\t" /* Multiply H and H^4 => H^5 */ "pmull v28.1q, v22.1d, v25.1d\n\t" "pmull2 v29.1q, v22.2d, v25.2d\n\t" @@ -29188,7 +29188,7 @@ void AES_GCM_decrypt_update_AARCH64(const byte* key, int nr, byte* out, "mov w9, v13.s[3]\n\t" "rev w9, w9\n\t" "cmp %w[nbytes], #32\n\t" - "bcc L_aes_gcm_decrypt_update_arm64_crypto_h_done_%=\n\t" + "b.cc L_aes_gcm_decrypt_update_arm64_crypto_h_done_%=\n\t" /* Square H => H^2 */ "pmull2 v31.1q, v22.2d, v22.2d\n\t" "pmull v30.1q, v22.1d, v22.1d\n\t" @@ -29199,7 +29199,7 @@ void AES_GCM_decrypt_update_AARCH64(const byte* key, int nr, byte* out, "mov v30.d[1], v29.d[0]\n\t" "eor v23.16b, v30.16b, v31.16b\n\t" "cmp %w[nbytes], #0x40\n\t" - "bcc L_aes_gcm_decrypt_update_arm64_crypto_h_done_%=\n\t" + "b.cc L_aes_gcm_decrypt_update_arm64_crypto_h_done_%=\n\t" /* Multiply H and H^2 => H^3 */ "pmull v28.1q, v22.1d, v23.1d\n\t" "pmull2 v29.1q, v22.2d, v23.2d\n\t" @@ -29226,7 +29226,7 @@ void AES_GCM_decrypt_update_AARCH64(const byte* key, int nr, byte* out, "eor v25.16b, v30.16b, v31.16b\n\t" /* Done */ "cmp %w[nbytes], #0x200\n\t" - "bcc L_aes_gcm_decrypt_update_arm64_crypto_h_done_%=\n\t" + "b.cc L_aes_gcm_decrypt_update_arm64_crypto_h_done_%=\n\t" /* Multiply H and H^4 => H^5 */ "pmull v28.1q, v22.1d, v25.1d\n\t" "pmull2 v29.1q, v22.2d, v25.2d\n\t" @@ -33718,7 +33718,7 @@ void AES_GCM_encrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, "mov w9, v13.s[3]\n\t" "rev w9, w9\n\t" "cmp %w[nbytes], #32\n\t" - "bcc L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done_%=\n\t" + "b.cc L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done_%=\n\t" /* Square H => H^2 */ "pmull2 v31.1q, v22.2d, v22.2d\n\t" "pmull v30.1q, v22.1d, v22.1d\n\t" @@ -33729,7 +33729,7 @@ void AES_GCM_encrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, "mov v30.d[1], v29.d[0]\n\t" "eor v23.16b, v30.16b, v31.16b\n\t" "cmp %w[nbytes], #0x40\n\t" - "bcc L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done_%=\n\t" + "b.cc L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done_%=\n\t" /* Multiply H and H^2 => H^3 */ "pmull v28.1q, v22.1d, v23.1d\n\t" "pmull2 v29.1q, v22.2d, v23.2d\n\t" @@ -33755,7 +33755,7 @@ void AES_GCM_encrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, "eor v25.16b, v30.16b, v31.16b\n\t" /* Done */ "cmp %w[nbytes], #0x200\n\t" - "bcc L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done_%=\n\t" + "b.cc L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done_%=\n\t" /* Multiply H and H^4 => H^5 */ "pmull v28.1q, v22.1d, v25.1d\n\t" "pmull2 v29.1q, v22.2d, v25.2d\n\t" @@ -37540,7 +37540,7 @@ void AES_GCM_decrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, "mov w9, v13.s[3]\n\t" "rev w9, w9\n\t" "cmp %w[nbytes], #32\n\t" - "bcc L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done_%=\n\t" + "b.cc L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done_%=\n\t" /* Square H => H^2 */ "pmull2 v31.1q, v22.2d, v22.2d\n\t" "pmull v30.1q, v22.1d, v22.1d\n\t" @@ -37551,7 +37551,7 @@ void AES_GCM_decrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, "mov v30.d[1], v29.d[0]\n\t" "eor v23.16b, v30.16b, v31.16b\n\t" "cmp %w[nbytes], #0x40\n\t" - "bcc L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done_%=\n\t" + "b.cc L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done_%=\n\t" /* Multiply H and H^2 => H^3 */ "pmull v28.1q, v22.1d, v23.1d\n\t" "pmull2 v29.1q, v22.2d, v23.2d\n\t" @@ -37577,7 +37577,7 @@ void AES_GCM_decrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, "eor v25.16b, v30.16b, v31.16b\n\t" /* Done */ "cmp %w[nbytes], #0x200\n\t" - "bcc L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done_%=\n\t" + "b.cc L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done_%=\n\t" /* Multiply H and H^4 => H^5 */ "pmull v28.1q, v22.1d, v25.1d\n\t" "pmull2 v29.1q, v22.2d, v25.2d\n\t" @@ -43525,6 +43525,1989 @@ void AES_XTS_decrypt_AARCH64(const byte* in, byte* out, word32 sz, } #endif /* HAVE_AES_DECRYPT */ +#ifdef WOLFSSL_AESXTS_STREAM +void AES_XTS_encrypt_update_AARCH64(const byte* in, byte* out, word32 sz, + byte* key, byte* tweak, byte* tmp, int nr) +{ + __asm__ __volatile__ ( + "ld1 {v4.16b}, [%x[tweak]]\n\t" + "mov x9, v4.d[0]\n\t" + "mov x10, v4.d[1]\n\t" + "lsr w7, %w[sz], #4\n\t" + "and %w[sz], %w[sz], #15\n\t" + "mov x17, #0x87\n\t" + "cmp %w[nr], #12\n\t" + "b.lt L_aes_xts_encrypt_update_arm64_crypto_start_128_%=\n\t" + "b.gt L_aes_xts_encrypt_update_arm64_crypto_start_256_%=\n\t" + /* AES_XTS_192 */ +#ifndef NO_AES_192 + "ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [%x[key]], #0x40\n\t" + "ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [%x[key]], #0x40\n\t" + "ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [%x[key]], #0x40\n\t" + "ld1 {v28.2d}, [%x[key]]\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "and x8, x17, x12, asr 63\n\t" + "extr x14, x12, x11, #63\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "and x8, x17, x14, asr 63\n\t" + "extr x16, x14, x13, #63\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "cmp w7, #4\n\t" + "b.lt L_aes_xts_encrypt_update_arm64_crypto_192_start_2_%=\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_192_start_4_%=:\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[in]], #0x40\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "mov v6.d[0], x13\n\t" + "mov v6.d[1], x14\n\t" + "mov v7.d[0], x15\n\t" + "mov v7.d[1], x16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x16, asr 63\n\t" + "aese v1.16b, v16.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v16.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "extr x10, x16, x15, #63\n\t" + "aese v3.16b, v16.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x15, lsl 1\n\t" + "aese v1.16b, v17.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v17.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v3.16b, v17.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aese v1.16b, v18.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v18.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aese v3.16b, v18.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aese v1.16b, v19.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v19.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "extr x14, x12, x11, #63\n\t" + "aese v3.16b, v19.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "aese v1.16b, v20.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v20.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "and x8, x17, x14, asr 63\n\t" + "aese v3.16b, v20.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x16, x14, x13, #63\n\t" + "aese v1.16b, v21.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v21.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "aese v3.16b, v21.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v22.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v22.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v22.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v23.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v23.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v23.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v24.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v24.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v24.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v25.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v25.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v25.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v26.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v26.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v26.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v26.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "aese v1.16b, v27.16b\n\t" + "eor v1.16b, v1.16b, v28.16b\n\t" + "aese v2.16b, v27.16b\n\t" + "eor v2.16b, v2.16b, v28.16b\n\t" + "aese v3.16b, v27.16b\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #4\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" + "cmp w7, #4\n\t" + "b.ge L_aes_xts_encrypt_update_arm64_crypto_192_start_4_%=\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_192_start_2_%=:\n\t" + "cmp w7, #2\n\t" + "b.lt L_aes_xts_encrypt_update_arm64_crypto_192_start_1_%=\n\t" + "ld1 {v0.16b, v1.16b}, [%x[in]], #32\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aese v1.16b, v16.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x10, x12, x11, #63\n\t" + "aese v1.16b, v17.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x11, lsl 1\n\t" + "aese v1.16b, v18.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v1.16b, v19.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aese v1.16b, v20.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aese v1.16b, v21.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v22.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v23.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v24.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v25.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v26.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v26.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "aese v1.16b, v27.16b\n\t" + "eor v1.16b, v1.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #2\n\t" + "st1 {v0.16b, v1.16b}, [%x[out]], #32\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_192_start_1_%=:\n\t" + "cbz w7, L_aes_xts_encrypt_update_arm64_crypto_192_done_%=\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x10, x10, x9, #63\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x9, lsl 1\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v26.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "st1 {v0.16b}, [%x[out]], #16\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_192_done_%=:\n\t" + "cbz %w[sz], " + "L_aes_xts_encrypt_update_arm64_crypto_192_partial_done_%=\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.16b}, [%x[out]], #16\n\t" + "st1 {v0.2d}, [%x[tmp]]\n\t" + "mov w8, %w[sz]\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_192_start_byte_%=:\n\t" + "ldrb w11, [%x[tmp]]\n\t" + "ldrb w12, [%x[in]], #1\n\t" + "strb w11, [%x[out]], #1\n\t" + "strb w12, [%x[tmp]], #1\n\t" + "subs w8, w8, #1\n\t" + "b.gt L_aes_xts_encrypt_update_arm64_crypto_192_start_byte_%=\n\t" + "sub %x[out], %x[out], %x[sz]\n\t" + "sub %x[tmp], %x[tmp], %x[sz]\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.2d}, [%x[tmp]]\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v26.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "st1 {v0.16b}, [%x[out]]\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_192_partial_done_%=:\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_xts_encrypt_update_arm64_crypto_done_%=\n\t" + /* AES_XTS_256 */ + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [%x[key]], #0x40\n\t" + "ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [%x[key]], #0x40\n\t" + "ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [%x[key]], #0x40\n\t" + "ld1 {v28.2d, v29.2d}, [%x[key]], #32\n\t" + "ld1 {v30.2d}, [%x[key]]\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "and x8, x17, x12, asr 63\n\t" + "extr x14, x12, x11, #63\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "and x8, x17, x14, asr 63\n\t" + "extr x16, x14, x13, #63\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "cmp w7, #4\n\t" + "b.lt L_aes_xts_encrypt_update_arm64_crypto_256_start_2_%=\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_256_start_4_%=:\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[in]], #0x40\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "mov v6.d[0], x13\n\t" + "mov v6.d[1], x14\n\t" + "mov v7.d[0], x15\n\t" + "mov v7.d[1], x16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x16, asr 63\n\t" + "aese v1.16b, v16.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v16.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "extr x10, x16, x15, #63\n\t" + "aese v3.16b, v16.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x15, lsl 1\n\t" + "aese v1.16b, v17.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v17.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v3.16b, v17.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aese v1.16b, v18.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v18.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aese v3.16b, v18.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aese v1.16b, v19.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v19.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "extr x14, x12, x11, #63\n\t" + "aese v3.16b, v19.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "aese v1.16b, v20.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v20.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "and x8, x17, x14, asr 63\n\t" + "aese v3.16b, v20.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x16, x14, x13, #63\n\t" + "aese v1.16b, v21.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v21.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "aese v3.16b, v21.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v22.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v22.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v22.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v23.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v23.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v23.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v24.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v24.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v24.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v25.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v25.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v25.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v26.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v26.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v26.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v26.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v27.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v27.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v27.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v27.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v28.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v28.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v28.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v28.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "aese v1.16b, v29.16b\n\t" + "eor v1.16b, v1.16b, v30.16b\n\t" + "aese v2.16b, v29.16b\n\t" + "eor v2.16b, v2.16b, v30.16b\n\t" + "aese v3.16b, v29.16b\n\t" + "eor v3.16b, v3.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #4\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" + "cmp w7, #4\n\t" + "b.ge L_aes_xts_encrypt_update_arm64_crypto_256_start_4_%=\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_256_start_2_%=:\n\t" + "cmp w7, #2\n\t" + "b.lt L_aes_xts_encrypt_update_arm64_crypto_256_start_1_%=\n\t" + "ld1 {v0.16b, v1.16b}, [%x[in]], #32\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aese v1.16b, v16.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x10, x12, x11, #63\n\t" + "aese v1.16b, v17.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x11, lsl 1\n\t" + "aese v1.16b, v18.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v1.16b, v19.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aese v1.16b, v20.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aese v1.16b, v21.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v22.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v23.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v24.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v25.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v26.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v26.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v27.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v27.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v28.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v28.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "aese v1.16b, v29.16b\n\t" + "eor v1.16b, v1.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #2\n\t" + "st1 {v0.16b, v1.16b}, [%x[out]], #32\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_256_start_1_%=:\n\t" + "cbz w7, L_aes_xts_encrypt_update_arm64_crypto_256_done_%=\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x10, x10, x9, #63\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x9, lsl 1\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v26.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v27.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v28.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "st1 {v0.16b}, [%x[out]], #16\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_256_done_%=:\n\t" + "cbz %w[sz], " + "L_aes_xts_encrypt_update_arm64_crypto_256_partial_done_%=\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.16b}, [%x[out]], #16\n\t" + "st1 {v0.2d}, [%x[tmp]]\n\t" + "mov w8, %w[sz]\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_256_start_byte_%=:\n\t" + "ldrb w11, [%x[tmp]]\n\t" + "ldrb w12, [%x[in]], #1\n\t" + "strb w11, [%x[out]], #1\n\t" + "strb w12, [%x[tmp]], #1\n\t" + "subs w8, w8, #1\n\t" + "b.gt L_aes_xts_encrypt_update_arm64_crypto_256_start_byte_%=\n\t" + "sub %x[out], %x[out], %x[sz]\n\t" + "sub %x[tmp], %x[tmp], %x[sz]\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.2d}, [%x[tmp]]\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v26.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v27.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v28.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "st1 {v0.16b}, [%x[out]]\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_256_partial_done_%=:\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_xts_encrypt_update_arm64_crypto_done_%=\n\t" + /* AES_XTS_128 */ + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [%x[key]], #0x40\n\t" + "ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [%x[key]], #0x40\n\t" + "ld1 {v24.2d, v25.2d}, [%x[key]], #32\n\t" + "ld1 {v26.2d}, [%x[key]]\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "and x8, x17, x12, asr 63\n\t" + "extr x14, x12, x11, #63\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "and x8, x17, x14, asr 63\n\t" + "extr x16, x14, x13, #63\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "cmp w7, #4\n\t" + "b.lt L_aes_xts_encrypt_update_arm64_crypto_128_start_2_%=\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_128_start_4_%=:\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[in]], #0x40\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "mov v6.d[0], x13\n\t" + "mov v6.d[1], x14\n\t" + "mov v7.d[0], x15\n\t" + "mov v7.d[1], x16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x16, asr 63\n\t" + "aese v1.16b, v16.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v16.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "extr x10, x16, x15, #63\n\t" + "aese v3.16b, v16.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x15, lsl 1\n\t" + "aese v1.16b, v17.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v17.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v3.16b, v17.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aese v1.16b, v18.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v18.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aese v3.16b, v18.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aese v1.16b, v19.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v19.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "extr x14, x12, x11, #63\n\t" + "aese v3.16b, v19.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "aese v1.16b, v20.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v20.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "and x8, x17, x14, asr 63\n\t" + "aese v3.16b, v20.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x16, x14, x13, #63\n\t" + "aese v1.16b, v21.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v21.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "aese v3.16b, v21.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v22.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v22.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v22.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v23.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v23.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v23.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v24.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v2.16b, v24.16b\n\t" + "aesmc v2.16b, v2.16b\n\t" + "aese v3.16b, v24.16b\n\t" + "aesmc v3.16b, v3.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "aese v1.16b, v25.16b\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "aese v2.16b, v25.16b\n\t" + "eor v2.16b, v2.16b, v26.16b\n\t" + "aese v3.16b, v25.16b\n\t" + "eor v3.16b, v3.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #4\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" + "cmp w7, #4\n\t" + "b.ge L_aes_xts_encrypt_update_arm64_crypto_128_start_4_%=\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_128_start_2_%=:\n\t" + "cmp w7, #2\n\t" + "b.lt L_aes_xts_encrypt_update_arm64_crypto_128_start_1_%=\n\t" + "ld1 {v0.16b, v1.16b}, [%x[in]], #32\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aese v1.16b, v16.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x10, x12, x11, #63\n\t" + "aese v1.16b, v17.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x11, lsl 1\n\t" + "aese v1.16b, v18.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v1.16b, v19.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aese v1.16b, v20.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aese v1.16b, v21.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v22.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v23.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v1.16b, v24.16b\n\t" + "aesmc v1.16b, v1.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "aese v1.16b, v25.16b\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #2\n\t" + "st1 {v0.16b, v1.16b}, [%x[out]], #32\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_128_start_1_%=:\n\t" + "cbz w7, L_aes_xts_encrypt_update_arm64_crypto_128_done_%=\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "extr x10, x10, x9, #63\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "eor x9, x8, x9, lsl 1\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "st1 {v0.16b}, [%x[out]], #16\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_128_done_%=:\n\t" + "cbz %w[sz], " + "L_aes_xts_encrypt_update_arm64_crypto_128_partial_done_%=\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.16b}, [%x[out]], #16\n\t" + "st1 {v0.2d}, [%x[tmp]]\n\t" + "mov w8, %w[sz]\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_128_start_byte_%=:\n\t" + "ldrb w11, [%x[tmp]]\n\t" + "ldrb w12, [%x[in]], #1\n\t" + "strb w11, [%x[out]], #1\n\t" + "strb w12, [%x[tmp]], #1\n\t" + "subs w8, w8, #1\n\t" + "b.gt L_aes_xts_encrypt_update_arm64_crypto_128_start_byte_%=\n\t" + "sub %x[out], %x[out], %x[sz]\n\t" + "sub %x[tmp], %x[tmp], %x[sz]\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.2d}, [%x[tmp]]\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aese v0.16b, v16.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v17.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v18.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v19.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v20.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v21.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v22.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v23.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v24.16b\n\t" + "aesmc v0.16b, v0.16b\n\t" + "aese v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "st1 {v0.16b}, [%x[out]]\n\t" + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_128_partial_done_%=:\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_xts_encrypt_update_arm64_crypto_done_%=:\n\t" + "st1 {v4.16b}, [%x[tweak]]\n\t" + : [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), [tmp] "+r" (tmp), + [nr] "+r" (nr) + : [in] "r" (in), [tweak] "r" (tweak) + : "memory", "cc", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", + "x15", "x16", "x17", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29", "v30" + ); +} + +#ifdef HAVE_AES_DECRYPT +void AES_XTS_decrypt_update_AARCH64(const byte* in, byte* out, word32 sz, + byte* key, byte* tweak, byte* tmp, int nr) +{ + __asm__ __volatile__ ( + "ld1 {v4.16b}, [%x[tweak]]\n\t" + "mov x9, v4.d[0]\n\t" + "mov x10, v4.d[1]\n\t" + "lsr w7, %w[sz], #4\n\t" + "ands %w[sz], %w[sz], #15\n\t" + "mov x17, #0x87\n\t" + "cset w8, ne\n\t" + "sub w7, w7, w8\n\t" + "cmp %w[nr], #12\n\t" + "b.lt L_aes_xts_decrypt_update_arm64_crypto_start_128_%=\n\t" + "b.gt L_aes_xts_decrypt_update_arm64_crypto_start_256_%=\n\t" + /* AES_XTS_192 */ +#ifndef NO_AES_192 + "ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [%x[key]], #0x40\n\t" + "ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [%x[key]], #0x40\n\t" + "ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [%x[key]], #0x40\n\t" + "ld1 {v28.2d}, [%x[key]]\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "and x8, x17, x12, asr 63\n\t" + "extr x14, x12, x11, #63\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "and x8, x17, x14, asr 63\n\t" + "extr x16, x14, x13, #63\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "cmp w7, #4\n\t" + "b.lt L_aes_xts_decrypt_update_arm64_crypto_192_start_2_%=\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_192_start_4_%=:\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[in]], #0x40\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "mov v6.d[0], x13\n\t" + "mov v6.d[1], x14\n\t" + "mov v7.d[0], x15\n\t" + "mov v7.d[1], x16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x16, asr 63\n\t" + "aesd v1.16b, v16.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v16.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "extr x10, x16, x15, #63\n\t" + "aesd v3.16b, v16.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x15, lsl 1\n\t" + "aesd v1.16b, v17.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v17.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v3.16b, v17.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aesd v1.16b, v18.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v18.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aesd v3.16b, v18.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aesd v1.16b, v19.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v19.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "extr x14, x12, x11, #63\n\t" + "aesd v3.16b, v19.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "aesd v1.16b, v20.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v20.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "and x8, x17, x14, asr 63\n\t" + "aesd v3.16b, v20.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x16, x14, x13, #63\n\t" + "aesd v1.16b, v21.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v21.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "aesd v3.16b, v21.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v22.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v22.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v22.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v23.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v23.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v23.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v24.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v24.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v24.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v25.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v25.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v25.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v26.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v26.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v26.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "aesd v1.16b, v27.16b\n\t" + "eor v1.16b, v1.16b, v28.16b\n\t" + "aesd v2.16b, v27.16b\n\t" + "eor v2.16b, v2.16b, v28.16b\n\t" + "aesd v3.16b, v27.16b\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #4\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" + "cmp w7, #4\n\t" + "b.ge L_aes_xts_decrypt_update_arm64_crypto_192_start_4_%=\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_192_start_2_%=:\n\t" + "cmp w7, #2\n\t" + "b.lt L_aes_xts_decrypt_update_arm64_crypto_192_start_1_%=\n\t" + "ld1 {v0.16b, v1.16b}, [%x[in]], #32\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aesd v1.16b, v16.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x10, x12, x11, #63\n\t" + "aesd v1.16b, v17.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x11, lsl 1\n\t" + "aesd v1.16b, v18.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v1.16b, v19.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aesd v1.16b, v20.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aesd v1.16b, v21.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v22.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v23.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v24.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v25.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v26.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "aesd v1.16b, v27.16b\n\t" + "eor v1.16b, v1.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #2\n\t" + "st1 {v0.16b, v1.16b}, [%x[out]], #32\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_192_start_1_%=:\n\t" + "cbz w7, L_aes_xts_decrypt_update_arm64_crypto_192_done_%=\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x10, x10, x9, #63\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x9, lsl 1\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "st1 {v0.16b}, [%x[out]], #16\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_192_done_%=:\n\t" + "cbz %w[sz], " + "L_aes_xts_decrypt_update_arm64_crypto_192_partial_done_%=\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v5.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v5.16b\n\t" + "st1 {v0.2d}, [%x[tmp]]\n\t" + "add %x[out], %x[out], #16\n\t" + "mov w8, %w[sz]\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_192_start_byte_%=:\n\t" + "ldrb w11, [%x[tmp]]\n\t" + "ldrb w12, [%x[in]], #1\n\t" + "strb w11, [%x[out]], #1\n\t" + "strb w12, [%x[tmp]], #1\n\t" + "subs w8, w8, #1\n\t" + "b.gt L_aes_xts_decrypt_update_arm64_crypto_192_start_byte_%=\n\t" + "sub %x[out], %x[out], %x[sz]\n\t" + "sub %x[tmp], %x[tmp], %x[sz]\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.2d}, [%x[tmp]]\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "eor v0.16b, v0.16b, v28.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "st1 {v0.16b}, [%x[out]]\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_192_partial_done_%=:\n\t" +#endif /* !NO_AES_192 */ + "b L_aes_xts_decrypt_update_arm64_crypto_done_%=\n\t" + /* AES_XTS_256 */ + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_start_256_%=:\n\t" +#ifndef NO_AES_256 + "ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [%x[key]], #0x40\n\t" + "ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [%x[key]], #0x40\n\t" + "ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [%x[key]], #0x40\n\t" + "ld1 {v28.2d, v29.2d}, [%x[key]], #32\n\t" + "ld1 {v30.2d}, [%x[key]]\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "and x8, x17, x12, asr 63\n\t" + "extr x14, x12, x11, #63\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "and x8, x17, x14, asr 63\n\t" + "extr x16, x14, x13, #63\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "cmp w7, #4\n\t" + "b.lt L_aes_xts_decrypt_update_arm64_crypto_256_start_2_%=\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_256_start_4_%=:\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[in]], #0x40\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "mov v6.d[0], x13\n\t" + "mov v6.d[1], x14\n\t" + "mov v7.d[0], x15\n\t" + "mov v7.d[1], x16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x16, asr 63\n\t" + "aesd v1.16b, v16.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v16.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "extr x10, x16, x15, #63\n\t" + "aesd v3.16b, v16.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x15, lsl 1\n\t" + "aesd v1.16b, v17.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v17.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v3.16b, v17.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aesd v1.16b, v18.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v18.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aesd v3.16b, v18.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aesd v1.16b, v19.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v19.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "extr x14, x12, x11, #63\n\t" + "aesd v3.16b, v19.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "aesd v1.16b, v20.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v20.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "and x8, x17, x14, asr 63\n\t" + "aesd v3.16b, v20.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x16, x14, x13, #63\n\t" + "aesd v1.16b, v21.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v21.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "aesd v3.16b, v21.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v22.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v22.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v22.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v23.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v23.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v23.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v24.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v24.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v24.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v25.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v25.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v25.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v26.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v26.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v26.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v27.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v27.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v27.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v28.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v28.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v28.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v28.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "aesd v1.16b, v29.16b\n\t" + "eor v1.16b, v1.16b, v30.16b\n\t" + "aesd v2.16b, v29.16b\n\t" + "eor v2.16b, v2.16b, v30.16b\n\t" + "aesd v3.16b, v29.16b\n\t" + "eor v3.16b, v3.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #4\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" + "cmp w7, #4\n\t" + "b.ge L_aes_xts_decrypt_update_arm64_crypto_256_start_4_%=\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_256_start_2_%=:\n\t" + "cmp w7, #2\n\t" + "b.lt L_aes_xts_decrypt_update_arm64_crypto_256_start_1_%=\n\t" + "ld1 {v0.16b, v1.16b}, [%x[in]], #32\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aesd v1.16b, v16.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x10, x12, x11, #63\n\t" + "aesd v1.16b, v17.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x11, lsl 1\n\t" + "aesd v1.16b, v18.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v1.16b, v19.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aesd v1.16b, v20.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aesd v1.16b, v21.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v22.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v23.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v24.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v25.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v26.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v27.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v28.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v28.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "aesd v1.16b, v29.16b\n\t" + "eor v1.16b, v1.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #2\n\t" + "st1 {v0.16b, v1.16b}, [%x[out]], #32\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_256_start_1_%=:\n\t" + "cbz w7, L_aes_xts_decrypt_update_arm64_crypto_256_done_%=\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x10, x10, x9, #63\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x9, lsl 1\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v28.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "st1 {v0.16b}, [%x[out]], #16\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_256_done_%=:\n\t" + "cbz %w[sz], " + "L_aes_xts_decrypt_update_arm64_crypto_256_partial_done_%=\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v5.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v28.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v5.16b\n\t" + "st1 {v0.2d}, [%x[tmp]]\n\t" + "add %x[out], %x[out], #16\n\t" + "mov w8, %w[sz]\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_256_start_byte_%=:\n\t" + "ldrb w11, [%x[tmp]]\n\t" + "ldrb w12, [%x[in]], #1\n\t" + "strb w11, [%x[out]], #1\n\t" + "strb w12, [%x[tmp]], #1\n\t" + "subs w8, w8, #1\n\t" + "b.gt L_aes_xts_decrypt_update_arm64_crypto_256_start_byte_%=\n\t" + "sub %x[out], %x[out], %x[sz]\n\t" + "sub %x[tmp], %x[tmp], %x[sz]\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.2d}, [%x[tmp]]\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v26.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v27.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v28.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v29.16b\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "st1 {v0.16b}, [%x[out]]\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_256_partial_done_%=:\n\t" +#endif /* !NO_AES_256 */ + "b L_aes_xts_decrypt_update_arm64_crypto_done_%=\n\t" + /* AES_XTS_128 */ + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_start_128_%=:\n\t" +#ifndef NO_AES_128 + "ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [%x[key]], #0x40\n\t" + "ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [%x[key]], #0x40\n\t" + "ld1 {v24.2d, v25.2d}, [%x[key]], #32\n\t" + "ld1 {v26.2d}, [%x[key]]\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "and x8, x17, x12, asr 63\n\t" + "extr x14, x12, x11, #63\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "and x8, x17, x14, asr 63\n\t" + "extr x16, x14, x13, #63\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "cmp w7, #4\n\t" + "b.lt L_aes_xts_decrypt_update_arm64_crypto_128_start_2_%=\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_128_start_4_%=:\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[in]], #0x40\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "mov v6.d[0], x13\n\t" + "mov v6.d[1], x14\n\t" + "mov v7.d[0], x15\n\t" + "mov v7.d[1], x16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x16, asr 63\n\t" + "aesd v1.16b, v16.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v16.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "extr x10, x16, x15, #63\n\t" + "aesd v3.16b, v16.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x15, lsl 1\n\t" + "aesd v1.16b, v17.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v17.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v3.16b, v17.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aesd v1.16b, v18.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v18.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aesd v3.16b, v18.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aesd v1.16b, v19.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v19.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "extr x14, x12, x11, #63\n\t" + "aesd v3.16b, v19.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x13, x8, x11, lsl 1\n\t" + "aesd v1.16b, v20.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v20.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "and x8, x17, x14, asr 63\n\t" + "aesd v3.16b, v20.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x16, x14, x13, #63\n\t" + "aesd v1.16b, v21.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v21.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "eor x15, x8, x13, lsl 1\n\t" + "aesd v3.16b, v21.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v22.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v22.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v22.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v23.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v23.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v23.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v24.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v2.16b, v24.16b\n\t" + "aesimc v2.16b, v2.16b\n\t" + "aesd v3.16b, v24.16b\n\t" + "aesimc v3.16b, v3.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "aesd v1.16b, v25.16b\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "aesd v2.16b, v25.16b\n\t" + "eor v2.16b, v2.16b, v26.16b\n\t" + "aesd v3.16b, v25.16b\n\t" + "eor v3.16b, v3.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "eor v2.16b, v2.16b, v6.16b\n\t" + "eor v3.16b, v3.16b, v7.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #4\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" + "cmp w7, #4\n\t" + "b.ge L_aes_xts_decrypt_update_arm64_crypto_128_start_4_%=\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_128_start_2_%=:\n\t" + "cmp w7, #2\n\t" + "b.lt L_aes_xts_decrypt_update_arm64_crypto_128_start_1_%=\n\t" + "ld1 {v0.16b, v1.16b}, [%x[in]], #32\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x12, asr 63\n\t" + "aesd v1.16b, v16.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x10, x12, x11, #63\n\t" + "aesd v1.16b, v17.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x11, lsl 1\n\t" + "aesd v1.16b, v18.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v1.16b, v19.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x12, x10, x9, #63\n\t" + "aesd v1.16b, v20.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "aesd v1.16b, v21.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v22.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v23.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v1.16b, v24.16b\n\t" + "aesimc v1.16b, v1.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "aesd v1.16b, v25.16b\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "eor v1.16b, v1.16b, v5.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "sub w7, w7, #2\n\t" + "st1 {v0.16b, v1.16b}, [%x[out]], #32\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_128_start_1_%=:\n\t" + "cbz w7, L_aes_xts_decrypt_update_arm64_crypto_128_done_%=\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "and x8, x17, x10, asr 63\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "extr x10, x10, x9, #63\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "eor x9, x8, x9, lsl 1\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "mov v4.d[0], x9\n\t" + "mov v4.d[1], x10\n\t" + "st1 {v0.16b}, [%x[out]], #16\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_128_done_%=:\n\t" + "cbz %w[sz], " + "L_aes_xts_decrypt_update_arm64_crypto_128_partial_done_%=\n\t" + "and x8, x17, x10, asr 63\n\t" + "extr x12, x10, x9, #63\n\t" + "eor x11, x8, x9, lsl 1\n\t" + "mov v5.d[0], x11\n\t" + "mov v5.d[1], x12\n\t" + "ld1 {v0.16b}, [%x[in]], #16\n\t" + "eor v0.16b, v0.16b, v5.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v5.16b\n\t" + "st1 {v0.2d}, [%x[tmp]]\n\t" + "add %x[out], %x[out], #16\n\t" + "mov w8, %w[sz]\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_128_start_byte_%=:\n\t" + "ldrb w11, [%x[tmp]]\n\t" + "ldrb w12, [%x[in]], #1\n\t" + "strb w11, [%x[out]], #1\n\t" + "strb w12, [%x[tmp]], #1\n\t" + "subs w8, w8, #1\n\t" + "b.gt L_aes_xts_decrypt_update_arm64_crypto_128_start_byte_%=\n\t" + "sub %x[out], %x[out], %x[sz]\n\t" + "sub %x[tmp], %x[tmp], %x[sz]\n\t" + "sub %x[out], %x[out], #16\n\t" + "ld1 {v0.2d}, [%x[tmp]]\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "aesd v0.16b, v16.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v17.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v18.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v19.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v20.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v21.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v22.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v23.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v24.16b\n\t" + "aesimc v0.16b, v0.16b\n\t" + "aesd v0.16b, v25.16b\n\t" + "eor v0.16b, v0.16b, v26.16b\n\t" + "eor v0.16b, v0.16b, v4.16b\n\t" + "st1 {v0.16b}, [%x[out]]\n\t" + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_128_partial_done_%=:\n\t" +#endif /* !NO_AES_128 */ + "\n" + "L_aes_xts_decrypt_update_arm64_crypto_done_%=:\n\t" + "st1 {v4.16b}, [%x[tweak]]\n\t" + : [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), [tmp] "+r" (tmp), + [nr] "+r" (nr) + : [in] "r" (in), [tweak] "r" (tweak) + : "memory", "cc", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", + "x15", "x16", "x17", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29", "v30" + ); +} + +#endif /* HAVE_AES_DECRYPT */ +#endif /* WOLFSSL_AESXTS_STREAM */ #endif /* WOLFSSL_AES_XTS */ #ifdef WOLFSSL_AESGCM_SIV void AES_GCMSIV_polyval_pmull(unsigned char* s, const unsigned char* h, @@ -44940,7 +46923,7 @@ void AES_ECB_encrypt_NEON(const unsigned char* in, unsigned char* out, "ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [%[te]], #0x40\n\t" "ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [%[te]]\n\t" "cmp %x[len], #0x40\n\t" - "bcc L_AES_ECB_encrypt_NEON_start_2_%=\n\t" + "b.cc L_AES_ECB_encrypt_NEON_start_2_%=\n\t" "\n" "L_AES_ECB_encrypt_NEON_loop_4_%=:\n\t" "mov x8, %x[ks]\n\t" @@ -45346,7 +47329,7 @@ void AES_ECB_encrypt_NEON(const unsigned char* in, unsigned char* out, "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "sub %x[len], %x[len], #0x40\n\t" "cmp %x[len], #0x40\n\t" - "bcs L_AES_ECB_encrypt_NEON_loop_4_%=\n\t" + "b.cs L_AES_ECB_encrypt_NEON_loop_4_%=\n\t" "\n" "L_AES_ECB_encrypt_NEON_start_2_%=:\n\t" "movi v12.16b, #0x40\n\t" @@ -45355,7 +47338,7 @@ void AES_ECB_encrypt_NEON(const unsigned char* in, unsigned char* out, "movi v15.16b, #27\n\t" "cmp %x[len], #16\n\t" "b.eq L_AES_ECB_encrypt_NEON_start_1_%=\n\t" - "bcc L_AES_ECB_encrypt_NEON_data_done_%=\n\t" + "b.cc L_AES_ECB_encrypt_NEON_data_done_%=\n\t" "\n" "L_AES_ECB_encrypt_NEON_loop_2_%=:\n\t" "mov x8, %x[ks]\n\t" @@ -45853,7 +47836,7 @@ void AES_CTR_encrypt_NEON(const unsigned char* in, unsigned char* out, "mov x10, v8.d[1]\n\t" "mov x11, v8.d[0]\n\t" "cmp %x[len], #0x40\n\t" - "bcc L_AES_CTR_encrypt_NEON_start_2_%=\n\t" + "b.cc L_AES_CTR_encrypt_NEON_start_2_%=\n\t" "\n" "L_AES_CTR_encrypt_NEON_loop_4_%=:\n\t" "mov x9, %x[ks]\n\t" @@ -46282,7 +48265,7 @@ void AES_CTR_encrypt_NEON(const unsigned char* in, unsigned char* out, "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "sub %x[len], %x[len], #0x40\n\t" "cmp %x[len], #0x40\n\t" - "bcs L_AES_CTR_encrypt_NEON_loop_4_%=\n\t" + "b.cs L_AES_CTR_encrypt_NEON_loop_4_%=\n\t" "mov v2.d[1], x10\n\t" "mov v2.d[0], x11\n\t" "rev64 v2.4s, v2.4s\n\t" @@ -46294,7 +48277,7 @@ void AES_CTR_encrypt_NEON(const unsigned char* in, unsigned char* out, "movi v15.16b, #27\n\t" "cmp %x[len], #16\n\t" "b.eq L_AES_CTR_encrypt_NEON_start_1_%=\n\t" - "bcc L_AES_CTR_encrypt_NEON_data_done_%=\n\t" + "b.cc L_AES_CTR_encrypt_NEON_data_done_%=\n\t" "\n" "L_AES_CTR_encrypt_NEON_loop_2_%=:\n\t" "mov x9, %x[ks]\n\t" @@ -46704,7 +48687,7 @@ void AES_ECB_decrypt_NEON(const unsigned char* in, unsigned char* out, "ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [%[td]], #0x40\n\t" "ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [%[td]]\n\t" "cmp %x[len], #0x40\n\t" - "bcc L_AES_ECB_decrypt_NEON_start_2_%=\n\t" + "b.cc L_AES_ECB_decrypt_NEON_start_2_%=\n\t" "\n" "L_AES_ECB_decrypt_NEON_loop_4_%=:\n\t" "mov x8, %x[ks]\n\t" @@ -47257,12 +49240,12 @@ void AES_ECB_decrypt_NEON(const unsigned char* in, unsigned char* out, "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "sub %x[len], %x[len], #0x40\n\t" "cmp %x[len], #0x40\n\t" - "bcs L_AES_ECB_decrypt_NEON_loop_4_%=\n\t" + "b.cs L_AES_ECB_decrypt_NEON_loop_4_%=\n\t" "\n" "L_AES_ECB_decrypt_NEON_start_2_%=:\n\t" "cmp %x[len], #16\n\t" "b.eq L_AES_ECB_decrypt_NEON_start_1_%=\n\t" - "bcc L_AES_ECB_decrypt_NEON_data_done_%=\n\t" + "b.cc L_AES_ECB_decrypt_NEON_data_done_%=\n\t" "\n" "L_AES_ECB_decrypt_NEON_loop_2_%=:\n\t" "mov x8, %x[ks]\n\t" @@ -47743,7 +49726,7 @@ void AES_CBC_decrypt_NEON(const unsigned char* in, unsigned char* out, "ld1 {v3.2d}, [%x[iv]]\n\t" "add x10, x29, #16\n\t" "cmp %x[len], #0x40\n\t" - "bcc L_AES_CBC_decrypt_NEON_start_2_%=\n\t" + "b.cc L_AES_CBC_decrypt_NEON_start_2_%=\n\t" "\n" "L_AES_CBC_decrypt_NEON_loop_4_%=:\n\t" "mov x9, %x[ks]\n\t" @@ -48304,12 +50287,12 @@ void AES_CBC_decrypt_NEON(const unsigned char* in, unsigned char* out, "st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [%x[out]], #0x40\n\t" "sub %x[len], %x[len], #0x40\n\t" "cmp %x[len], #0x40\n\t" - "bcs L_AES_CBC_decrypt_NEON_loop_4_%=\n\t" + "b.cs L_AES_CBC_decrypt_NEON_loop_4_%=\n\t" "\n" "L_AES_CBC_decrypt_NEON_start_2_%=:\n\t" "cmp %x[len], #16\n\t" "b.eq L_AES_CBC_decrypt_NEON_start_1_%=\n\t" - "bcc L_AES_CBC_decrypt_NEON_data_done_%=\n\t" + "b.cc L_AES_CBC_decrypt_NEON_data_done_%=\n\t" "\n" "L_AES_CBC_decrypt_NEON_loop_2_%=:\n\t" "mov x9, %x[ks]\n\t" @@ -48611,7 +50594,7 @@ void AES_CBC_decrypt_NEON(const unsigned char* in, unsigned char* out, "st1 {v4.16b, v5.16b}, [%x[out]], #32\n\t" "sub %x[len], %x[len], #32\n\t" "cmp %x[len], #32\n\t" - "bcs L_AES_CBC_decrypt_NEON_loop_2_%=\n\t" + "b.cs L_AES_CBC_decrypt_NEON_loop_2_%=\n\t" "cmp %x[len], #0\n\t" "b.eq L_AES_CBC_decrypt_NEON_data_done_%=\n\t" "\n" @@ -49134,7 +51117,7 @@ void AES_GCM_encrypt_NEON(const unsigned char* in, unsigned char* out, "rev32 v2.16b, v2.16b\n\t" "mov w6, v2.s[3]\n\t" "cmp %x[len], #0x40\n\t" - "bcc L_AES_GCM_encrypt_NEON_start_2_%=\n\t" + "b.cc L_AES_GCM_encrypt_NEON_start_2_%=\n\t" "mov x7, v2.d[0]\n\t" "mov x8, v2.d[1]\n\t" "\n" @@ -49552,7 +51535,7 @@ void AES_GCM_encrypt_NEON(const unsigned char* in, unsigned char* out, "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "sub %x[len], %x[len], #0x40\n\t" "cmp %x[len], #0x40\n\t" - "bcs L_AES_GCM_encrypt_NEON_loop_4_%=\n\t" + "b.cs L_AES_GCM_encrypt_NEON_loop_4_%=\n\t" "mov v2.d[0], x7\n\t" "mov v2.d[1], x8\n\t" "mov v2.s[3], w6\n\t" @@ -49564,7 +51547,7 @@ void AES_GCM_encrypt_NEON(const unsigned char* in, unsigned char* out, "movi v15.16b, #27\n\t" "cmp %x[len], #16\n\t" "b.eq L_AES_GCM_encrypt_NEON_start_1_%=\n\t" - "bcc L_AES_GCM_encrypt_NEON_data_done_%=\n\t" + "b.cc L_AES_GCM_encrypt_NEON_data_done_%=\n\t" "\n" "L_AES_GCM_encrypt_NEON_loop_2_%=:\n\t" "mov x12, %x[ks]\n\t" @@ -50035,7 +52018,7 @@ void AES_XTS_encrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "mov x8, v2.d[0]\n\t" "mov x9, v2.d[1]\n\t" "cmp %w[sz], #0x40\n\t" - "bcc L_AES_XTS_encrypt_NEON_start_2_%=\n\t" + "b.cc L_AES_XTS_encrypt_NEON_start_2_%=\n\t" "\n" "L_AES_XTS_encrypt_NEON_loop_4_%=:\n\t" "mov x22, %x[key]\n\t" @@ -50476,7 +52459,7 @@ void AES_XTS_encrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "eor x8, x16, x14, lsl 1\n\t" "sub %w[sz], %w[sz], #0x40\n\t" "cmp %w[sz], #0x40\n\t" - "bcs L_AES_XTS_encrypt_NEON_loop_4_%=\n\t" + "b.cs L_AES_XTS_encrypt_NEON_loop_4_%=\n\t" "movi v12.16b, #0x40\n\t" "movi v13.16b, #0x80\n\t" "movi v14.16b, #0xc0\n\t" @@ -50484,7 +52467,7 @@ void AES_XTS_encrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "\n" "L_AES_XTS_encrypt_NEON_start_2_%=:\n\t" "cmp %w[sz], #32\n\t" - "bcc L_AES_XTS_encrypt_NEON_start_1_%=\n\t" + "b.cc L_AES_XTS_encrypt_NEON_start_1_%=\n\t" "mov x22, %x[key]\n\t" "ld1 {v0.16b, v1.16b}, [%x[in]], #32\n\t" "ld1 {v4.16b}, [x22], #16\n\t" @@ -50714,7 +52697,7 @@ void AES_XTS_encrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "mov v2.d[0], x8\n\t" "mov v2.d[1], x9\n\t" "cmp %w[sz], #16\n\t" - "bcc L_AES_XTS_encrypt_NEON_start_partial_%=\n\t" + "b.cc L_AES_XTS_encrypt_NEON_start_partial_%=\n\t" "mov x22, %x[key]\n\t" "ld1 {v0.16b}, [%x[in]], #16\n\t" "ld1 {v4.2d}, [x22], #16\n\t" @@ -51108,7 +53091,7 @@ void AES_XTS_decrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [%[td]]\n\t" "ld1 {v3.2d}, [%[invshuffle]]\n\t" "cmp %w[sz], #0x40\n\t" - "bcc L_AES_XTS_decrypt_NEON_start_2_%=\n\t" + "b.cc L_AES_XTS_decrypt_NEON_start_2_%=\n\t" "\n" "L_AES_XTS_decrypt_NEON_loop_4_%=:\n\t" "mov x25, %x[key]\n\t" @@ -51696,7 +53679,7 @@ void AES_XTS_decrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "eor x8, x16, x14, lsl 1\n\t" "sub %w[sz], %w[sz], #0x40\n\t" "cmp %w[sz], #0x40\n\t" - "bcs L_AES_XTS_decrypt_NEON_loop_4_%=\n\t" + "b.cs L_AES_XTS_decrypt_NEON_loop_4_%=\n\t" "movi v12.16b, #0x40\n\t" "movi v13.16b, #0x80\n\t" "movi v14.16b, #0xc0\n\t" @@ -51704,7 +53687,7 @@ void AES_XTS_decrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "\n" "L_AES_XTS_decrypt_NEON_start_2_%=:\n\t" "cmp %w[sz], #32\n\t" - "bcc L_AES_XTS_decrypt_NEON_start_1_%=\n\t" + "b.cc L_AES_XTS_decrypt_NEON_start_1_%=\n\t" "mov x25, %x[key]\n\t" "ld1 {v0.16b, v1.16b}, [%x[in]], #32\n\t" "ld1 {v4.16b}, [x25], #16\n\t" @@ -52021,7 +54004,7 @@ void AES_XTS_decrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "mov v2.d[0], x8\n\t" "mov v2.d[1], x9\n\t" "cmp %w[sz], #16\n\t" - "bcc L_AES_XTS_decrypt_NEON_start_partial_%=\n\t" + "b.cc L_AES_XTS_decrypt_NEON_start_partial_%=\n\t" "mov x25, %x[key]\n\t" "ld1 {v0.16b}, [%x[in]], #16\n\t" "ld1 {v4.2d}, [x25], #16\n\t" @@ -52888,7 +54871,7 @@ void AES_GCMSIV_ctr_neon(const unsigned char* in, unsigned char* out, "ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [%[te]]\n\t" "ldr w10, [%x[ctr]]\n\t" "cmp %x[length], #0x40\n\t" - "bcc L_AES_GCMSIV_ctr_neon_start_2_%=\n\t" + "b.cc L_AES_GCMSIV_ctr_neon_start_2_%=\n\t" "\n" "L_AES_GCMSIV_ctr_neon_loop_4_%=:\n\t" "mov x9, %x[KS]\n\t" @@ -53310,7 +55293,7 @@ void AES_GCMSIV_ctr_neon(const unsigned char* in, unsigned char* out, "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "sub %x[length], %x[length], #0x40\n\t" "cmp %x[length], #0x40\n\t" - "bcs L_AES_GCMSIV_ctr_neon_loop_4_%=\n\t" + "b.cs L_AES_GCMSIV_ctr_neon_loop_4_%=\n\t" "\n" "L_AES_GCMSIV_ctr_neon_start_2_%=:\n\t" "movi v12.16b, #0x40\n\t" @@ -53319,7 +55302,7 @@ void AES_GCMSIV_ctr_neon(const unsigned char* in, unsigned char* out, "movi v15.16b, #27\n\t" "cmp %x[length], #16\n\t" "b.eq L_AES_GCMSIV_ctr_neon_start_1_%=\n\t" - "bcc L_AES_GCMSIV_ctr_neon_data_done_%=\n\t" + "b.cc L_AES_GCMSIV_ctr_neon_data_done_%=\n\t" "\n" "L_AES_GCMSIV_ctr_neon_loop_2_%=:\n\t" "mov x9, %x[KS]\n\t" @@ -57805,7 +59788,7 @@ void AES_XTS_encrypt(const byte* in, byte* out, word32 sz, const byte* i, "add %x[in], %x[in], #16\n\t" "add %x[out], %x[out], #16\n\t" "cmp %w[sz], #16\n\t" - "bcs L_AES_XTS_encrypt_loop_block_%=\n\t" + "b.cs L_AES_XTS_encrypt_loop_block_%=\n\t" "cbz %w[sz], L_AES_XTS_encrypt_done_data_%=\n\t" "mov x26, %x[key]\n\t" "sub %x[out], %x[out], #16\n\t" @@ -58440,7 +60423,7 @@ void AES_XTS_decrypt(const byte* in, byte* out, word32 sz, const byte* i, "rev32 x23, x23\n\t" "rev32 x24, x24\n\t" "cmp %w[sz], #16\n\t" - "bcc L_AES_XTS_decrypt_start_partail_%=\n\t" + "b.cc L_AES_XTS_decrypt_start_partail_%=\n\t" "\n" "L_AES_XTS_decrypt_loop_block_%=:\n\t" "mov x28, %x[key]\n\t" @@ -58722,7 +60705,7 @@ void AES_XTS_decrypt(const byte* in, byte* out, word32 sz, const byte* i, "add %x[in], %x[in], #16\n\t" "add %x[out], %x[out], #16\n\t" "cmp %w[sz], #16\n\t" - "bcs L_AES_XTS_decrypt_loop_block_%=\n\t" + "b.cs L_AES_XTS_decrypt_loop_block_%=\n\t" "cbz %w[sz], L_AES_XTS_decrypt_done_data_%=\n\t" "\n" "L_AES_XTS_decrypt_start_partail_%=:\n\t" diff --git a/wolfcrypt/src/sha256.c b/wolfcrypt/src/sha256.c index 14c1f99caa0..dc80ece2f26 100644 --- a/wolfcrypt/src/sha256.c +++ b/wolfcrypt/src/sha256.c @@ -1304,12 +1304,26 @@ static WC_INLINE int Transform_Sha256_Len_aarch64(wc_Sha256* sha256, return (*Transform_Sha256_Len_p)(sha256, data, len); } +/* Both transforms below run on v0-v31 and save d8-d15 + * (port/arm/armv8-sha256-asm.S), so a kernel module must bracket them. */ +#if defined(__aarch64__) && defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) + #define WC_SHA256_ARM64_SVR_BEGIN() \ + do { int _svr_ret = SAVE_VECTOR_REGISTERS2(); \ + if (_svr_ret != 0) return _svr_ret; } while (0) + #define WC_SHA256_ARM64_SVR_END() RESTORE_VECTOR_REGISTERS() +#else + #define WC_SHA256_ARM64_SVR_BEGIN() WC_DO_NOTHING + #define WC_SHA256_ARM64_SVR_END() WC_DO_NOTHING +#endif + #if !defined(WOLFSSL_ARMASM_NO_NEON) #if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) static int Transform_Sha256_Len_crypto_aarch64(wc_Sha256* sha256, const byte* data, word32 len) { + WC_SHA256_ARM64_SVR_BEGIN(); Transform_Sha256_Len_crypto(sha256, data, len); + WC_SHA256_ARM64_SVR_END(); return 0; } #endif @@ -1317,7 +1331,9 @@ static int Transform_Sha256_Len_crypto_aarch64(wc_Sha256* sha256, static int Transform_Sha256_Len_neon_aarch64(wc_Sha256* sha256, const byte* data, word32 len) { + WC_SHA256_ARM64_SVR_BEGIN(); Transform_Sha256_Len_neon(sha256, data, len); + WC_SHA256_ARM64_SVR_END(); return 0; } #endif diff --git a/wolfcrypt/src/sha3.c b/wolfcrypt/src/sha3.c index 15f6ba2a2ed..24abb0eef11 100644 --- a/wolfcrypt/src/sha3.c +++ b/wolfcrypt/src/sha3.c @@ -152,6 +152,14 @@ #endif #endif +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ + defined(WOLFSSL_ARMASM_CRYPTO_SHA3) + /* BlockSha3_crypto is the ARMv8.2 crypto-extension permutation and writes + * v0-v31, so the caller must hold the vector registers for it. */ + #define SHA3_BLOCK_VREGS(f) ((f) == BlockSha3_crypto) + #define SHA3_NEEDS_VREG_CLAIM +#endif + #if !defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_RISCV_ASM) && \ !defined(WOLFSSL_PPC64_ASM) && !defined(WOLFSSL_PPC32_ASM) @@ -992,7 +1000,7 @@ static int Sha3Update(wc_Sha3* sha3, const byte* data, word32 len, word32 p) if ((p < WC_SHA3_512_COUNT) || (p > WC_SHA3_128_COUNT)) return BAD_STATE_E; -#ifdef USE_INTEL_SPEEDUP +#if defined(USE_INTEL_SPEEDUP) || defined(SHA3_NEEDS_VREG_CLAIM) if (SHA3_BLOCK_VREGS(sha3_block)) { ret = SAVE_VECTOR_REGISTERS2(); if (ret != 0) { @@ -1111,7 +1119,7 @@ static int Sha3Update(wc_Sha3* sha3, const byte* data, word32 len, word32 p) out: -#ifdef USE_INTEL_SPEEDUP +#if defined(USE_INTEL_SPEEDUP) || defined(SHA3_NEEDS_VREG_CLAIM) if (SHA3_BLOCK_VREGS(sha3_block)) { RESTORE_VECTOR_REGISTERS(); } @@ -1204,7 +1212,7 @@ static int Sha3Final(wc_Sha3* sha3, byte padChar, byte* hash, word32 p, word32 l #endif #endif -#ifdef USE_INTEL_SPEEDUP +#if defined(USE_INTEL_SPEEDUP) || defined(SHA3_NEEDS_VREG_CLAIM) if (SHA3_BLOCK_VREGS(sha3_block)) { int ret = SAVE_VECTOR_REGISTERS2(); if (ret != 0) { @@ -1246,7 +1254,7 @@ static int Sha3Final(wc_Sha3* sha3, byte padChar, byte* hash, word32 p, word32 l XMEMCPY(hash + j, sha3->s, l - j); #endif } -#ifdef USE_INTEL_SPEEDUP +#if defined(USE_INTEL_SPEEDUP) || defined(SHA3_NEEDS_VREG_CLAIM) if (SHA3_BLOCK_VREGS(sha3_block)) { RESTORE_VECTOR_REGISTERS(); } @@ -2265,7 +2273,7 @@ int wc_Shake128_SqueezeBlocks(wc_Shake* shake, byte* out, word32 blockCnt) return BAD_FUNC_ARG; } -#ifdef USE_INTEL_SPEEDUP +#if defined(USE_INTEL_SPEEDUP) || defined(SHA3_NEEDS_VREG_CLAIM) #ifdef WC_C_DYNAMIC_FALLBACK sha3_block = SHA3_BLOCK; #endif @@ -2280,7 +2288,7 @@ int wc_Shake128_SqueezeBlocks(wc_Shake* shake, byte* out, word32 blockCnt) #endif } } -#endif /* USE_INTEL_SPEEDUP */ +#endif for (; (blockCnt > 0); blockCnt--) { #ifdef SHA3_FUNC_PTR @@ -2298,7 +2306,7 @@ int wc_Shake128_SqueezeBlocks(wc_Shake* shake, byte* out, word32 blockCnt) out += WC_SHA3_128_COUNT * 8; } -#ifdef USE_INTEL_SPEEDUP +#if defined(USE_INTEL_SPEEDUP) || defined(SHA3_NEEDS_VREG_CLAIM) if (SHA3_BLOCK_VREGS(sha3_block)) RESTORE_VECTOR_REGISTERS(); #endif @@ -2571,7 +2579,7 @@ int wc_Shake256_SqueezeBlocks(wc_Shake* shake, byte* out, word32 blockCnt) return BAD_FUNC_ARG; } -#ifdef USE_INTEL_SPEEDUP +#if defined(USE_INTEL_SPEEDUP) || defined(SHA3_NEEDS_VREG_CLAIM) #ifdef WC_C_DYNAMIC_FALLBACK sha3_block = SHA3_BLOCK; #endif @@ -2586,7 +2594,7 @@ int wc_Shake256_SqueezeBlocks(wc_Shake* shake, byte* out, word32 blockCnt) #endif } } -#endif /* USE_INTEL_SPEEDUP */ +#endif for (; (blockCnt > 0); blockCnt--) { #ifdef SHA3_FUNC_PTR @@ -2604,7 +2612,7 @@ int wc_Shake256_SqueezeBlocks(wc_Shake* shake, byte* out, word32 blockCnt) out += WC_SHA3_256_COUNT * 8; } -#ifdef USE_INTEL_SPEEDUP +#if defined(USE_INTEL_SPEEDUP) || defined(SHA3_NEEDS_VREG_CLAIM) if (SHA3_BLOCK_VREGS(sha3_block)) RESTORE_VECTOR_REGISTERS(); #endif diff --git a/wolfcrypt/src/sha512.c b/wolfcrypt/src/sha512.c index 108af7ed2f8..b1c435890c5 100644 --- a/wolfcrypt/src/sha512.c +++ b/wolfcrypt/src/sha512.c @@ -1527,29 +1527,49 @@ static int Transform_Sha512_Len_C(wc_Sha512* sha512, const byte* data, /* The SHA-512 crypto instructions operate on SIMD registers, so the assembly * only defines these when NEON is available - see armv8-sha512-asm.S and the * prototype guard in sha512.h. */ +/* Both transforms below run on v0-v31 and save d8-d15 + * (port/arm/armv8-sha512-asm.S), so a kernel module must bracket them. */ +#if defined(__aarch64__) && defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) + #define WC_SHA512_ARM64_SVR_BEGIN() \ + do { int _svr_ret = SAVE_VECTOR_REGISTERS2(); \ + if (_svr_ret != 0) return _svr_ret; } while (0) + #define WC_SHA512_ARM64_SVR_END() RESTORE_VECTOR_REGISTERS() +#else + #define WC_SHA512_ARM64_SVR_BEGIN() WC_DO_NOTHING + #define WC_SHA512_ARM64_SVR_END() WC_DO_NOTHING +#endif + #if defined(WOLFSSL_ARMASM_CRYPTO_SHA512) && !defined(WOLFSSL_ARMASM_NO_NEON) static int Transform_Sha512_crypto_aarch64(wc_Sha512* sha512, const byte* data) { + WC_SHA512_ARM64_SVR_BEGIN(); Transform_Sha512_Len_crypto(sha512, data, WC_SHA512_BLOCK_SIZE); + WC_SHA512_ARM64_SVR_END(); return 0; } static int Transform_Sha512_Len_crypto_aarch64(wc_Sha512* sha512, const byte* data, word32 len) { + WC_SHA512_ARM64_SVR_BEGIN(); Transform_Sha512_Len_crypto(sha512, data, len); + WC_SHA512_ARM64_SVR_END(); return 0; } #endif #ifndef WOLFSSL_ARMASM_NO_NEON static int Transform_Sha512_neon_aarch64(wc_Sha512* sha512, const byte* data) { + WC_SHA512_ARM64_SVR_BEGIN(); Transform_Sha512_Len_neon(sha512, data, WC_SHA512_BLOCK_SIZE); + WC_SHA512_ARM64_SVR_END(); return 0; } static int Transform_Sha512_Len_neon_aarch64(wc_Sha512* sha512, const byte* data, word32 len) { + WC_SHA512_ARM64_SVR_BEGIN(); Transform_Sha512_Len_neon(sha512, data, len); + WC_SHA512_ARM64_SVR_END(); return 0; } #endif diff --git a/wolfcrypt/src/wc_mlkem.c b/wolfcrypt/src/wc_mlkem.c index 616bc1e018c..84207e24cca 100644 --- a/wolfcrypt/src/wc_mlkem.c +++ b/wolfcrypt/src/wc_mlkem.c @@ -91,8 +91,24 @@ #endif #include + #include #include + +/* aarch64 claim around the NEON helpers in wc_mlkem_poly.c. stmt runs only + * when ret is 0; other builds run stmt as is. */ +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + #define MLKEM_ARM64_SVR(stmt) \ + do { \ + ret = SAVE_VECTOR_REGISTERS2(); \ + if (ret == 0) { \ + stmt; \ + RESTORE_VECTOR_REGISTERS(); \ + } \ + } while (0) +#else + #define MLKEM_ARM64_SVR(stmt) do { stmt; } while (0) +#endif #ifdef WOLF_CRYPTO_CB #include #endif @@ -955,7 +971,9 @@ int wc_MlKemKey_MakeKeyWithRandom(MlKemKey* key, const unsigned char* rand, /* Generate key pair from random data. * Alg 13: Steps 16-18. */ - mlkem_keygen(s, t, e, a, k); + ret = mlkem_keygen(s, t, e, a, k); + } + if (ret == 0) { #else /* Generate noise using PRF. * Alg 13: Steps 8-11: generate s @@ -1321,8 +1339,9 @@ static int mlkemkey_encapsulate(MlKemKey* key, const byte* m, byte* r, byte* c) /* Convert msg to a polynomial. * Step 20: mu <- Decompress_1(ByteDecode_1(m)) */ - mlkem_from_msg(mu, m); - + MLKEM_ARM64_SVR(mlkem_from_msg(mu, m)); + } + if (ret == 0) { /* Initialize the PRF for use in the noise generation. */ mlkem_prf_init(&key->prf); /* Generate noise using PRF. @@ -1359,7 +1378,7 @@ static int mlkemkey_encapsulate(MlKemKey* key, const byte* m, byte* r, byte* c) /* Perform encapsulation maths. * Steps 18-19, 21: calculate u and v */ - mlkem_encapsulate(key->pub, u, v, a, y, e1, e2, mu, (int)k); + ret = mlkem_encapsulate(key->pub, u, v, a, y, e1, e2, mu, (int)k); } #else /* WOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM */ if (ret == 0) { @@ -1393,28 +1412,34 @@ static int mlkemkey_encapsulate(MlKemKey* key, const byte* m, byte* r, byte* c) #if defined(WOLFSSL_KYBER512) || defined(WOLFSSL_WC_ML_KEM_512) if (k == WC_ML_KEM_512_K) { - /* Step 22: c_1 <- ByteEncode_d_u(Compress_d_u(u)) */ - mlkem_vec_compress_10(c1, u, k); - /* Step 23: c_2 <- ByteEncode_d_v(Compress_d_v(v)) */ - mlkem_compress_4(c2, v); + /* Step 22: c_1 <- ByteEncode_d_u(Compress_d_u(u)) + * Step 23: c_2 <- ByteEncode_d_v(Compress_d_v(v)) */ + MLKEM_ARM64_SVR({ + mlkem_vec_compress_10(c1, u, k); + mlkem_compress_4(c2, v); + }); /* Step 24: return c <- (c_1||c_2) */ } #endif #if defined(WOLFSSL_KYBER768) || defined(WOLFSSL_WC_ML_KEM_768) if (k == WC_ML_KEM_768_K) { - /* Step 22: c_1 <- ByteEncode_d_u(Compress_d_u(u)) */ - mlkem_vec_compress_10(c1, u, k); - /* Step 23: c_2 <- ByteEncode_d_v(Compress_d_v(v)) */ - mlkem_compress_4(c2, v); + /* Step 22: c_1 <- ByteEncode_d_u(Compress_d_u(u)) + * Step 23: c_2 <- ByteEncode_d_v(Compress_d_v(v)) */ + MLKEM_ARM64_SVR({ + mlkem_vec_compress_10(c1, u, k); + mlkem_compress_4(c2, v); + }); /* Step 24: return c <- (c_1||c_2) */ } #endif #if defined(WOLFSSL_KYBER1024) || defined(WOLFSSL_WC_ML_KEM_1024) if (k == WC_ML_KEM_1024_K) { - /* Step 22: c_1 <- ByteEncode_d_u(Compress_d_u(u)) */ - mlkem_vec_compress_11(c1, u); - /* Step 23: c_2 <- ByteEncode_d_v(Compress_d_v(v)) */ - mlkem_compress_5(c2, v); + /* Step 22: c_1 <- ByteEncode_d_u(Compress_d_u(u)) + * Step 23: c_2 <- ByteEncode_d_v(Compress_d_v(v)) */ + MLKEM_ARM64_SVR({ + mlkem_vec_compress_11(c1, u); + mlkem_compress_5(c2, v); + }); /* Step 24: return c <- (c_1||c_2) */ } #endif @@ -1819,7 +1844,7 @@ static MLKEM_NOINLINE int mlkemkey_decapsulate(MlKemKey* key, byte* m, { int ret = 0; sword16* v; - sword16* w; + sword16* w = NULL; unsigned int k = 0; unsigned int compVecSz = 0; #if defined(WOLFSSL_SMALL_STACK) || \ @@ -1927,11 +1952,13 @@ static MLKEM_NOINLINE int mlkemkey_decapsulate(MlKemKey* key, byte* m, /* Decapsulate the cipher text into polynomial. * Step 6: w <- v' - InvNTT(s_hat_trans o NTT(u')) */ - mlkem_decapsulate(key->priv, w, u, v, (int)k); + ret = mlkem_decapsulate(key->priv, w, u, v, (int)k); + } + if (ret == 0) { /* Convert the polynomial into a array of bytes (message). * Step 7: m <- ByteEncode_1(Compress_1(w)) */ - mlkem_to_msg(m, w); + MLKEM_ARM64_SVR(mlkem_to_msg(m, w)); /* Step 8: return m */ } @@ -2011,7 +2038,7 @@ int wc_MlKemKey_Decapsulate(MlKemKey* key, unsigned char* ss, int ret = 0; unsigned int ctSz = 0; unsigned int i = 0; - int fail = 0; + int fail = -1; /* mismatch until mlkem_cmp() says otherwise */ #if !defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_NO_MALLOC) byte* cmp = NULL; #else @@ -2130,8 +2157,9 @@ int wc_MlKemKey_Decapsulate(MlKemKey* key, unsigned char* ss, } if (ret == 0) { /* Compare generated cipher text with that passed in. */ - fail = mlkem_cmp(ct, cmp, (int)ctSz); - + MLKEM_ARM64_SVR(fail = mlkem_cmp(ct, cmp, (int)ctSz)); + } + if (ret == 0) { #if defined(WOLFSSL_MLKEM_KYBER) && !defined(WOLFSSL_NO_ML_KEM) if (key->type & MLKEM_KYBER) #endif @@ -2727,7 +2755,9 @@ int wc_MlKemKey_EncodePrivateKey(MlKemKey* key, unsigned char* out, word32 len) if (ret == 0) { /* Encode private key that is vector of polynomials. */ - mlkem_to_bytes(p, key->priv, (int)k); + MLKEM_ARM64_SVR(mlkem_to_bytes(p, key->priv, (int)k)); + } + if (ret == 0) { p += WC_ML_KEM_POLY_SIZE * k; /* Encode public key - calculates hash of public key. */ @@ -2835,11 +2865,13 @@ int wc_MlKemKey_EncodePublicKey(MlKemKey* key, unsigned char* out, word32 len) ret = BUFFER_E; } + if (ret == 0) { + /* Encode public key polynomial by polynomial. */ + MLKEM_ARM64_SVR(mlkem_to_bytes(p, key->pub, (int)k)); + } if (ret == 0) { int i; - /* Encode public key polynomial by polynomial. */ - mlkem_to_bytes(p, key->pub, (int)k); p += k * WC_ML_KEM_POLY_SIZE; /* Append public seed. */ diff --git a/wolfcrypt/src/wc_mlkem_poly.c b/wolfcrypt/src/wc_mlkem_poly.c index eb988c52cba..bd2cfde25f4 100644 --- a/wolfcrypt/src/wc_mlkem_poly.c +++ b/wolfcrypt/src/wc_mlkem_poly.c @@ -1330,9 +1330,14 @@ static void mlkem_shake256_blocksx3_seed(word64* state, byte* seed) * @param [in] a Random values in an array of vectors of polynomials. * @param [in] k Number of polynomials in vector. */ -void mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) +int mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) { int i; + /* Every routine below is NEON asm (armv8-mlkem-asm.S): one claim. */ + int svr_ret = SAVE_VECTOR_REGISTERS2(); + if (svr_ret != 0) { + return svr_ret; + } #ifndef WOLFSSL_AARCH64_NO_SQRDMLSH if (IS_AARCH64_RDM(cpuid_flags)) { @@ -1387,6 +1392,8 @@ void mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) mlkem_add_reduce(t + i * MLKEM_N, e + i * MLKEM_N); } } + RESTORE_VECTOR_REGISTERS(); + return 0; } #endif /* WOLFSSL_MLKEM_NO_MAKE_KEY */ @@ -1412,11 +1419,16 @@ void mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) * @param [in] m Message polynomial. * @param [in] k Number of polynomials in vector. */ -void mlkem_encapsulate(const sword16* t, sword16* u, sword16* v, +int mlkem_encapsulate(const sword16* t, sword16* u, sword16* v, const sword16* a, sword16* y, const sword16* e1, const sword16* e2, const sword16* m, int k) { int i; + /* Every routine below is NEON asm (armv8-mlkem-asm.S): one claim. */ + int svr_ret = SAVE_VECTOR_REGISTERS2(); + if (svr_ret != 0) { + return svr_ret; + } #ifndef WOLFSSL_AARCH64_NO_SQRDMLSH if (IS_AARCH64_RDM(cpuid_flags)) { @@ -1482,6 +1494,8 @@ void mlkem_encapsulate(const sword16* t, sword16* u, sword16* v, /* Add errors and message to v and reduce. * Step 21: v <- InvNTT(t_hat_trans o y_hat) + e_2 + mu */ mlkem_add3_reduce(v, e2, m); + RESTORE_VECTOR_REGISTERS(); + return 0; } #endif /* !WOLFSSL_MLKEM_NO_ENCAPSULATE || !WOLFSSL_MLKEM_NO_DECAPSULATE */ @@ -1500,10 +1514,15 @@ void mlkem_encapsulate(const sword16* t, sword16* u, sword16* v, * @param [in] v Encapsulated message polynomial. * @param [in] k Number of polynomials in vector. */ -void mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, +int mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, const sword16* v, int k) { int i; + /* Every routine below is NEON asm (armv8-mlkem-asm.S): one claim. */ + int svr_ret = SAVE_VECTOR_REGISTERS2(); + if (svr_ret != 0) { + return svr_ret; + } #ifndef WOLFSSL_AARCH64_NO_SQRDMLSH if (IS_AARCH64_RDM(cpuid_flags)) { @@ -1539,6 +1558,8 @@ void mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, /* Subtract errors (in w) out of v and reduce into w. * Step 6: w <- v' - InvNTT(s_hat_trans o NTT(u')) */ mlkem_rsub_reduce(w, v); + RESTORE_VECTOR_REGISTERS(); + return 0; } #endif /* !WOLFSSL_MLKEM_NO_DECAPSULATE */ @@ -1923,7 +1944,7 @@ static void mlkem_keygen_c(sword16* s, sword16* t, sword16* e, const sword16* a, * @param [in] a Random values in an array of vectors of polynomials. * @param [in] k Number of polynomials in vector. */ -void mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) +int mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -1945,6 +1966,8 @@ void mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) /* Alg 13: Steps 16-18 */ mlkem_keygen_c(s, t, e, a, k); } + + return 0; } #else /* WOLFSSL_MLKEM_MAKEKEY_SMALL_MEM */ @@ -2134,7 +2157,7 @@ static void mlkem_encapsulate_c(const sword16* pub, sword16* u, sword16* v, * @param [in] m Message polynomial. * @param [in] k Number of polynomials in vector. */ -void mlkem_encapsulate(const sword16* pub, sword16* u, sword16* v, +int mlkem_encapsulate(const sword16* pub, sword16* u, sword16* v, const sword16* a, sword16* y, const sword16* e1, const sword16* e2, const sword16* m, int k) { @@ -2155,6 +2178,8 @@ void mlkem_encapsulate(const sword16* pub, sword16* u, sword16* v, { mlkem_encapsulate_c(pub, u, v, a, y, e1, e2, m, k); } + + return 0; } #else @@ -2338,7 +2363,7 @@ static void mlkem_decapsulate_c(const sword16* s, sword16* w, sword16* u, * @param [in] v Encapsulated message polynomial. * @param [in] k Number of polynomials in vector. */ -void mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, +int mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, const sword16* v, int k) { #ifdef USE_INTEL_SPEEDUP @@ -2358,6 +2383,8 @@ void mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, { mlkem_decapsulate_c(s, w, u, v, k); } + + return 0; } #endif /* !WOLFSSL_MLKEM_NO_DECAPSULATE */ @@ -4090,7 +4117,12 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #if defined(WOLFSSL_KYBER512) || defined(WOLFSSL_WC_ML_KEM_512) if (k == WC_ML_KEM_512_K) { #if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_gen_matrix_k2_aarch64(a, seed, transposed); + /* SHAKE-128 and the rejection sampler are NEON asm in this lane. */ + ret = SAVE_VECTOR_REGISTERS2(); + if (ret == 0) { + ret = mlkem_gen_matrix_k2_aarch64(a, seed, transposed); + RESTORE_VECTOR_REGISTERS(); + } #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -4116,7 +4148,12 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #if defined(WOLFSSL_KYBER768) || defined(WOLFSSL_WC_ML_KEM_768) if (k == WC_ML_KEM_768_K) { #if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_gen_matrix_k3_aarch64(a, seed, transposed); + /* SHAKE-128 and the rejection sampler are NEON asm in this lane. */ + ret = SAVE_VECTOR_REGISTERS2(); + if (ret == 0) { + ret = mlkem_gen_matrix_k3_aarch64(a, seed, transposed); + RESTORE_VECTOR_REGISTERS(); + } #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -4142,7 +4179,12 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #if defined(WOLFSSL_KYBER1024) || defined(WOLFSSL_WC_ML_KEM_1024) if (k == WC_ML_KEM_1024_K) { #if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_gen_matrix_k4_aarch64(a, seed, transposed); + /* SHAKE-128 and the rejection sampler are NEON asm in this lane. */ + ret = SAVE_VECTOR_REGISTERS2(); + if (ret == 0) { + ret = mlkem_gen_matrix_k4_aarch64(a, seed, transposed); + RESTORE_VECTOR_REGISTERS(); + } #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -5507,7 +5549,12 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #if defined(WOLFSSL_KYBER512) || defined(WOLFSSL_WC_ML_KEM_512) if (k == WC_ML_KEM_512_K) { #if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_get_noise_k2_aarch64(vec1, vec2, poly, seed); + /* SHAKE-256 and the CBD sampler are NEON asm in this lane. */ + ret = SAVE_VECTOR_REGISTERS2(); + if (ret == 0) { + ret = mlkem_get_noise_k2_aarch64(vec1, vec2, poly, seed); + RESTORE_VECTOR_REGISTERS(); + } #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -5538,7 +5585,12 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #if defined(WOLFSSL_KYBER768) || defined(WOLFSSL_WC_ML_KEM_768) if (k == WC_ML_KEM_768_K) { #if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_get_noise_k3_aarch64(vec1, vec2, poly, seed); + /* SHAKE-256 and the CBD sampler are NEON asm in this lane. */ + ret = SAVE_VECTOR_REGISTERS2(); + if (ret == 0) { + ret = mlkem_get_noise_k3_aarch64(vec1, vec2, poly, seed); + RESTORE_VECTOR_REGISTERS(); + } #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -5565,7 +5617,12 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #if defined(WOLFSSL_KYBER1024) || defined(WOLFSSL_WC_ML_KEM_1024) if (k == WC_ML_KEM_1024_K) { #if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_get_noise_k4_aarch64(vec1, vec2, poly, seed); + /* SHAKE-256 and the CBD sampler are NEON asm in this lane. */ + ret = SAVE_VECTOR_REGISTERS2(); + if (ret == 0) { + ret = mlkem_get_noise_k4_aarch64(vec1, vec2, poly, seed); + RESTORE_VECTOR_REGISTERS(); + } #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index e4407f710d8..8e2a6a17e81 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -17297,6 +17297,320 @@ static wc_test_ret_t aes_xts_large_test_common(XtsAes *aes, } } #endif /* HAVE_AES_DECRYPT */ + + /* Stream multi-block chunks against the one-shot. Pass 0 ends partial, + * pass 1 whole blocks, pass 2 repeats pass 0 in place. */ + { +#define XTS_STREAM_SZ (WC_AES_BLOCK_SIZE * 19 + 5) + /* 9 blocks runs the four-block loop twice; the last chunk adds a + * remainder so one call does whole blocks then the stealing tail. */ + static const word32 chunk_tail[] = { WC_AES_BLOCK_SIZE * 9, + WC_AES_BLOCK_SIZE * 4, + WC_AES_BLOCK_SIZE * 2, + WC_AES_BLOCK_SIZE * 4 + 5 }; + static const word32 chunk_exact[] = { WC_AES_BLOCK_SIZE * 9, + WC_AES_BLOCK_SIZE * 4, + WC_AES_BLOCK_SIZE * 3 }; + /* Carved out of large_input: already sized, heap on small stack. */ + wc_static_assert2(XTS_STREAM_SZ * 3 <= LARGE_XTS_SZ, + "plain/ref/buf must fit inside large_input"); + byte* plain = large_input; + byte* ref = large_input + XTS_STREAM_SZ; + byte* buf = large_input + (XTS_STREAM_SZ * 2); + const word32* chunk; + const byte* src; + word32 nchunk, total, off; + int pass, inplace; + size_t ci; + + for (pass = 0; pass < 3; pass++) { + inplace = (pass == 2); + if (pass == 1) { + chunk = chunk_exact; + nchunk = (word32)(sizeof(chunk_exact) / sizeof(chunk_exact[0])); + } + else { + chunk = chunk_tail; + nchunk = (word32)(sizeof(chunk_tail) / sizeof(chunk_tail[0])); + } + total = 0; + for (ci = 0; ci < (size_t)nchunk; ci++) + total += chunk[ci]; + /* Each of plain/ref/buf is one XTS_STREAM_SZ slice of large_input. */ + if (total > (word32)XTS_STREAM_SZ) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + + for (i = 0; i < (int)total; i++) + plain[i] = (byte)i; + + ret = wc_AesXtsSetKeyNoInit(aes, k1, k1Sz, AES_ENCRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + + ret = wc_AesXtsEncrypt(aes, ref, plain, total, i1, i1Sz); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + + if (inplace) + XMEMCPY(buf, plain, total); + else + XMEMSET(buf, 0, total); + src = inplace ? buf : plain; + ret = wc_AesXtsEncryptInit(aes, i1, i1Sz, &stream); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + off = 0; + for (ci = 0; ci < (size_t)nchunk - 1; ci++) { + ret = wc_AesXtsEncryptUpdate(aes, buf + off, src + off, + chunk[ci], &stream); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, + WC_ASYNC_FLAG_NONE); +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + off += chunk[ci]; + } + ret = wc_AesXtsEncryptFinal(aes, buf + off, src + off, + chunk[ci], &stream); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(buf, ref, total) != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + +#ifdef HAVE_AES_DECRYPT + ret = wc_AesXtsSetKeyNoInit(aes, k1, k1Sz, AES_DECRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + + if (inplace) + XMEMCPY(buf, ref, total); + else + XMEMSET(buf, 0, total); + src = inplace ? buf : ref; + ret = wc_AesXtsDecryptInit(aes, i1, i1Sz, &stream); +#if defined(WOLFSSL_ASYNC_CRYPT) +#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS + ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, + WC_ASYNC_FLAG_NONE); +#else + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + off = 0; + for (ci = 0; ci < (size_t)nchunk - 1; ci++) { + ret = wc_AesXtsDecryptUpdate(aes, buf + off, src + off, + chunk[ci], &stream); +#if defined(WOLFSSL_ASYNC_CRYPT) +#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS + ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, + WC_ASYNC_FLAG_NONE); +#else + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, + WC_ASYNC_FLAG_NONE); +#endif +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + off += chunk[ci]; + } + ret = wc_AesXtsDecryptFinal(aes, buf + off, src + off, chunk[ci], + &stream); +#if defined(WOLFSSL_ASYNC_CRYPT) +#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS + ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, + WC_ASYNC_FLAG_NONE); +#else + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (XMEMCMP(buf, plain, total) != 0) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif /* HAVE_AES_DECRYPT */ + } + + /* Update() rejects a partial block in the wrapper; Final() rejects a + * short sz before the assembly. */ + ret = wc_AesXtsSetKeyNoInit(aes, k1, k1Sz, AES_ENCRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + ret = wc_AesXtsEncryptInit(aes, i1, i1Sz, &stream); +#if defined(WOLFSSL_ASYNC_CRYPT) + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + ret = wc_AesXtsEncryptUpdate(aes, buf, plain, + WC_AES_BLOCK_SIZE + 1, &stream); + if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + ret = wc_AesXtsEncryptFinal(aes, buf, plain, WC_AES_BLOCK_SIZE - 1, + &stream); + if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + +#ifdef HAVE_AES_DECRYPT + ret = wc_AesXtsSetKeyNoInit(aes, k1, k1Sz, AES_DECRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + ret = wc_AesXtsDecryptInit(aes, i1, i1Sz, &stream); +#if defined(WOLFSSL_ASYNC_CRYPT) +#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS + ret = wc_AsyncWait(ret, &aes->aes_decrypt.asyncDev, + WC_ASYNC_FLAG_NONE); +#else + ret = wc_AsyncWait(ret, &aes->aes.asyncDev, WC_ASYNC_FLAG_NONE); +#endif +#endif + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + ret = wc_AesXtsDecryptUpdate(aes, buf, ref, + WC_AES_BLOCK_SIZE + 1, &stream); + if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + ret = wc_AesXtsDecryptFinal(aes, buf, ref, WC_AES_BLOCK_SIZE - 1, + &stream); + if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); +#endif /* HAVE_AES_DECRYPT */ + + /* Near the top of the 32-bit count a stream must either be refused + * or still agree with the one-shot. */ +#ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING + { + word32 before; + + ret = wc_AesXtsSetKeyNoInit(aes, k1, k1Sz, AES_ENCRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + ret = wc_AesXtsEncrypt(aes, ref, plain, WC_AES_BLOCK_SIZE * 2, + i1, i1Sz); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + + ret = wc_AesXtsEncryptInit(aes, i1, i1Sz, &stream); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + /* a whole number of blocks, so the post-finalize guard is clear */ + stream.bytes_crypted_with_this_tweak = 0xFFFFFFF0U; + before = stream.bytes_crypted_with_this_tweak; + XMEMSET(buf, 0x5A, WC_AES_BLOCK_SIZE * 2); + ret = wc_AesXtsEncryptUpdate(aes, buf, plain, + WC_AES_BLOCK_SIZE * 2, &stream); + /* The count can no longer advance, so the call must be refused + * rather than run unaccounted. */ + if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (stream.bytes_crypted_with_this_tweak != before) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + { + word32 b; + for (b = 0; b < (word32)WC_AES_BLOCK_SIZE * 2; b++) { + if (buf[b] != 0x5A) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + } + } + +#ifdef HAVE_AES_DECRYPT + /* Decrypt is held to the same rule: refuse rather than run + * unaccounted once the count can no longer advance. */ + ret = wc_AesXtsSetKeyNoInit(aes, k1, k1Sz, AES_DECRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + ret = wc_AesXtsDecryptInit(aes, i1, i1Sz, &stream); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + stream.bytes_crypted_with_this_tweak = 0xFFFFFFF0U; + before = stream.bytes_crypted_with_this_tweak; + XMEMSET(buf, 0x5A, WC_AES_BLOCK_SIZE * 2); + ret = wc_AesXtsDecryptUpdate(aes, buf, ref, + WC_AES_BLOCK_SIZE * 2, &stream); + if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (stream.bytes_crypted_with_this_tweak != before) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + { + word32 b; + for (b = 0; b < (word32)WC_AES_BLOCK_SIZE * 2; b++) { + if (buf[b] != 0x5A) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + } + } + +#if FIPS_VERSION3_GE(6,0,0) + /* SP800-38E caps a data unit at 2^20 blocks. Decrypt is held to + * the same limit as encrypt. */ + ret = wc_AesXtsSetKeyNoInit(aes, k1, k1Sz, AES_DECRYPTION); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + ret = wc_AesXtsDecryptInit(aes, i1, i1Sz, &stream); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + /* A block that lands exactly ON the cap is still allowed: the + * check is "greater than", not "greater or equal". */ + stream.bytes_crypted_with_this_tweak = + FIPS_AES_XTS_MAX_BYTES_PER_TWEAK - WC_AES_BLOCK_SIZE; + ret = wc_AesXtsDecryptUpdate(aes, buf, ref, + WC_AES_BLOCK_SIZE, &stream); + if (ret != 0) + ERROR_OUT(WC_TEST_RET_ENC_EC(ret), out); + if (stream.bytes_crypted_with_this_tweak != + (word32)FIPS_AES_XTS_MAX_BYTES_PER_TWEAK) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + + /* The next block is over the cap: refused, and it must leave the + * output and the count exactly as they were. */ + before = stream.bytes_crypted_with_this_tweak; + XMEMSET(buf, 0x5A, WC_AES_BLOCK_SIZE); + ret = wc_AesXtsDecryptUpdate(aes, buf, ref, + WC_AES_BLOCK_SIZE, &stream); + if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + if (stream.bytes_crypted_with_this_tweak != before) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + { + word32 b; + for (b = 0; b < (word32)WC_AES_BLOCK_SIZE; b++) { + if (buf[b] != 0x5A) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + } + } + + /* The one-shot decrypt entry enforces the same cap, and refuses + * the size before touching either buffer. */ + XMEMSET(buf, 0x5A, WC_AES_BLOCK_SIZE); + ret = wc_AesXtsDecrypt(aes, buf, ref, + FIPS_AES_XTS_MAX_BYTES_PER_TWEAK + WC_AES_BLOCK_SIZE, + i1, i1Sz); + if (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + { + word32 b; + for (b = 0; b < (word32)WC_AES_BLOCK_SIZE; b++) { + if (buf[b] != 0x5A) + ERROR_OUT(WC_TEST_RET_ENC_NC, out); + } + } +#endif +#endif /* HAVE_AES_DECRYPT */ + } +#endif /* !WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING */ + ret = 0; +#undef XTS_STREAM_SZ + } #endif /* WOLFSSL_AESXTS_STREAM */ out: diff --git a/wolfssl/wolfcrypt/aes.h b/wolfssl/wolfcrypt/aes.h index 455f30088b7..82bf44e9143 100644 --- a/wolfssl/wolfcrypt/aes.h +++ b/wolfssl/wolfcrypt/aes.h @@ -1221,6 +1221,14 @@ WOLFSSL_LOCAL void AES_XTS_encrypt_AARCH64(const byte* in, byte* out, word32 sz, const byte* i, byte* key, byte* key2, byte* tmp, int nr); WOLFSSL_LOCAL void AES_XTS_decrypt_AARCH64(const byte* in, byte* out, word32 sz, const byte* i, byte* key, byte* key2, byte* tmp, int nr); +#ifdef WOLFSSL_AESXTS_STREAM +/* Streaming twins of the pair above: "tweak" is read and written back, "tmp" + * is one block of caller scratch, and sz must be at least one block. */ +WOLFSSL_LOCAL void AES_XTS_encrypt_update_AARCH64(const byte* in, byte* out, + word32 sz, byte* key, byte* tweak, byte* tmp, int nr); +WOLFSSL_LOCAL void AES_XTS_decrypt_update_AARCH64(const byte* in, byte* out, + word32 sz, byte* key, byte* tweak, byte* tmp, int nr); +#endif /* WOLFSSL_AESXTS_STREAM */ #endif /* WOLFSSL_AES_XTS */ #endif /* __aarch64__ && !WOLFSSL_ARMASM_NO_HW_CRYPTO */ diff --git a/wolfssl/wolfcrypt/wc_mlkem.h b/wolfssl/wolfcrypt/wc_mlkem.h index 4c45786525e..d1328a5fea3 100644 --- a/wolfssl/wolfcrypt/wc_mlkem.h +++ b/wolfssl/wolfcrypt/wc_mlkem.h @@ -501,7 +501,7 @@ void mlkem_init(void); #ifndef WOLFSSL_MLKEM_MAKEKEY_SMALL_MEM WOLFSSL_LOCAL -void mlkem_keygen(sword16* priv, sword16* pub, sword16* e, const sword16* a, +int mlkem_keygen(sword16* priv, sword16* pub, sword16* e, const sword16* a, int kp); #else WOLFSSL_LOCAL @@ -510,7 +510,7 @@ int mlkem_keygen_seeds(sword16* priv, sword16* pub, MLKEM_PRF_T* prf, #endif #ifndef WOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM WOLFSSL_LOCAL -void mlkem_encapsulate(const sword16* pub, sword16* bp, sword16* v, +int mlkem_encapsulate(const sword16* pub, sword16* bp, sword16* v, const sword16* at, sword16* sp, const sword16* ep, const sword16* epp, const sword16* m, int kp); #else @@ -520,7 +520,7 @@ int mlkem_encapsulate_seeds(const sword16* pub, MLKEM_PRF_T* prf, sword16* bp, byte* coins); #endif WOLFSSL_LOCAL -void mlkem_decapsulate(const sword16* priv, sword16* mp, sword16* bp, +int mlkem_decapsulate(const sword16* priv, sword16* mp, sword16* bp, const sword16* v, int kp); WOLFSSL_LOCAL