From 2a3c050fcaf9c39df9bd33767adb09da2647acc7 Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Fri, 4 Sep 2026 09:30:03 +1000 Subject: [PATCH] ML-DSA: make checking low constant time Changed C and x64 assembly code. --- wolfcrypt/src/wc_mldsa.c | 35 +++++---- wolfcrypt/src/wc_mldsa_asm.S | 125 +++++++++---------------------- wolfcrypt/src/wc_mldsa_asm.asm | 131 ++++++++++----------------------- 3 files changed, 92 insertions(+), 199 deletions(-) diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index 7e63e33a1f4..1b52b4ecf88 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -5658,27 +5658,38 @@ static void mldsa_vec_decompose(const sword32* r, byte k, sword32 gamma2, * Many places in FIPS 204. One example from Algorithm 2: * 23: if ||z||inf >= GAMMA1 - BETA or ..., then (z, h) = falsam * + * Constant time with respect to the values of the polynomial. Whether any + * value is out of range is public - it is the result of the check - but which + * value is out of range is not. Therefore every value is checked and no early + * exit is performed. + * + * Values are always small - magnitude less than MLDSA_Q - and hi is positive + * and less than MLDSA_Q. The differences calculated below therefore never + * overflow 32 bits. + * * @param [in] a Polynomial. * @param [in] hi Largest value in range. + * @return 1 when all values are in range. + * @return 0 when at least one value is out of range. */ static int mldsa_check_low(const sword32* a, sword32 hi) { - int ret = 1; unsigned int j; /* Calculate lowest range value. */ sword32 nhi = -hi; + /* Top bit stays set while all values checked are in range. */ + word32 in = 0xffffffffU; /* For each value of polynomial. */ for (j = 0; j < MLDSA_N; j++) { - /* Check range is -(hi-1)..(hi-1). */ - if ((a[j] <= nhi) || (a[j] >= hi)) { - /* Check failed. */ - ret = 0; - break; - } + /* Check range is -(hi-1)..(hi-1). + * Top bit of a[j] - hi is set when a[j] < hi. + * Top bit of nhi - a[j] is set when a[j] > nhi. */ + in &= ((word32)a[j] - (word32)hi) & ((word32)nhi - (word32)a[j]); } - return ret; + /* Top bit set when all values are in range. */ + return (int)(in >> 31); } #if !defined(WOLFSSL_MLDSA_NO_VERIFY) || \ @@ -5698,12 +5709,10 @@ static int mldsa_vec_check_low_c(const sword32* a, byte l, sword32 hi) int ret = 1; unsigned int i; - /* For each polynomial of vector. */ + /* For each polynomial of vector. + * Constant time - all polynomials are checked. */ for (i = 0; i < l; i++) { - ret = mldsa_check_low(a, hi); - if (ret == 0) { - break; - } + ret &= mldsa_check_low(a, hi); /* Next polynomial. */ a += MLDSA_N; } diff --git a/wolfcrypt/src/wc_mldsa_asm.S b/wolfcrypt/src/wc_mldsa_asm.S index 1fc0d8646e0..26669947a60 100644 --- a/wolfcrypt/src/wc_mldsa_asm.S +++ b/wolfcrypt/src/wc_mldsa_asm.S @@ -39065,6 +39065,7 @@ _wc_mldsa_vec_check_low_avx2: movd %edx, %xmm3 vpbroadcastd %xmm2, %ymm2 vpbroadcastd %xmm3, %ymm3 + vpxor %ymm8, %ymm8, %ymm8 L_mldsa_vec_check_low_vx2_start_256: vmovdqu (%rdi), %ymm0 vmovdqu 32(%rdi), %ymm1 @@ -39075,10 +39076,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 64(%rdi), %ymm0 vmovdqu 96(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39088,10 +39086,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 128(%rdi), %ymm0 vmovdqu 160(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39101,10 +39096,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 192(%rdi), %ymm0 vmovdqu 224(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39114,10 +39106,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 256(%rdi), %ymm0 vmovdqu 288(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39127,10 +39116,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 320(%rdi), %ymm0 vmovdqu 352(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39140,10 +39126,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 384(%rdi), %ymm0 vmovdqu 416(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39153,10 +39136,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 448(%rdi), %ymm0 vmovdqu 480(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39166,10 +39146,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 512(%rdi), %ymm0 vmovdqu 544(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39179,10 +39156,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 576(%rdi), %ymm0 vmovdqu 608(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39192,10 +39166,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 640(%rdi), %ymm0 vmovdqu 672(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39205,10 +39176,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 704(%rdi), %ymm0 vmovdqu 736(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39218,10 +39186,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 768(%rdi), %ymm0 vmovdqu 800(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39231,10 +39196,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 832(%rdi), %ymm0 vmovdqu 864(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39244,10 +39206,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 896(%rdi), %ymm0 vmovdqu 928(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39257,10 +39216,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 vmovdqu 960(%rdi), %ymm0 vmovdqu 992(%rdi), %ymm1 vpcmpgtd %ymm2, %ymm0, %ymm4 @@ -39270,15 +39226,14 @@ L_mldsa_vec_check_low_vx2_start_256: vpor %ymm5, %ymm4, %ymm4 vpor %ymm7, %ymm6, %ymm6 vpor %ymm6, %ymm4, %ymm4 - vpmovmskb %ymm4, %rax - cmp $0x00, %rax - movq $0x00, %rax - jne L_mldsa_vec_check_low_vx2_done + vpor %ymm4, %ymm8, %ymm8 addq $0x400, %rdi sub $0x01, %rsi jne L_mldsa_vec_check_low_vx2_start_256 - movq $0x01, %rax -L_mldsa_vec_check_low_vx2_done: + vpmovmskb %ymm8, %rax + cmpl $0x00, %eax + sete %al + movzbl %al, %eax vzeroupper repz retq #ifndef __APPLE__ @@ -63501,6 +63456,7 @@ _wc_mldsa_vec_check_low_avx512: movd %edx, %xmm3 vpbroadcastd %xmm2, %zmm2 vpbroadcastd %xmm3, %zmm3 + kxorw %k5, %k5, %k5 L_mldsa_vec_check_low_avx512_start_256: vmovdqu64 (%rdi), %zmm0 vmovdqu64 64(%rdi), %zmm1 @@ -63511,9 +63467,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw %k3, %k1, %k1 korw %k4, %k2, %k2 korw %k2, %k1, %k1 - kortestw %k1, %k1 - movq $0x00, %rax - jne L_mldsa_vec_check_low_avx512_done + korw %k1, %k5, %k5 vmovdqu64 128(%rdi), %zmm0 vmovdqu64 192(%rdi), %zmm1 vpcmpgtd %zmm2, %zmm0, %k1 @@ -63523,9 +63477,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw %k3, %k1, %k1 korw %k4, %k2, %k2 korw %k2, %k1, %k1 - kortestw %k1, %k1 - movq $0x00, %rax - jne L_mldsa_vec_check_low_avx512_done + korw %k1, %k5, %k5 vmovdqu64 256(%rdi), %zmm0 vmovdqu64 320(%rdi), %zmm1 vpcmpgtd %zmm2, %zmm0, %k1 @@ -63535,9 +63487,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw %k3, %k1, %k1 korw %k4, %k2, %k2 korw %k2, %k1, %k1 - kortestw %k1, %k1 - movq $0x00, %rax - jne L_mldsa_vec_check_low_avx512_done + korw %k1, %k5, %k5 vmovdqu64 384(%rdi), %zmm0 vmovdqu64 448(%rdi), %zmm1 vpcmpgtd %zmm2, %zmm0, %k1 @@ -63547,9 +63497,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw %k3, %k1, %k1 korw %k4, %k2, %k2 korw %k2, %k1, %k1 - kortestw %k1, %k1 - movq $0x00, %rax - jne L_mldsa_vec_check_low_avx512_done + korw %k1, %k5, %k5 vmovdqu64 512(%rdi), %zmm0 vmovdqu64 576(%rdi), %zmm1 vpcmpgtd %zmm2, %zmm0, %k1 @@ -63559,9 +63507,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw %k3, %k1, %k1 korw %k4, %k2, %k2 korw %k2, %k1, %k1 - kortestw %k1, %k1 - movq $0x00, %rax - jne L_mldsa_vec_check_low_avx512_done + korw %k1, %k5, %k5 vmovdqu64 640(%rdi), %zmm0 vmovdqu64 704(%rdi), %zmm1 vpcmpgtd %zmm2, %zmm0, %k1 @@ -63571,9 +63517,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw %k3, %k1, %k1 korw %k4, %k2, %k2 korw %k2, %k1, %k1 - kortestw %k1, %k1 - movq $0x00, %rax - jne L_mldsa_vec_check_low_avx512_done + korw %k1, %k5, %k5 vmovdqu64 768(%rdi), %zmm0 vmovdqu64 832(%rdi), %zmm1 vpcmpgtd %zmm2, %zmm0, %k1 @@ -63583,9 +63527,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw %k3, %k1, %k1 korw %k4, %k2, %k2 korw %k2, %k1, %k1 - kortestw %k1, %k1 - movq $0x00, %rax - jne L_mldsa_vec_check_low_avx512_done + korw %k1, %k5, %k5 vmovdqu64 896(%rdi), %zmm0 vmovdqu64 960(%rdi), %zmm1 vpcmpgtd %zmm2, %zmm0, %k1 @@ -63595,14 +63537,13 @@ L_mldsa_vec_check_low_avx512_start_256: korw %k3, %k1, %k1 korw %k4, %k2, %k2 korw %k2, %k1, %k1 - kortestw %k1, %k1 - movq $0x00, %rax - jne L_mldsa_vec_check_low_avx512_done + korw %k1, %k5, %k5 addq $0x400, %rdi sub $0x01, %rsi jne L_mldsa_vec_check_low_avx512_start_256 - movq $0x01, %rax -L_mldsa_vec_check_low_avx512_done: + kortestw %k5, %k5 + sete %al + movzbl %al, %eax vzeroupper repz retq #ifndef __APPLE__ diff --git a/wolfcrypt/src/wc_mldsa_asm.asm b/wolfcrypt/src/wc_mldsa_asm.asm index ba23c544dea..d8b6d249a8b 100644 --- a/wolfcrypt/src/wc_mldsa_asm.asm +++ b/wolfcrypt/src/wc_mldsa_asm.asm @@ -38305,15 +38305,17 @@ wc_mldsa_use_hint_32_avx2 ENDP _TEXT ENDS _TEXT SEGMENT READONLY PARA wc_mldsa_vec_check_low_avx2 PROC - sub rsp, 40 + sub rsp, 56 vmovdqu OWORD PTR [rsp+8], xmm6 vmovdqu OWORD PTR [rsp+24], xmm7 + vmovdqu OWORD PTR [rsp+40], xmm8 sub r8d, 1 movd xmm2, r8d neg r8d movd xmm3, r8d vpbroadcastd ymm2, xmm2 vpbroadcastd ymm3, xmm3 + vpxor ymm8, ymm8, ymm8 L_mldsa_vec_check_low_vx2_start_256: vmovdqu ymm0, YMMWORD PTR [rcx] vmovdqu ymm1, YMMWORD PTR [rcx+32] @@ -38324,10 +38326,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+64] vmovdqu ymm1, YMMWORD PTR [rcx+96] vpcmpgtd ymm4, ymm0, ymm2 @@ -38337,10 +38336,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+128] vmovdqu ymm1, YMMWORD PTR [rcx+160] vpcmpgtd ymm4, ymm0, ymm2 @@ -38350,10 +38346,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+192] vmovdqu ymm1, YMMWORD PTR [rcx+224] vpcmpgtd ymm4, ymm0, ymm2 @@ -38363,10 +38356,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+256] vmovdqu ymm1, YMMWORD PTR [rcx+288] vpcmpgtd ymm4, ymm0, ymm2 @@ -38376,10 +38366,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+320] vmovdqu ymm1, YMMWORD PTR [rcx+352] vpcmpgtd ymm4, ymm0, ymm2 @@ -38389,10 +38376,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+384] vmovdqu ymm1, YMMWORD PTR [rcx+416] vpcmpgtd ymm4, ymm0, ymm2 @@ -38402,10 +38386,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+448] vmovdqu ymm1, YMMWORD PTR [rcx+480] vpcmpgtd ymm4, ymm0, ymm2 @@ -38415,10 +38396,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+512] vmovdqu ymm1, YMMWORD PTR [rcx+544] vpcmpgtd ymm4, ymm0, ymm2 @@ -38428,10 +38406,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+576] vmovdqu ymm1, YMMWORD PTR [rcx+608] vpcmpgtd ymm4, ymm0, ymm2 @@ -38441,10 +38416,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+640] vmovdqu ymm1, YMMWORD PTR [rcx+672] vpcmpgtd ymm4, ymm0, ymm2 @@ -38454,10 +38426,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+704] vmovdqu ymm1, YMMWORD PTR [rcx+736] vpcmpgtd ymm4, ymm0, ymm2 @@ -38467,10 +38436,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+768] vmovdqu ymm1, YMMWORD PTR [rcx+800] vpcmpgtd ymm4, ymm0, ymm2 @@ -38480,10 +38446,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+832] vmovdqu ymm1, YMMWORD PTR [rcx+864] vpcmpgtd ymm4, ymm0, ymm2 @@ -38493,10 +38456,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+896] vmovdqu ymm1, YMMWORD PTR [rcx+928] vpcmpgtd ymm4, ymm0, ymm2 @@ -38506,10 +38466,7 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 vmovdqu ymm0, YMMWORD PTR [rcx+960] vmovdqu ymm1, YMMWORD PTR [rcx+992] vpcmpgtd ymm4, ymm0, ymm2 @@ -38519,19 +38476,19 @@ L_mldsa_vec_check_low_vx2_start_256: vpor ymm4, ymm4, ymm5 vpor ymm6, ymm6, ymm7 vpor ymm4, ymm4, ymm6 - vpmovmskb rax, ymm4 - cmp rax, 0 - mov rax, 0 - jne L_mldsa_vec_check_low_vx2_done + vpor ymm8, ymm8, ymm4 add rcx, 1024 sub rdx, 1 jne L_mldsa_vec_check_low_vx2_start_256 - mov rax, 1 -L_mldsa_vec_check_low_vx2_done: + vpmovmskb rax, ymm8 + cmp eax, 0 + sete al + movzx eax, al vzeroupper vmovdqu xmm6, OWORD PTR [rsp+8] vmovdqu xmm7, OWORD PTR [rsp+24] - add rsp, 40 + vmovdqu xmm8, OWORD PTR [rsp+40] + add rsp, 56 ret wc_mldsa_vec_check_low_avx2 ENDP _TEXT ENDS @@ -62084,6 +62041,7 @@ wc_mldsa_vec_check_low_avx512 PROC movd xmm3, r8d vpbroadcastd zmm2, xmm2 vpbroadcastd zmm3, xmm3 + kxorw k5, k5, k5 L_mldsa_vec_check_low_avx512_start_256: vmovdqu64 zmm0, [rcx] vmovdqu64 zmm1, [rcx+64] @@ -62094,9 +62052,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw k1, k1, k3 korw k2, k2, k4 korw k1, k1, k2 - kortestw k1, k1 - mov rax, 0 - jne L_mldsa_vec_check_low_avx512_done + korw k5, k5, k1 vmovdqu64 zmm0, [rcx+128] vmovdqu64 zmm1, [rcx+192] vpcmpgtd k1, zmm0, zmm2 @@ -62106,9 +62062,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw k1, k1, k3 korw k2, k2, k4 korw k1, k1, k2 - kortestw k1, k1 - mov rax, 0 - jne L_mldsa_vec_check_low_avx512_done + korw k5, k5, k1 vmovdqu64 zmm0, [rcx+256] vmovdqu64 zmm1, [rcx+320] vpcmpgtd k1, zmm0, zmm2 @@ -62118,9 +62072,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw k1, k1, k3 korw k2, k2, k4 korw k1, k1, k2 - kortestw k1, k1 - mov rax, 0 - jne L_mldsa_vec_check_low_avx512_done + korw k5, k5, k1 vmovdqu64 zmm0, [rcx+384] vmovdqu64 zmm1, [rcx+448] vpcmpgtd k1, zmm0, zmm2 @@ -62130,9 +62082,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw k1, k1, k3 korw k2, k2, k4 korw k1, k1, k2 - kortestw k1, k1 - mov rax, 0 - jne L_mldsa_vec_check_low_avx512_done + korw k5, k5, k1 vmovdqu64 zmm0, [rcx+512] vmovdqu64 zmm1, [rcx+576] vpcmpgtd k1, zmm0, zmm2 @@ -62142,9 +62092,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw k1, k1, k3 korw k2, k2, k4 korw k1, k1, k2 - kortestw k1, k1 - mov rax, 0 - jne L_mldsa_vec_check_low_avx512_done + korw k5, k5, k1 vmovdqu64 zmm0, [rcx+640] vmovdqu64 zmm1, [rcx+704] vpcmpgtd k1, zmm0, zmm2 @@ -62154,9 +62102,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw k1, k1, k3 korw k2, k2, k4 korw k1, k1, k2 - kortestw k1, k1 - mov rax, 0 - jne L_mldsa_vec_check_low_avx512_done + korw k5, k5, k1 vmovdqu64 zmm0, [rcx+768] vmovdqu64 zmm1, [rcx+832] vpcmpgtd k1, zmm0, zmm2 @@ -62166,9 +62112,7 @@ L_mldsa_vec_check_low_avx512_start_256: korw k1, k1, k3 korw k2, k2, k4 korw k1, k1, k2 - kortestw k1, k1 - mov rax, 0 - jne L_mldsa_vec_check_low_avx512_done + korw k5, k5, k1 vmovdqu64 zmm0, [rcx+896] vmovdqu64 zmm1, [rcx+960] vpcmpgtd k1, zmm0, zmm2 @@ -62178,14 +62122,13 @@ L_mldsa_vec_check_low_avx512_start_256: korw k1, k1, k3 korw k2, k2, k4 korw k1, k1, k2 - kortestw k1, k1 - mov rax, 0 - jne L_mldsa_vec_check_low_avx512_done + korw k5, k5, k1 add rcx, 1024 sub rdx, 1 jne L_mldsa_vec_check_low_avx512_start_256 - mov rax, 1 -L_mldsa_vec_check_low_avx512_done: + kortestw k5, k5 + sete al + movzx eax, al vzeroupper ret wc_mldsa_vec_check_low_avx512 ENDP