From e4952a48056677f83bd6c239857b07ba1035209c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Frauenschl=C3=A4ger?= Date: Thu, 12 Nov 2026 11:14:36 +0100 Subject: [PATCH 1/7] Fix the ML-DSA small memory signer build with clang The non-blocking resume check added a WC_MLDSA_NONBLOCK-only clause to the rho'' derivation condition. Without non-blocking support the condition reduced to if ((ret == 0)), which clang rejects with -Werror,-Wparentheses-equality. --- wolfcrypt/src/wc_mldsa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index d80f3ebae57..4ecf52aa4bb 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -13014,7 +13014,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, } } - if ((ret == 0) + if (ret == 0 #ifdef WC_MLDSA_NONBLOCK /* Resuming part way through an attempt: its y is still in the working * allocation, so the seed it is expanded from is not needed. */ From e4a1a57486a4671e4b07212591d7cca4072b23c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Frauenschl=C3=A4ger?= Date: Thu, 12 Nov 2026 11:53:07 +0100 Subject: [PATCH 2/7] Leave the arm ML-DSA gen_y lanes out of the smallest memory signer The smallest memory signer expands y one polynomial at a time and never builds mldsa_vec_expand_mask(), the only caller of the batched NEON lanes wc_mldsa_gen_y_4/5/7_arm. With --enable-armasm they were still compiled and failed -Werror,-Wunused-function. Guard them like the AVX2 lanes, which already exclude WOLFSSL_MLDSA_SIGN_SMALLEST_MEM, and drop the smallest memory signer from the conditions of the SHA-3 instruction wrappers mlkem_sha3_blocksx3() and mlkem_shake256_blocksx3_seed_64(), which those lanes call. A pq-all row builds AArch64 NEON with sha3-crypto, the smallest memory signer and verifier, small memory key generation and no key check, the combination that leaves all three lane users out, and runs testwolfcrypt under qemu. --- .github/configs/pq-all.json | 8 ++++++++ wolfcrypt/src/wc_mldsa.c | 9 ++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/configs/pq-all.json b/.github/configs/pq-all.json index 0bc09c3612f..a4482733d6c 100644 --- a/.github/configs/pq-all.json +++ b/.github/configs/pq-all.json @@ -361,6 +361,14 @@ "--disable-examples"], "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "QEMU_CPU=max", "./wolfcrypt/test/testwolfcrypt"]]}, +{"name": "mldsa-aarch64-armasm-smallest", "minutes": 0.5, "check": false, + "cc": "ccache aarch64-linux-gnu-gcc", + "comment": "AArch64 NEON with the smallest memory signer and verifier and the small memory key generation, which leave the batched gen_y and gen_s lanes out. sha3-crypto builds the SHA-3 instruction wrappers as functions, and without the key check matrix A expansion does not use them either", + "configure": ["--host=aarch64-linux-gnu", "--enable-cryptonly", + "--enable-armasm=sha3-crypto", "--enable-mldsa", "--disable-examples", + "CPPFLAGS=-DWOLFSSL_MLDSA_SIGN_SMALLEST_MEM -DWOLFSSL_MLDSA_VERIFY_SMALLEST_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_CHECK_KEY"], + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "QEMU_CPU=max", + "./wolfcrypt/test/testwolfcrypt"]]}, {"name": "mldsa-mlkem-aarch64-armasm-inline", "minutes": 0.5, "check": false, "cc": "ccache aarch64-linux-gnu-gcc", "comment": "Same, through the inline-asm backend.", diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index 4ecf52aa4bb..93889237089 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -551,7 +551,8 @@ static WC_INLINE cpuid_flags_t mldsa_cpuid_flags(void) /* Expanding a, s (make key) and y (sign) all squeeze more blocks. */ #if defined(MLDSA_NEON_EXPAND_A) || !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ - !defined(WOLFSSL_MLDSA_NO_SIGN) + (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ + !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM)) static void mlkem_sha3_blocksx3(word64* state) { if (IS_AARCH64_SHA3(mldsa_cpuid_flags())) { @@ -576,7 +577,9 @@ static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) #endif /* Only expanding s (make key) and y (sign) use the 64-byte seed. */ -#if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || !defined(WOLFSSL_MLDSA_NO_SIGN) +#if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ + (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ + !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM)) static void mlkem_shake256_blocksx3_seed_64(word64* state, byte* seed) { if (IS_AARCH64_SHA3(mldsa_cpuid_flags())) { @@ -6734,7 +6737,7 @@ static int wc_mldsa_gen_y_avx512(sword32* y, byte* seed, word16 kappa, return 0; } #endif /* WOLFSSL_MLDSA_HAVE_INTEL_AVX512 */ -#elif defined(MLDSA_ASM_BLOCKSX3) +#elif defined(MLDSA_ASM_BLOCKSX3) && !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM) /* Decode polynomial with range -(GAMMA1-1)..GAMMA1 inside a gen_y lane. * * The lane's caller already holds the vector registers, so this calls the From be43081d2ba5bfaac02099e2d50b42f5e464586d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Frauenschl=C3=A4ger?= Date: Mon, 9 Nov 2026 13:27:59 +0100 Subject: [PATCH 3/7] Store exact lengths in the AVX2 ML-DSA vector encoders The AVX2 eta and t0/t1 vector encoders stored 16 bytes for each 12, 13 or 10 they produced, so the last store of a vector ran 4, 3 or 6 bytes past its end. The eta-4 encoder also took no count and always encoded six polynomials, which for the five polynomials of an ML-DSA-65 s1 wrote a whole polynomial past it. Callers only got away with it through the order they encode in and the padding on the key buffers. Regenerate wc_mldsa_asm.S and wc_mldsa_asm.asm from the scripts generator, which now stores the last chunk of each polynomial exactly and gives the eta-4 encoder a polynomial count, and pass the count. New test-only wrappers let an API test check each vector length against a reference encoding with guard bytes after the output. --- tests/api/test_mldsa.c | 146 +++++ tests/api/test_mldsa.h | 2 + wolfcrypt/src/wc_mldsa.c | 14 +- wolfcrypt/src/wc_mldsa_asm.S | 1010 ++++---------------------------- wolfcrypt/src/wc_mldsa_asm.asm | 980 +++---------------------------- wolfssl/wolfcrypt/wc_mldsa.h | 10 +- 6 files changed, 353 insertions(+), 1809 deletions(-) diff --git a/tests/api/test_mldsa.c b/tests/api/test_mldsa.c index 6a89e617983..ea84c8b156e 100644 --- a/tests/api/test_mldsa.c +++ b/tests/api/test_mldsa.c @@ -31684,6 +31684,152 @@ int test_mldsa_pkcs8_export_import_wolfSSL_form(void) return EXPECT_RESULT(); } +#if defined(WOLFSSL_HAVE_MLDSA) && defined(WC_MLDSA_HAVE_NATIVE) && \ + !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) +#define MLDSA_TEST_ENC_MAX_K 8 +#define MLDSA_TEST_ENC_GUARD 32 +#define MLDSA_TEST_ENC_SZ \ + (MLDSA_TEST_ENC_MAX_K * MLDSA_N * MLDSA_D / 8 + MLDSA_TEST_ENC_GUARD) + +/* Pack n values of bits bits each, little-endian, as FIPS 204 does. */ +static void test_mldsa_pack_bits(const sword32* v, word32 n, word32 bits, + byte* out) +{ + word32 i; + word32 accBits = 0; + word64 acc = 0; + + for (i = 0; i < n; i++) { + acc |= (word64)(word32)v[i] << accBits; + accBits += bits; + while (accBits >= 8) { + *(out++) = (byte)acc; + acc >>= 8; + accBits -= 8; + } + } +} + +static int test_mldsa_guard_intact(const byte* p) +{ + int i; + + for (i = 0; i < MLDSA_TEST_ENC_GUARD; i++) { + if (p[i] != 0xa5) { + return 0; + } + } + return 1; +} + +static word32 test_mldsa_rand(word32* state) +{ + *state = *state * 1103515245U + 12345U; + return *state >> 8; +} +#endif + +/* The vector encoders write exactly the encoded length, for every count. */ +int test_mldsa_vec_encode_exact_len(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_HAVE_MLDSA) && defined(WC_MLDSA_HAVE_NATIVE) && \ + !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) + static const byte etas[] = { + #if !defined(WOLFSSL_NO_ML_DSA_44) || !defined(WOLFSSL_NO_ML_DSA_87) + 2, + #endif + #ifndef WOLFSSL_NO_ML_DSA_65 + 4, + #endif + }; + sword32* s = NULL; + sword32* v = NULL; + byte* enc = NULL; + byte* enc1 = NULL; + byte* exp = NULL; + word32 rnd = 1; + word32 len; + word32 len1; + word32 i; + word32 e; + byte d; + + ExpectNotNull(s = (sword32*)XMALLOC(MLDSA_TEST_ENC_MAX_K * MLDSA_N * + sizeof(sword32), NULL, DYNAMIC_TYPE_TMP_BUFFER)); + ExpectNotNull(v = (sword32*)XMALLOC(MLDSA_TEST_ENC_MAX_K * MLDSA_N * + sizeof(sword32), NULL, DYNAMIC_TYPE_TMP_BUFFER)); + ExpectNotNull(enc = (byte*)XMALLOC(MLDSA_TEST_ENC_SZ, NULL, + DYNAMIC_TYPE_TMP_BUFFER)); + ExpectNotNull(enc1 = (byte*)XMALLOC(MLDSA_TEST_ENC_SZ, NULL, + DYNAMIC_TYPE_TMP_BUFFER)); + ExpectNotNull(exp = (byte*)XMALLOC(MLDSA_TEST_ENC_SZ, NULL, + DYNAMIC_TYPE_TMP_BUFFER)); + +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && \ + defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); +#endif + + /* Every count, so the last store of each vector length is checked. */ + for (e = 0; EXPECT_SUCCESS() && (e < sizeof(etas)); e++) { + byte eta = etas[e]; + word32 bits = (eta == 2) ? 3 : 4; + + for (d = 1; EXPECT_SUCCESS() && (d <= MLDSA_TEST_ENC_MAX_K); d++) { + len = (word32)d * MLDSA_N * bits / 8; + for (i = 0; i < (word32)d * MLDSA_N; i++) { + s[i] = (sword32)(test_mldsa_rand(&rnd) % (2U * eta + 1)) - + eta; + v[i] = eta - s[i]; + } + test_mldsa_pack_bits(v, (word32)d * MLDSA_N, bits, exp); + XMEMSET(enc, 0xa5, MLDSA_TEST_ENC_SZ); + ExpectIntEQ(wc_mldsa_vec_encode_eta(s, d, eta, enc), 0); + ExpectBufEQ(enc, exp, len); + ExpectIntEQ(test_mldsa_guard_intact(enc + len), 1); + } + } + + for (d = 1; EXPECT_SUCCESS() && (d <= MLDSA_TEST_ENC_MAX_K); d++) { + len = (word32)d * MLDSA_N * MLDSA_D / 8; + len1 = (word32)d * MLDSA_N * MLDSA_U / 8; + for (i = 0; i < (word32)d * MLDSA_N; i++) { + s[i] = (sword32)(test_mldsa_rand(&rnd) % MLDSA_Q); + } + /* Power2Round: t1 in the top bits, 2^(d-1) - t0 in the low ones. */ + for (i = 0; i < (word32)d * MLDSA_N; i++) { + sword32 t1 = (s[i] + (1 << (MLDSA_D - 1)) - 1) >> MLDSA_D; + v[i] = (1 << (MLDSA_D - 1)) - (s[i] - (t1 << MLDSA_D)); + } + test_mldsa_pack_bits(v, (word32)d * MLDSA_N, MLDSA_D, exp); + XMEMSET(enc, 0xa5, MLDSA_TEST_ENC_SZ); + XMEMSET(enc1, 0xa5, MLDSA_TEST_ENC_SZ); + ExpectIntEQ(wc_mldsa_vec_encode_t0_t1(s, d, enc, enc1), 0); + ExpectBufEQ(enc, exp, len); + ExpectIntEQ(test_mldsa_guard_intact(enc + len), 1); + for (i = 0; i < (word32)d * MLDSA_N; i++) { + v[i] = (s[i] + (1 << (MLDSA_D - 1)) - 1) >> MLDSA_D; + } + test_mldsa_pack_bits(v, (word32)d * MLDSA_N, MLDSA_U, exp); + ExpectBufEQ(enc1, exp, len1); + ExpectIntEQ(test_mldsa_guard_intact(enc1 + len1), 1); + } + +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && \ + defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); +#endif + + XFREE(exp, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(enc1, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(enc, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(v, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(s, NULL, DYNAMIC_TYPE_TMP_BUFFER); +#endif + return EXPECT_RESULT(); +} + /* Exercise w1 encoding with adversarial inputs to catch undefined behavior. * * The word32-optimized paths in mldsa_encode_w1_88_c and diff --git a/tests/api/test_mldsa.h b/tests/api/test_mldsa.h index 74f212d9451..9fc042bab8b 100644 --- a/tests/api/test_mldsa.h +++ b/tests/api/test_mldsa.h @@ -70,6 +70,7 @@ int test_mldsa_verify_mu_kats(void); int test_mldsa_PrivateKeyDecode_OpenSSL_form(void); int test_mldsa_pkcs8_import_OpenSSL_form(void); int test_mldsa_pkcs8_export_import_wolfSSL_form(void); +int test_mldsa_vec_encode_exact_len(void); int test_mldsa_encode_w1_large_values(void); int test_mldsa_pkcs12(void); int test_mldsa_x509_pubkey_sigtype(void); @@ -132,6 +133,7 @@ int test_mldsa_make_key_id_dev(void); TEST_DECL_GROUP("mldsa", test_mldsa_PrivateKeyDecode_OpenSSL_form), \ TEST_DECL_GROUP("mldsa", test_mldsa_pkcs8_import_OpenSSL_form), \ TEST_DECL_GROUP("mldsa", test_mldsa_pkcs8_export_import_wolfSSL_form), \ + TEST_DECL_GROUP("mldsa", test_mldsa_vec_encode_exact_len), \ TEST_DECL_GROUP("mldsa", test_mldsa_encode_w1_large_values), \ TEST_DECL_GROUP("mldsa", test_mldsa_pkcs12), \ TEST_DECL_GROUP("mldsa", test_mldsa_x509_pubkey_sigtype), \ diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index 93889237089..afdbec0df44 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -1861,7 +1861,7 @@ static int mldsa_vec_encode_eta_bits(const sword32* s, byte d, byte eta, #endif #ifndef WOLFSSL_NO_ML_DSA_65 if (eta == MLDSA_ETA_4) { - wc_mldsa_vec_encode_eta_4_avx2(s, p); + wc_mldsa_vec_encode_eta_4_avx2(s, d, p); } #endif RESTORE_VECTOR_REGISTERS(); @@ -1873,6 +1873,12 @@ static int mldsa_vec_encode_eta_bits(const sword32* s, byte d, byte eta, } return ret; } + +WOLFSSL_TEST_VIS int wc_mldsa_vec_encode_eta(const sword32* s, byte d, + byte eta, byte* p) +{ + return mldsa_vec_encode_eta_bits(s, d, eta, p); +} #endif /* !WOLFSSL_MLDSA_NO_MAKE_KEY */ #if !defined(WOLFSSL_MLDSA_NO_SIGN) || defined(WOLFSSL_MLDSA_CHECK_KEY) || \ @@ -2343,6 +2349,12 @@ static int mldsa_vec_encode_t0_t1(const sword32* t, byte d, byte* t0, } return ret; } + +WOLFSSL_TEST_VIS int wc_mldsa_vec_encode_t0_t1(const sword32* t, byte d, + byte* t0, byte* t1) +{ + return mldsa_vec_encode_t0_t1(t, d, t0, t1); +} #endif /* !WOLFSSL_MLDSA_NO_MAKE_KEY */ #if !defined(WOLFSSL_MLDSA_NO_SIGN) || defined(WOLFSSL_MLDSA_CHECK_KEY) diff --git a/wolfcrypt/src/wc_mldsa_asm.S b/wolfcrypt/src/wc_mldsa_asm.S index 89e71b21a54..ad0ea56f15d 100644 --- a/wolfcrypt/src/wc_mldsa_asm.S +++ b/wolfcrypt/src/wc_mldsa_asm.S @@ -23685,887 +23685,68 @@ L_mldsa_encode_eta_2_avx2_loop: vpor %ymm4, %ymm0, %ymm0 vextracti128 $0x01, %ymm0, %xmm4 vpor %ymm4, %ymm0, %ymm0 - vmovdqu %xmm0, (%rdx) - addq $12, %rdx - addq $0x400, %rdi - decb %sil - jnz L_mldsa_encode_eta_2_avx2_loop - vzeroupper - repz retq -#ifndef __APPLE__ -.size wc_mldsa_vec_encode_eta_2_avx2,.-wc_mldsa_vec_encode_eta_2_avx2 -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.data -#else -.section __DATA,__data -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.align 16 -#else -.p2align 4 -#endif /* __APPLE__ */ -L_mldsa_encode_eta_4_avx2_four: -.long 0x00000004,0x00000004,0x00000004,0x00000004 -.long 0x00000004,0x00000004,0x00000004,0x00000004 -#ifndef __APPLE__ -.data -#else -.section __DATA,__data -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.align 16 -#else -.p2align 4 -#endif /* __APPLE__ */ -L_mldsa_encode_eta_4_avx2_vs_4: -.long 0x00000000,0x00000004,0x00000000,0x00000004 -.long 0x00000000,0x00000004,0x00000000,0x00000004 -#ifndef __APPLE__ -.text -.globl wc_mldsa_vec_encode_eta_4_avx2 -WC_ASM_ATT_HIDDEN(wc_mldsa_vec_encode_eta_4_avx2) -.type wc_mldsa_vec_encode_eta_4_avx2,@function -.align 16 -wc_mldsa_vec_encode_eta_4_avx2: -#else -.section __TEXT,__text -.globl _wc_mldsa_vec_encode_eta_4_avx2 -WC_ASM_ATT_HIDDEN(_wc_mldsa_vec_encode_eta_4_avx2) -.p2align 4 -_wc_mldsa_vec_encode_eta_4_avx2: -#endif /* __APPLE__ */ - vpxor %ymm8, %ymm8, %ymm8 - vmovdqu L_mldsa_encode_eta_4_avx2_four(%rip), %ymm8 - vmovdqu L_mldsa_encode_eta_4_avx2_vs_4(%rip), %ymm9 - vmovdqu (%rdi), %ymm0 - vmovdqu 32(%rdi), %ymm1 - vmovdqu 64(%rdi), %ymm2 - vmovdqu 96(%rdi), %ymm3 - vmovdqu 128(%rdi), %ymm4 - vmovdqu 160(%rdi), %ymm5 - vmovdqu 192(%rdi), %ymm6 - vmovdqu 224(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, (%rsi) - vmovdqu 256(%rdi), %ymm0 - vmovdqu 288(%rdi), %ymm1 - vmovdqu 320(%rdi), %ymm2 - vmovdqu 352(%rdi), %ymm3 - vmovdqu 384(%rdi), %ymm4 - vmovdqu 416(%rdi), %ymm5 - vmovdqu 448(%rdi), %ymm6 - vmovdqu 480(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 32(%rsi) - vmovdqu 512(%rdi), %ymm0 - vmovdqu 544(%rdi), %ymm1 - vmovdqu 576(%rdi), %ymm2 - vmovdqu 608(%rdi), %ymm3 - vmovdqu 640(%rdi), %ymm4 - vmovdqu 672(%rdi), %ymm5 - vmovdqu 704(%rdi), %ymm6 - vmovdqu 736(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 64(%rsi) - vmovdqu 768(%rdi), %ymm0 - vmovdqu 800(%rdi), %ymm1 - vmovdqu 832(%rdi), %ymm2 - vmovdqu 864(%rdi), %ymm3 - vmovdqu 896(%rdi), %ymm4 - vmovdqu 928(%rdi), %ymm5 - vmovdqu 960(%rdi), %ymm6 - vmovdqu 992(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 96(%rsi) - vmovdqu 1024(%rdi), %ymm0 - vmovdqu 1056(%rdi), %ymm1 - vmovdqu 1088(%rdi), %ymm2 - vmovdqu 1120(%rdi), %ymm3 - vmovdqu 1152(%rdi), %ymm4 - vmovdqu 1184(%rdi), %ymm5 - vmovdqu 1216(%rdi), %ymm6 - vmovdqu 1248(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 128(%rsi) - vmovdqu 1280(%rdi), %ymm0 - vmovdqu 1312(%rdi), %ymm1 - vmovdqu 1344(%rdi), %ymm2 - vmovdqu 1376(%rdi), %ymm3 - vmovdqu 1408(%rdi), %ymm4 - vmovdqu 1440(%rdi), %ymm5 - vmovdqu 1472(%rdi), %ymm6 - vmovdqu 1504(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 160(%rsi) - vmovdqu 1536(%rdi), %ymm0 - vmovdqu 1568(%rdi), %ymm1 - vmovdqu 1600(%rdi), %ymm2 - vmovdqu 1632(%rdi), %ymm3 - vmovdqu 1664(%rdi), %ymm4 - vmovdqu 1696(%rdi), %ymm5 - vmovdqu 1728(%rdi), %ymm6 - vmovdqu 1760(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 192(%rsi) - vmovdqu 1792(%rdi), %ymm0 - vmovdqu 1824(%rdi), %ymm1 - vmovdqu 1856(%rdi), %ymm2 - vmovdqu 1888(%rdi), %ymm3 - vmovdqu 1920(%rdi), %ymm4 - vmovdqu 1952(%rdi), %ymm5 - vmovdqu 1984(%rdi), %ymm6 - vmovdqu 2016(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 224(%rsi) - vmovdqu 2048(%rdi), %ymm0 - vmovdqu 2080(%rdi), %ymm1 - vmovdqu 2112(%rdi), %ymm2 - vmovdqu 2144(%rdi), %ymm3 - vmovdqu 2176(%rdi), %ymm4 - vmovdqu 2208(%rdi), %ymm5 - vmovdqu 2240(%rdi), %ymm6 - vmovdqu 2272(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 256(%rsi) - vmovdqu 2304(%rdi), %ymm0 - vmovdqu 2336(%rdi), %ymm1 - vmovdqu 2368(%rdi), %ymm2 - vmovdqu 2400(%rdi), %ymm3 - vmovdqu 2432(%rdi), %ymm4 - vmovdqu 2464(%rdi), %ymm5 - vmovdqu 2496(%rdi), %ymm6 - vmovdqu 2528(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 288(%rsi) - vmovdqu 2560(%rdi), %ymm0 - vmovdqu 2592(%rdi), %ymm1 - vmovdqu 2624(%rdi), %ymm2 - vmovdqu 2656(%rdi), %ymm3 - vmovdqu 2688(%rdi), %ymm4 - vmovdqu 2720(%rdi), %ymm5 - vmovdqu 2752(%rdi), %ymm6 - vmovdqu 2784(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 320(%rsi) - vmovdqu 2816(%rdi), %ymm0 - vmovdqu 2848(%rdi), %ymm1 - vmovdqu 2880(%rdi), %ymm2 - vmovdqu 2912(%rdi), %ymm3 - vmovdqu 2944(%rdi), %ymm4 - vmovdqu 2976(%rdi), %ymm5 - vmovdqu 3008(%rdi), %ymm6 - vmovdqu 3040(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 352(%rsi) - vmovdqu 3072(%rdi), %ymm0 - vmovdqu 3104(%rdi), %ymm1 - vmovdqu 3136(%rdi), %ymm2 - vmovdqu 3168(%rdi), %ymm3 - vmovdqu 3200(%rdi), %ymm4 - vmovdqu 3232(%rdi), %ymm5 - vmovdqu 3264(%rdi), %ymm6 - vmovdqu 3296(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 384(%rsi) - vmovdqu 3328(%rdi), %ymm0 - vmovdqu 3360(%rdi), %ymm1 - vmovdqu 3392(%rdi), %ymm2 - vmovdqu 3424(%rdi), %ymm3 - vmovdqu 3456(%rdi), %ymm4 - vmovdqu 3488(%rdi), %ymm5 - vmovdqu 3520(%rdi), %ymm6 - vmovdqu 3552(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 416(%rsi) - vmovdqu 3584(%rdi), %ymm0 - vmovdqu 3616(%rdi), %ymm1 - vmovdqu 3648(%rdi), %ymm2 - vmovdqu 3680(%rdi), %ymm3 - vmovdqu 3712(%rdi), %ymm4 - vmovdqu 3744(%rdi), %ymm5 - vmovdqu 3776(%rdi), %ymm6 - vmovdqu 3808(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 448(%rsi) - vmovdqu 3840(%rdi), %ymm0 - vmovdqu 3872(%rdi), %ymm1 - vmovdqu 3904(%rdi), %ymm2 - vmovdqu 3936(%rdi), %ymm3 - vmovdqu 3968(%rdi), %ymm4 - vmovdqu 4000(%rdi), %ymm5 - vmovdqu 4032(%rdi), %ymm6 - vmovdqu 4064(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 480(%rsi) - vmovdqu 4096(%rdi), %ymm0 - vmovdqu 4128(%rdi), %ymm1 - vmovdqu 4160(%rdi), %ymm2 - vmovdqu 4192(%rdi), %ymm3 - vmovdqu 4224(%rdi), %ymm4 - vmovdqu 4256(%rdi), %ymm5 - vmovdqu 4288(%rdi), %ymm6 - vmovdqu 4320(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 512(%rsi) - vmovdqu 4352(%rdi), %ymm0 - vmovdqu 4384(%rdi), %ymm1 - vmovdqu 4416(%rdi), %ymm2 - vmovdqu 4448(%rdi), %ymm3 - vmovdqu 4480(%rdi), %ymm4 - vmovdqu 4512(%rdi), %ymm5 - vmovdqu 4544(%rdi), %ymm6 - vmovdqu 4576(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 544(%rsi) - vmovdqu 4608(%rdi), %ymm0 - vmovdqu 4640(%rdi), %ymm1 - vmovdqu 4672(%rdi), %ymm2 - vmovdqu 4704(%rdi), %ymm3 - vmovdqu 4736(%rdi), %ymm4 - vmovdqu 4768(%rdi), %ymm5 - vmovdqu 4800(%rdi), %ymm6 - vmovdqu 4832(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 576(%rsi) - vmovdqu 4864(%rdi), %ymm0 - vmovdqu 4896(%rdi), %ymm1 - vmovdqu 4928(%rdi), %ymm2 - vmovdqu 4960(%rdi), %ymm3 - vmovdqu 4992(%rdi), %ymm4 - vmovdqu 5024(%rdi), %ymm5 - vmovdqu 5056(%rdi), %ymm6 - vmovdqu 5088(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 608(%rsi) - vmovdqu 5120(%rdi), %ymm0 - vmovdqu 5152(%rdi), %ymm1 - vmovdqu 5184(%rdi), %ymm2 - vmovdqu 5216(%rdi), %ymm3 - vmovdqu 5248(%rdi), %ymm4 - vmovdqu 5280(%rdi), %ymm5 - vmovdqu 5312(%rdi), %ymm6 - vmovdqu 5344(%rdi), %ymm7 + vmovq %xmm0, (%rdx) + vpextrd $2, %xmm0, 8(%rdx) + addq $12, %rdx + addq $0x400, %rdi + decb %sil + jnz L_mldsa_encode_eta_2_avx2_loop + vzeroupper + repz retq +#ifndef __APPLE__ +.size wc_mldsa_vec_encode_eta_2_avx2,.-wc_mldsa_vec_encode_eta_2_avx2 +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.data +#else +.section __DATA,__data +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.align 16 +#else +.p2align 4 +#endif /* __APPLE__ */ +L_mldsa_encode_eta_4_avx2_four: +.long 0x00000004,0x00000004,0x00000004,0x00000004 +.long 0x00000004,0x00000004,0x00000004,0x00000004 +#ifndef __APPLE__ +.data +#else +.section __DATA,__data +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.align 16 +#else +.p2align 4 +#endif /* __APPLE__ */ +L_mldsa_encode_eta_4_avx2_vs_4: +.long 0x00000000,0x00000004,0x00000000,0x00000004 +.long 0x00000000,0x00000004,0x00000000,0x00000004 +#ifndef __APPLE__ +.text +.globl wc_mldsa_vec_encode_eta_4_avx2 +WC_ASM_ATT_HIDDEN(wc_mldsa_vec_encode_eta_4_avx2) +.type wc_mldsa_vec_encode_eta_4_avx2,@function +.align 16 +wc_mldsa_vec_encode_eta_4_avx2: +#else +.section __TEXT,__text +.globl _wc_mldsa_vec_encode_eta_4_avx2 +WC_ASM_ATT_HIDDEN(_wc_mldsa_vec_encode_eta_4_avx2) +.p2align 4 +_wc_mldsa_vec_encode_eta_4_avx2: +#endif /* __APPLE__ */ + vmovdqu L_mldsa_encode_eta_4_avx2_four(%rip), %ymm8 + vmovdqu L_mldsa_encode_eta_4_avx2_vs_4(%rip), %ymm9 +L_mldsa_encode_eta_4_avx2_loop: + vmovdqu (%rdi), %ymm0 + vmovdqu 32(%rdi), %ymm1 + vmovdqu 64(%rdi), %ymm2 + vmovdqu 96(%rdi), %ymm3 + vmovdqu 128(%rdi), %ymm4 + vmovdqu 160(%rdi), %ymm5 + vmovdqu 192(%rdi), %ymm6 + vmovdqu 224(%rdi), %ymm7 vpsubd %ymm0, %ymm8, %ymm0 vpsubd %ymm1, %ymm8, %ymm1 vpsubd %ymm2, %ymm8, %ymm2 @@ -24598,15 +23779,15 @@ _wc_mldsa_vec_encode_eta_4_avx2: vpermq $0xd8, %ymm1, %ymm1 vpackuswb %ymm1, %ymm0, %ymm0 vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 640(%rsi) - vmovdqu 5376(%rdi), %ymm0 - vmovdqu 5408(%rdi), %ymm1 - vmovdqu 5440(%rdi), %ymm2 - vmovdqu 5472(%rdi), %ymm3 - vmovdqu 5504(%rdi), %ymm4 - vmovdqu 5536(%rdi), %ymm5 - vmovdqu 5568(%rdi), %ymm6 - vmovdqu 5600(%rdi), %ymm7 + vmovdqu %ymm0, (%rdx) + vmovdqu 256(%rdi), %ymm0 + vmovdqu 288(%rdi), %ymm1 + vmovdqu 320(%rdi), %ymm2 + vmovdqu 352(%rdi), %ymm3 + vmovdqu 384(%rdi), %ymm4 + vmovdqu 416(%rdi), %ymm5 + vmovdqu 448(%rdi), %ymm6 + vmovdqu 480(%rdi), %ymm7 vpsubd %ymm0, %ymm8, %ymm0 vpsubd %ymm1, %ymm8, %ymm1 vpsubd %ymm2, %ymm8, %ymm2 @@ -24639,15 +23820,15 @@ _wc_mldsa_vec_encode_eta_4_avx2: vpermq $0xd8, %ymm1, %ymm1 vpackuswb %ymm1, %ymm0, %ymm0 vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 672(%rsi) - vmovdqu 5632(%rdi), %ymm0 - vmovdqu 5664(%rdi), %ymm1 - vmovdqu 5696(%rdi), %ymm2 - vmovdqu 5728(%rdi), %ymm3 - vmovdqu 5760(%rdi), %ymm4 - vmovdqu 5792(%rdi), %ymm5 - vmovdqu 5824(%rdi), %ymm6 - vmovdqu 5856(%rdi), %ymm7 + vmovdqu %ymm0, 32(%rdx) + vmovdqu 512(%rdi), %ymm0 + vmovdqu 544(%rdi), %ymm1 + vmovdqu 576(%rdi), %ymm2 + vmovdqu 608(%rdi), %ymm3 + vmovdqu 640(%rdi), %ymm4 + vmovdqu 672(%rdi), %ymm5 + vmovdqu 704(%rdi), %ymm6 + vmovdqu 736(%rdi), %ymm7 vpsubd %ymm0, %ymm8, %ymm0 vpsubd %ymm1, %ymm8, %ymm1 vpsubd %ymm2, %ymm8, %ymm2 @@ -24680,15 +23861,15 @@ _wc_mldsa_vec_encode_eta_4_avx2: vpermq $0xd8, %ymm1, %ymm1 vpackuswb %ymm1, %ymm0, %ymm0 vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 704(%rsi) - vmovdqu 5888(%rdi), %ymm0 - vmovdqu 5920(%rdi), %ymm1 - vmovdqu 5952(%rdi), %ymm2 - vmovdqu 5984(%rdi), %ymm3 - vmovdqu 6016(%rdi), %ymm4 - vmovdqu 6048(%rdi), %ymm5 - vmovdqu 6080(%rdi), %ymm6 - vmovdqu 6112(%rdi), %ymm7 + vmovdqu %ymm0, 64(%rdx) + vmovdqu 768(%rdi), %ymm0 + vmovdqu 800(%rdi), %ymm1 + vmovdqu 832(%rdi), %ymm2 + vmovdqu 864(%rdi), %ymm3 + vmovdqu 896(%rdi), %ymm4 + vmovdqu 928(%rdi), %ymm5 + vmovdqu 960(%rdi), %ymm6 + vmovdqu 992(%rdi), %ymm7 vpsubd %ymm0, %ymm8, %ymm0 vpsubd %ymm1, %ymm8, %ymm1 vpsubd %ymm2, %ymm8, %ymm2 @@ -24721,7 +23902,11 @@ _wc_mldsa_vec_encode_eta_4_avx2: vpermq $0xd8, %ymm1, %ymm1 vpackuswb %ymm1, %ymm0, %ymm0 vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 736(%rsi) + vmovdqu %ymm0, 96(%rdx) + addq $0x400, %rdi + addq $0x80, %rdx + decb %sil + jnz L_mldsa_encode_eta_4_avx2_loop vzeroupper repz retq #ifndef __APPLE__ @@ -26808,7 +25993,9 @@ L_mldsa_encode_t0_t1_avx2_loop: vpor %ymm1, %ymm3, %ymm3 vmovdqu %xmm2, (%rdx) addq $13, %rdx - vmovdqu %xmm3, (%rdx) + vmovq %xmm3, (%rdx) + vpextrd $2, %xmm3, 8(%rdx) + vpextrb $12, %xmm3, 12(%rdx) addq $13, %rdx vpsllvd %ymm11, %ymm4, %ymm4 vpsllvd %ymm11, %ymm5, %ymm5 @@ -26824,7 +26011,8 @@ L_mldsa_encode_t0_t1_avx2_loop: vpor %ymm1, %ymm5, %ymm5 vmovdqu %xmm4, (%rcx) addq $10, %rcx - vmovdqu %xmm5, (%rcx) + vmovq %xmm5, (%rcx) + vpextrw $4, %xmm5, 8(%rcx) addq $10, %rcx addq $0x400, %rdi decb %sil diff --git a/wolfcrypt/src/wc_mldsa_asm.asm b/wolfcrypt/src/wc_mldsa_asm.asm index 9649525a62c..2198aec09f3 100644 --- a/wolfcrypt/src/wc_mldsa_asm.asm +++ b/wolfcrypt/src/wc_mldsa_asm.asm @@ -23567,872 +23567,53 @@ L_mldsa_encode_eta_2_avx2_loop: vpor ymm0, ymm0, ymm4 vextracti128 xmm4, ymm0, 1 vpor ymm0, ymm0, ymm4 - vmovdqu OWORD PTR [r8], xmm0 - add r8, 12 - add rcx, 1024 - dec dl - jnz L_mldsa_encode_eta_2_avx2_loop - vzeroupper - vmovdqu xmm6, OWORD PTR [rsp+8] - vmovdqu xmm7, OWORD PTR [rsp+24] - vmovdqu xmm8, OWORD PTR [rsp+40] - vmovdqu xmm9, OWORD PTR [rsp+56] - vmovdqu xmm10, OWORD PTR [rsp+72] - vmovdqu xmm11, OWORD PTR [rsp+88] - add rsp, 104 - ret -wc_mldsa_vec_encode_eta_2_avx2 ENDP -_TEXT ENDS -_DATA SEGMENT -ALIGN 16 -L_mldsa_encode_eta_4_avx2_four DWORD 00000004h, 00000004h, 00000004h, 00000004h - DWORD 00000004h, 00000004h, 00000004h, 00000004h -ptr_L_mldsa_encode_eta_4_avx2_four QWORD L_mldsa_encode_eta_4_avx2_four -_DATA ENDS -_DATA SEGMENT -ALIGN 16 -L_mldsa_encode_eta_4_avx2_vs_4 DWORD 00000000h, 00000004h, 00000000h, 00000004h - DWORD 00000000h, 00000004h, 00000000h, 00000004h -ptr_L_mldsa_encode_eta_4_avx2_vs_4 QWORD L_mldsa_encode_eta_4_avx2_vs_4 -_DATA ENDS -_TEXT SEGMENT READONLY PARA -wc_mldsa_vec_encode_eta_4_avx2 PROC - sub rsp, 72 - vmovdqu OWORD PTR [rsp+8], xmm6 - vmovdqu OWORD PTR [rsp+24], xmm7 - vmovdqu OWORD PTR [rsp+40], xmm8 - vmovdqu OWORD PTR [rsp+56], xmm9 - vpxor ymm8, ymm8, ymm8 - vmovdqu ymm8, YMMWORD PTR L_mldsa_encode_eta_4_avx2_four - vmovdqu ymm9, YMMWORD PTR L_mldsa_encode_eta_4_avx2_vs_4 - vmovdqu ymm0, YMMWORD PTR [rcx] - vmovdqu ymm1, YMMWORD PTR [rcx+32] - vmovdqu ymm2, YMMWORD PTR [rcx+64] - vmovdqu ymm3, YMMWORD PTR [rcx+96] - vmovdqu ymm4, YMMWORD PTR [rcx+128] - vmovdqu ymm5, YMMWORD PTR [rcx+160] - vmovdqu ymm6, YMMWORD PTR [rcx+192] - vmovdqu ymm7, YMMWORD PTR [rcx+224] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+256] - vmovdqu ymm1, YMMWORD PTR [rcx+288] - vmovdqu ymm2, YMMWORD PTR [rcx+320] - vmovdqu ymm3, YMMWORD PTR [rcx+352] - vmovdqu ymm4, YMMWORD PTR [rcx+384] - vmovdqu ymm5, YMMWORD PTR [rcx+416] - vmovdqu ymm6, YMMWORD PTR [rcx+448] - vmovdqu ymm7, YMMWORD PTR [rcx+480] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+32], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+512] - vmovdqu ymm1, YMMWORD PTR [rcx+544] - vmovdqu ymm2, YMMWORD PTR [rcx+576] - vmovdqu ymm3, YMMWORD PTR [rcx+608] - vmovdqu ymm4, YMMWORD PTR [rcx+640] - vmovdqu ymm5, YMMWORD PTR [rcx+672] - vmovdqu ymm6, YMMWORD PTR [rcx+704] - vmovdqu ymm7, YMMWORD PTR [rcx+736] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+64], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+768] - vmovdqu ymm1, YMMWORD PTR [rcx+800] - vmovdqu ymm2, YMMWORD PTR [rcx+832] - vmovdqu ymm3, YMMWORD PTR [rcx+864] - vmovdqu ymm4, YMMWORD PTR [rcx+896] - vmovdqu ymm5, YMMWORD PTR [rcx+928] - vmovdqu ymm6, YMMWORD PTR [rcx+960] - vmovdqu ymm7, YMMWORD PTR [rcx+992] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+96], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+1024] - vmovdqu ymm1, YMMWORD PTR [rcx+1056] - vmovdqu ymm2, YMMWORD PTR [rcx+1088] - vmovdqu ymm3, YMMWORD PTR [rcx+1120] - vmovdqu ymm4, YMMWORD PTR [rcx+1152] - vmovdqu ymm5, YMMWORD PTR [rcx+1184] - vmovdqu ymm6, YMMWORD PTR [rcx+1216] - vmovdqu ymm7, YMMWORD PTR [rcx+1248] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+128], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+1280] - vmovdqu ymm1, YMMWORD PTR [rcx+1312] - vmovdqu ymm2, YMMWORD PTR [rcx+1344] - vmovdqu ymm3, YMMWORD PTR [rcx+1376] - vmovdqu ymm4, YMMWORD PTR [rcx+1408] - vmovdqu ymm5, YMMWORD PTR [rcx+1440] - vmovdqu ymm6, YMMWORD PTR [rcx+1472] - vmovdqu ymm7, YMMWORD PTR [rcx+1504] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+160], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+1536] - vmovdqu ymm1, YMMWORD PTR [rcx+1568] - vmovdqu ymm2, YMMWORD PTR [rcx+1600] - vmovdqu ymm3, YMMWORD PTR [rcx+1632] - vmovdqu ymm4, YMMWORD PTR [rcx+1664] - vmovdqu ymm5, YMMWORD PTR [rcx+1696] - vmovdqu ymm6, YMMWORD PTR [rcx+1728] - vmovdqu ymm7, YMMWORD PTR [rcx+1760] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+192], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+1792] - vmovdqu ymm1, YMMWORD PTR [rcx+1824] - vmovdqu ymm2, YMMWORD PTR [rcx+1856] - vmovdqu ymm3, YMMWORD PTR [rcx+1888] - vmovdqu ymm4, YMMWORD PTR [rcx+1920] - vmovdqu ymm5, YMMWORD PTR [rcx+1952] - vmovdqu ymm6, YMMWORD PTR [rcx+1984] - vmovdqu ymm7, YMMWORD PTR [rcx+2016] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+224], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+2048] - vmovdqu ymm1, YMMWORD PTR [rcx+2080] - vmovdqu ymm2, YMMWORD PTR [rcx+2112] - vmovdqu ymm3, YMMWORD PTR [rcx+2144] - vmovdqu ymm4, YMMWORD PTR [rcx+2176] - vmovdqu ymm5, YMMWORD PTR [rcx+2208] - vmovdqu ymm6, YMMWORD PTR [rcx+2240] - vmovdqu ymm7, YMMWORD PTR [rcx+2272] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+256], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+2304] - vmovdqu ymm1, YMMWORD PTR [rcx+2336] - vmovdqu ymm2, YMMWORD PTR [rcx+2368] - vmovdqu ymm3, YMMWORD PTR [rcx+2400] - vmovdqu ymm4, YMMWORD PTR [rcx+2432] - vmovdqu ymm5, YMMWORD PTR [rcx+2464] - vmovdqu ymm6, YMMWORD PTR [rcx+2496] - vmovdqu ymm7, YMMWORD PTR [rcx+2528] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+288], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+2560] - vmovdqu ymm1, YMMWORD PTR [rcx+2592] - vmovdqu ymm2, YMMWORD PTR [rcx+2624] - vmovdqu ymm3, YMMWORD PTR [rcx+2656] - vmovdqu ymm4, YMMWORD PTR [rcx+2688] - vmovdqu ymm5, YMMWORD PTR [rcx+2720] - vmovdqu ymm6, YMMWORD PTR [rcx+2752] - vmovdqu ymm7, YMMWORD PTR [rcx+2784] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+320], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+2816] - vmovdqu ymm1, YMMWORD PTR [rcx+2848] - vmovdqu ymm2, YMMWORD PTR [rcx+2880] - vmovdqu ymm3, YMMWORD PTR [rcx+2912] - vmovdqu ymm4, YMMWORD PTR [rcx+2944] - vmovdqu ymm5, YMMWORD PTR [rcx+2976] - vmovdqu ymm6, YMMWORD PTR [rcx+3008] - vmovdqu ymm7, YMMWORD PTR [rcx+3040] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+352], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+3072] - vmovdqu ymm1, YMMWORD PTR [rcx+3104] - vmovdqu ymm2, YMMWORD PTR [rcx+3136] - vmovdqu ymm3, YMMWORD PTR [rcx+3168] - vmovdqu ymm4, YMMWORD PTR [rcx+3200] - vmovdqu ymm5, YMMWORD PTR [rcx+3232] - vmovdqu ymm6, YMMWORD PTR [rcx+3264] - vmovdqu ymm7, YMMWORD PTR [rcx+3296] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+384], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+3328] - vmovdqu ymm1, YMMWORD PTR [rcx+3360] - vmovdqu ymm2, YMMWORD PTR [rcx+3392] - vmovdqu ymm3, YMMWORD PTR [rcx+3424] - vmovdqu ymm4, YMMWORD PTR [rcx+3456] - vmovdqu ymm5, YMMWORD PTR [rcx+3488] - vmovdqu ymm6, YMMWORD PTR [rcx+3520] - vmovdqu ymm7, YMMWORD PTR [rcx+3552] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+416], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+3584] - vmovdqu ymm1, YMMWORD PTR [rcx+3616] - vmovdqu ymm2, YMMWORD PTR [rcx+3648] - vmovdqu ymm3, YMMWORD PTR [rcx+3680] - vmovdqu ymm4, YMMWORD PTR [rcx+3712] - vmovdqu ymm5, YMMWORD PTR [rcx+3744] - vmovdqu ymm6, YMMWORD PTR [rcx+3776] - vmovdqu ymm7, YMMWORD PTR [rcx+3808] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+448], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+3840] - vmovdqu ymm1, YMMWORD PTR [rcx+3872] - vmovdqu ymm2, YMMWORD PTR [rcx+3904] - vmovdqu ymm3, YMMWORD PTR [rcx+3936] - vmovdqu ymm4, YMMWORD PTR [rcx+3968] - vmovdqu ymm5, YMMWORD PTR [rcx+4000] - vmovdqu ymm6, YMMWORD PTR [rcx+4032] - vmovdqu ymm7, YMMWORD PTR [rcx+4064] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+480], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+4096] - vmovdqu ymm1, YMMWORD PTR [rcx+4128] - vmovdqu ymm2, YMMWORD PTR [rcx+4160] - vmovdqu ymm3, YMMWORD PTR [rcx+4192] - vmovdqu ymm4, YMMWORD PTR [rcx+4224] - vmovdqu ymm5, YMMWORD PTR [rcx+4256] - vmovdqu ymm6, YMMWORD PTR [rcx+4288] - vmovdqu ymm7, YMMWORD PTR [rcx+4320] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+512], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+4352] - vmovdqu ymm1, YMMWORD PTR [rcx+4384] - vmovdqu ymm2, YMMWORD PTR [rcx+4416] - vmovdqu ymm3, YMMWORD PTR [rcx+4448] - vmovdqu ymm4, YMMWORD PTR [rcx+4480] - vmovdqu ymm5, YMMWORD PTR [rcx+4512] - vmovdqu ymm6, YMMWORD PTR [rcx+4544] - vmovdqu ymm7, YMMWORD PTR [rcx+4576] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+544], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+4608] - vmovdqu ymm1, YMMWORD PTR [rcx+4640] - vmovdqu ymm2, YMMWORD PTR [rcx+4672] - vmovdqu ymm3, YMMWORD PTR [rcx+4704] - vmovdqu ymm4, YMMWORD PTR [rcx+4736] - vmovdqu ymm5, YMMWORD PTR [rcx+4768] - vmovdqu ymm6, YMMWORD PTR [rcx+4800] - vmovdqu ymm7, YMMWORD PTR [rcx+4832] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+576], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+4864] - vmovdqu ymm1, YMMWORD PTR [rcx+4896] - vmovdqu ymm2, YMMWORD PTR [rcx+4928] - vmovdqu ymm3, YMMWORD PTR [rcx+4960] - vmovdqu ymm4, YMMWORD PTR [rcx+4992] - vmovdqu ymm5, YMMWORD PTR [rcx+5024] - vmovdqu ymm6, YMMWORD PTR [rcx+5056] - vmovdqu ymm7, YMMWORD PTR [rcx+5088] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+608], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+5120] - vmovdqu ymm1, YMMWORD PTR [rcx+5152] - vmovdqu ymm2, YMMWORD PTR [rcx+5184] - vmovdqu ymm3, YMMWORD PTR [rcx+5216] - vmovdqu ymm4, YMMWORD PTR [rcx+5248] - vmovdqu ymm5, YMMWORD PTR [rcx+5280] - vmovdqu ymm6, YMMWORD PTR [rcx+5312] - vmovdqu ymm7, YMMWORD PTR [rcx+5344] + vmovq QWORD PTR [r8], xmm0 + vpextrd DWORD PTR [r8+8], xmm0, 2 + add r8, 12 + add rcx, 1024 + dec dl + jnz L_mldsa_encode_eta_2_avx2_loop + vzeroupper + vmovdqu xmm6, OWORD PTR [rsp+8] + vmovdqu xmm7, OWORD PTR [rsp+24] + vmovdqu xmm8, OWORD PTR [rsp+40] + vmovdqu xmm9, OWORD PTR [rsp+56] + vmovdqu xmm10, OWORD PTR [rsp+72] + vmovdqu xmm11, OWORD PTR [rsp+88] + add rsp, 104 + ret +wc_mldsa_vec_encode_eta_2_avx2 ENDP +_TEXT ENDS +_DATA SEGMENT +ALIGN 16 +L_mldsa_encode_eta_4_avx2_four DWORD 00000004h, 00000004h, 00000004h, 00000004h + DWORD 00000004h, 00000004h, 00000004h, 00000004h +ptr_L_mldsa_encode_eta_4_avx2_four QWORD L_mldsa_encode_eta_4_avx2_four +_DATA ENDS +_DATA SEGMENT +ALIGN 16 +L_mldsa_encode_eta_4_avx2_vs_4 DWORD 00000000h, 00000004h, 00000000h, 00000004h + DWORD 00000000h, 00000004h, 00000000h, 00000004h +ptr_L_mldsa_encode_eta_4_avx2_vs_4 QWORD L_mldsa_encode_eta_4_avx2_vs_4 +_DATA ENDS +_TEXT SEGMENT READONLY PARA +wc_mldsa_vec_encode_eta_4_avx2 PROC + sub rsp, 72 + vmovdqu OWORD PTR [rsp+8], xmm6 + vmovdqu OWORD PTR [rsp+24], xmm7 + vmovdqu OWORD PTR [rsp+40], xmm8 + vmovdqu OWORD PTR [rsp+56], xmm9 + vmovdqu ymm8, YMMWORD PTR L_mldsa_encode_eta_4_avx2_four + vmovdqu ymm9, YMMWORD PTR L_mldsa_encode_eta_4_avx2_vs_4 +L_mldsa_encode_eta_4_avx2_loop: + vmovdqu ymm0, YMMWORD PTR [rcx] + vmovdqu ymm1, YMMWORD PTR [rcx+32] + vmovdqu ymm2, YMMWORD PTR [rcx+64] + vmovdqu ymm3, YMMWORD PTR [rcx+96] + vmovdqu ymm4, YMMWORD PTR [rcx+128] + vmovdqu ymm5, YMMWORD PTR [rcx+160] + vmovdqu ymm6, YMMWORD PTR [rcx+192] + vmovdqu ymm7, YMMWORD PTR [rcx+224] vpsubd ymm0, ymm8, ymm0 vpsubd ymm1, ymm8, ymm1 vpsubd ymm2, ymm8, ymm2 @@ -24465,15 +23646,15 @@ wc_mldsa_vec_encode_eta_4_avx2 PROC vpermq ymm1, ymm1, 216 vpackuswb ymm0, ymm0, ymm1 vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+640], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+5376] - vmovdqu ymm1, YMMWORD PTR [rcx+5408] - vmovdqu ymm2, YMMWORD PTR [rcx+5440] - vmovdqu ymm3, YMMWORD PTR [rcx+5472] - vmovdqu ymm4, YMMWORD PTR [rcx+5504] - vmovdqu ymm5, YMMWORD PTR [rcx+5536] - vmovdqu ymm6, YMMWORD PTR [rcx+5568] - vmovdqu ymm7, YMMWORD PTR [rcx+5600] + vmovdqu YMMWORD PTR [r8], ymm0 + vmovdqu ymm0, YMMWORD PTR [rcx+256] + vmovdqu ymm1, YMMWORD PTR [rcx+288] + vmovdqu ymm2, YMMWORD PTR [rcx+320] + vmovdqu ymm3, YMMWORD PTR [rcx+352] + vmovdqu ymm4, YMMWORD PTR [rcx+384] + vmovdqu ymm5, YMMWORD PTR [rcx+416] + vmovdqu ymm6, YMMWORD PTR [rcx+448] + vmovdqu ymm7, YMMWORD PTR [rcx+480] vpsubd ymm0, ymm8, ymm0 vpsubd ymm1, ymm8, ymm1 vpsubd ymm2, ymm8, ymm2 @@ -24506,15 +23687,15 @@ wc_mldsa_vec_encode_eta_4_avx2 PROC vpermq ymm1, ymm1, 216 vpackuswb ymm0, ymm0, ymm1 vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+672], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+5632] - vmovdqu ymm1, YMMWORD PTR [rcx+5664] - vmovdqu ymm2, YMMWORD PTR [rcx+5696] - vmovdqu ymm3, YMMWORD PTR [rcx+5728] - vmovdqu ymm4, YMMWORD PTR [rcx+5760] - vmovdqu ymm5, YMMWORD PTR [rcx+5792] - vmovdqu ymm6, YMMWORD PTR [rcx+5824] - vmovdqu ymm7, YMMWORD PTR [rcx+5856] + vmovdqu YMMWORD PTR [r8+32], ymm0 + vmovdqu ymm0, YMMWORD PTR [rcx+512] + vmovdqu ymm1, YMMWORD PTR [rcx+544] + vmovdqu ymm2, YMMWORD PTR [rcx+576] + vmovdqu ymm3, YMMWORD PTR [rcx+608] + vmovdqu ymm4, YMMWORD PTR [rcx+640] + vmovdqu ymm5, YMMWORD PTR [rcx+672] + vmovdqu ymm6, YMMWORD PTR [rcx+704] + vmovdqu ymm7, YMMWORD PTR [rcx+736] vpsubd ymm0, ymm8, ymm0 vpsubd ymm1, ymm8, ymm1 vpsubd ymm2, ymm8, ymm2 @@ -24547,15 +23728,15 @@ wc_mldsa_vec_encode_eta_4_avx2 PROC vpermq ymm1, ymm1, 216 vpackuswb ymm0, ymm0, ymm1 vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+704], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+5888] - vmovdqu ymm1, YMMWORD PTR [rcx+5920] - vmovdqu ymm2, YMMWORD PTR [rcx+5952] - vmovdqu ymm3, YMMWORD PTR [rcx+5984] - vmovdqu ymm4, YMMWORD PTR [rcx+6016] - vmovdqu ymm5, YMMWORD PTR [rcx+6048] - vmovdqu ymm6, YMMWORD PTR [rcx+6080] - vmovdqu ymm7, YMMWORD PTR [rcx+6112] + vmovdqu YMMWORD PTR [r8+64], ymm0 + vmovdqu ymm0, YMMWORD PTR [rcx+768] + vmovdqu ymm1, YMMWORD PTR [rcx+800] + vmovdqu ymm2, YMMWORD PTR [rcx+832] + vmovdqu ymm3, YMMWORD PTR [rcx+864] + vmovdqu ymm4, YMMWORD PTR [rcx+896] + vmovdqu ymm5, YMMWORD PTR [rcx+928] + vmovdqu ymm6, YMMWORD PTR [rcx+960] + vmovdqu ymm7, YMMWORD PTR [rcx+992] vpsubd ymm0, ymm8, ymm0 vpsubd ymm1, ymm8, ymm1 vpsubd ymm2, ymm8, ymm2 @@ -24588,7 +23769,11 @@ wc_mldsa_vec_encode_eta_4_avx2 PROC vpermq ymm1, ymm1, 216 vpackuswb ymm0, ymm0, ymm1 vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+736], ymm0 + vmovdqu YMMWORD PTR [r8+96], ymm0 + add rcx, 1024 + add r8, 128 + dec dl + jnz L_mldsa_encode_eta_4_avx2_loop vzeroupper vmovdqu xmm6, OWORD PTR [rsp+8] vmovdqu xmm7, OWORD PTR [rsp+24] @@ -26420,7 +25605,9 @@ L_mldsa_encode_t0_t1_avx2_loop: vpor ymm3, ymm3, ymm1 vmovdqu OWORD PTR [r8], xmm2 add r8, 13 - vmovdqu OWORD PTR [r8], xmm3 + vmovq QWORD PTR [r8], xmm3 + vpextrd DWORD PTR [r8+8], xmm3, 2 + vpextrb BYTE PTR [r8+12], xmm3, 12 add r8, 13 vpsllvd ymm4, ymm4, ymm11 vpsllvd ymm5, ymm5, ymm11 @@ -26436,7 +25623,8 @@ L_mldsa_encode_t0_t1_avx2_loop: vpor ymm5, ymm5, ymm1 vmovdqu OWORD PTR [r9], xmm4 add r9, 10 - vmovdqu OWORD PTR [r9], xmm5 + vmovq QWORD PTR [r9], xmm5 + vpextrw WORD PTR [r9+8], xmm5, 4 add r9, 10 add rcx, 1024 dec dl diff --git a/wolfssl/wolfcrypt/wc_mldsa.h b/wolfssl/wolfcrypt/wc_mldsa.h index 17f9d9135f3..d2aa0cb6204 100644 --- a/wolfssl/wolfcrypt/wc_mldsa.h +++ b/wolfssl/wolfcrypt/wc_mldsa.h @@ -1029,7 +1029,8 @@ WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta4_avx2(const byte* z, WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_2_avx2(const sword32* s, byte d, byte* p); -WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_4_avx2(const sword32* t, byte* p); +WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_4_avx2(const sword32* s, byte d, + byte* p); WOLFSSL_LOCAL void wc_mldsa_decode_eta_2_avx2(const byte* p, sword32* s); WOLFSSL_LOCAL void wc_mldsa_decode_eta_4_avx2(const byte* p, sword32* s); @@ -1326,6 +1327,13 @@ WOLFSSL_TEST_VIS int wc_mldsa_encode_w1_32(const sword32* w1, byte* w1e); #endif #endif +#ifndef WOLFSSL_MLDSA_NO_MAKE_KEY +WOLFSSL_TEST_VIS int wc_mldsa_vec_encode_eta(const sword32* s, byte d, + byte eta, byte* p); +WOLFSSL_TEST_VIS int wc_mldsa_vec_encode_t0_t1(const sword32* t, byte d, + byte* t0, byte* t1); +#endif + #if defined(WOLFSSL_MLDSA_TEST_REJ_NTT_FAIL) && \ !defined(WOLF_CRYPTO_CB_ONLY_MLDSA) WOLFSSL_TEST_VIS int wc_mldsa_test_rej_ntt_fail(int n); From ee586fb74bdb011fe6572329bba9ec9f30b3606d Mon Sep 17 00:00:00 2001 From: Tobias Frauenschlaeger Date: Mon, 9 Nov 2026 07:51:24 +0000 Subject: [PATCH 4/7] Stream s2 in the small memory ML-DSA key generation The small memory key generation held the whole s2 vector only to encode it into the private key and decode one row of it again per row of t. Expand s2 a polynomial at a time into the scratch polynomial of A and encode it straight into the private key. The encoding lies past rho' and K, so it does not overwrite the seeds the later phases still read. mldsa_calc_t_row() decodes the row of s2 into the same scratch once the row of A is done, which also takes a polynomial out of mldsa_make_pub_key(). s1 is expanded with the C sampler, so the multi-lane s1/s2 generators are no longer built with WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM. That costs 4 to 8% of key generation time with the AArch64 NEON code; the C build changes by under 3%. Heap for ML-DSA-44/65/87 key generation drops from 10057/13129/17225 to 6985/8009/10057 bytes. Keys and signatures are unchanged. --- wolfcrypt/src/wc_mldsa.c | 108 +++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index afdbec0df44..a50642472ce 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -550,7 +550,9 @@ static WC_INLINE cpuid_flags_t mldsa_cpuid_flags(void) #endif /* Expanding a, s (make key) and y (sign) all squeeze more blocks. */ -#if defined(MLDSA_NEON_EXPAND_A) || !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ +#if defined(MLDSA_NEON_EXPAND_A) || \ + (!defined(WOLFSSL_MLDSA_NO_MAKE_KEY) && \ + !defined(WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM)) || \ (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM)) static void mlkem_sha3_blocksx3(word64* state) @@ -577,7 +579,8 @@ static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) #endif /* Only expanding s (make key) and y (sign) use the 64-byte seed. */ -#if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ +#if (!defined(WOLFSSL_MLDSA_NO_MAKE_KEY) && \ + !defined(WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM)) || \ (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM)) static void mlkem_shake256_blocksx3_seed_64(word64* state, byte* seed) @@ -5362,6 +5365,7 @@ static int mldsa_rej_bound_poly(wc_Shake* shake256, byte* seed, sword32* s, #endif } +#ifndef WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifndef WOLFSSL_NO_ML_DSA_44 /* Deterministically generate a matrix (or transpose) of uniform integers mod q. @@ -6213,6 +6217,7 @@ static int wc_mldsa_gen_s_7_8_arm(sword32* s[2], byte* seed) } #endif #endif +#endif /* !WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM */ /* Expand private seed into vectors s1 and s2. * @@ -6281,6 +6286,7 @@ static int mldsa_expand_s_c(wc_Shake* shake256, byte* priv_seed, byte eta, return ret; } +#ifndef WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM /* Expand private seed into vectors s1 and s2. * * @param [in, out] shake256 SHAKE-256 object. @@ -6396,6 +6402,7 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, return ret; } +#endif /* !WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM */ #endif /* !WOLFSSL_MLDSA_NO_MAKE_KEY */ #ifndef WOLFSSL_MLDSA_NO_SIGN @@ -11556,7 +11563,7 @@ static void mldsa_shake_wipe(wc_Shake* shake) /* Calculate row r of t = NTT-1(A o NTT(s1)) + s2 and make it positive. * * Matrix A is expanded a polynomial at a time and the row of s2 is decoded - * from its encoding, so only single polynomial scratch is needed. + * from its encoding into the same scratch polynomial once A is done with. * * @param [in, out] shake SHAKE-128 object to expand A with. * @param [in, out] aseed Public seed followed by space for column and row. @@ -11566,8 +11573,8 @@ static void mldsa_shake_wipe(wc_Shake* shake) * @param [in] s2p Encoding of row r of s2. * @param [in] eta Range of s2 coefficients. * @param [out] tt Row r of t, made positive. - * @param [out] s2t Scratch polynomial for row r of s2. - * @param [out] a Scratch polynomial for an element of A. + * @param [out] a Scratch polynomial for an element of A and the row + * of s2. * @param [out] h Rejection sampling scratch. * @param [out] t64 Accumulator polynomial. Only used with * WOLFSSL_MLDSA_SMALL_MEM_POLY64. @@ -11576,7 +11583,7 @@ static void mldsa_shake_wipe(wc_Shake* shake) */ static int mldsa_calc_t_row(wc_Shake* shake, byte* aseed, byte r, const sword32* s1, byte l, const byte* s2p, byte eta, sword32* tt, - sword32* s2t, sword32* a, byte* h, sword64* t64) + sword32* a, byte* h, sword64* t64) { int ret = 0; const sword32* s1t = s1; @@ -11701,10 +11708,10 @@ static int mldsa_calc_t_row(wc_Shake* shake, byte* aseed, byte r, } /* Decode the row of s2 as the vector is not kept by the callers. */ if (ret == 0) { - ret = mldsa_vec_decode_eta_bits(s2p, eta, s2t, 1); + ret = mldsa_vec_decode_eta_bits(s2p, eta, a, 1); } if (ret == 0) { - ret = mldsa_add(tt, s2t); + ret = mldsa_add(tt, a); } /* Make positive for decomposing. */ if (ret == 0) { @@ -11966,7 +11973,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) const wc_MlDsaParams* params = key->params; sword32* a = NULL; sword32* s1 = NULL; - sword32* s2 = NULL; sword32* t = NULL; sword64* t64 = NULL; byte* h = NULL; @@ -11995,12 +12001,10 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) /* Allocate memory for large intermediates. */ if (ret == 0) { - /* s1-l, s2-k, a-1, [t64], h. t is encoded a polynomial at a time, - * so it and the one decoded s2 polynomial share the s2 vector, which - * is dead once s2 has been encoded into the private key. - * h is last as its size may be odd. */ - allocSz = (unsigned int)params->s1Sz + params->s2Sz + - (unsigned int)MLDSA_POLY_SIZE + + /* s1-l, t-1, a-1, [t64], h. s2 and t are encoded a polynomial at a + * time. h is last as its size may be odd. */ + allocSz = (unsigned int)params->s1Sz + + 2U * (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 /* t64 */ @@ -12028,9 +12032,8 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) ret = MEMORY_E; } else { - s2 = s1 + params->s1Sz / sizeof(*s1); - t = s2; - a = s2 + params->s2Sz / sizeof(*s2); + t = s1 + params->s1Sz / sizeof(*s1); + a = t + MLDSA_N; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 t64 = (sword64*)(a + MLDSA_N); h = (byte*)(t64 + MLDSA_N); @@ -12081,13 +12084,40 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) case 1: if (ret == 0) { byte* priv_seed = key->k + MLDSA_PUB_SEED_SZ; + byte* s2p = priv_seed + MLDSA_K_SZ + MLDSA_TR_SZ + params->s1EncSz; + word32 s2Stride = (word32)params->s2EncSz / params->k; + byte sseed[MLDSA_GEN_S_SEED_SZ]; /* Step 7; Alg 22 Step 1: Copy public seed into public key. */ XMEMCPY(key->p, pub_seed, MLDSA_PUB_SEED_SZ); - /* Step 4: Expand private seed into to vectors of polynomials. */ - ret = mldsa_expand_s(&key->shake, priv_seed, params->eta, s1, - params->l, s2, params->k, key->heap); + /* Step 4: Expand private seed into vector s1. */ + ret = mldsa_expand_s_c(&key->shake, priv_seed, params->eta, s1, + params->l, NULL, 0); + /* Step 9. Alg 24 Steps 2-4: Encode s1 into private key. */ + if (ret == 0) { + ret = mldsa_vec_encode_eta_bits(s1, params->l, params->eta, + s2p - params->s1EncSz); + } + /* Step 4, Step 9. Alg 24 Steps 5-7: Expand and encode s2 a polynomial + * at a time. The encoding lies past rho' and K, still in use. */ + XMEMCPY(sseed, priv_seed, MLDSA_PRIV_SEED_SZ); + #ifdef WOLFSSL_CHECK_MEM_ZERO + wc_MemZero_Add("mldsa make_key sseed", sseed, sizeof(sseed)); + #endif + sseed[MLDSA_PRIV_SEED_SZ + 1] = 0; + for (r = 0; (ret == 0) && (r < params->k); r++) { + sseed[MLDSA_PRIV_SEED_SZ] = (byte)(r + params->l); + ret = mldsa_rej_bound_poly(&key->shake, sseed, a, params->eta); + if (ret == 0) { + ret = mldsa_vec_encode_eta_bits(a, 1, params->eta, + s2p + (size_t)r * s2Stride); + } + } + ForceZero(sseed, sizeof(sseed)); + #ifdef WOLFSSL_CHECK_MEM_ZERO + wc_MemZero_Check(sseed, sizeof(sseed)); + #endif } #ifdef WC_MLDSA_NONBLOCK if ((key->nb != NULL) && (ret == 0)) { @@ -12100,8 +12130,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) case 2: if (ret == 0) { byte* k = pub_seed + MLDSA_PUB_SEED_SZ; - byte* s1p = k + MLDSA_K_SZ + MLDSA_TR_SZ; - byte* s2p = s1p + params->s1EncSz; byte kBuf[MLDSA_K_SZ]; #ifdef WOLFSSL_CHECK_MEM_ZERO @@ -12112,14 +12140,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) ForceZero(k, MLDSA_PRIV_SEED_SZ + MLDSA_K_SZ); XMEMCPY(k, kBuf, MLDSA_K_SZ); ForceZero(kBuf, sizeof(kBuf)); - /* Step 9. Alg 24 Steps 2-4: Encode s1 into private key. */ - if (ret == 0) { - ret = mldsa_vec_encode_eta_bits(s1, params->l, params->eta, s1p); - } - /* Step 9. Alg 24 Steps 5-7: Encode s2 into private key. */ - if (ret == 0) { - ret = mldsa_vec_encode_eta_bits(s2, params->k, params->eta, s2p); - } /* Step 5: NTT(s1) */ if (ret == 0) { ret = mldsa_vec_ntt_small_full(s1, params->l); @@ -12149,8 +12169,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) word32 s2Stride = (word32)params->s2EncSz / params->k; unsigned int r0 = 0; byte aseed[MLDSA_GEN_A_SEED_SZ]; - /* One decoded s2 polynomial, held after t in the dead s2 vector. */ - sword32* s2t = t + MLDSA_N; sword32* tt = t; const byte* s2pt; byte* t0; @@ -12168,7 +12186,7 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) XMEMCPY(aseed, pub_seed, MLDSA_PUB_SEED_SZ); for (r = r0; (ret == 0) && (r < params->k); r++) { ret = mldsa_calc_t_row(&key->shake, aseed, (byte)r, s1, - params->l, s2pt, params->eta, tt, s2t, a, h, t64); + params->l, s2pt, params->eta, tt, a, h, t64); /* Decompose t into t0 and t1 and encode into the keys, one poly. */ if (ret == 0) { ret = mldsa_vec_encode_t0_t1(tt, 1, t0, t1); @@ -12219,12 +12237,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) break; } - /* Zeroize the whole buffer before freeing. It holds the private vectors - * s1, s2 and t at the front, then the public matrix polynomial a, then - * (POLY64 builds) the t64 accumulator holding A o NTT(s1) - from which - * s1 is recoverable - and finally the public rejection-sampling buffer - * h. As the secret material is not contiguous, zeroize the entire - * allocation rather than a sub-range. */ #ifdef WC_MLDSA_NONBLOCK if ((key->nb != NULL) && (ret == WC_NO_ERR_TRACE(MP_WOULDBLOCK))) { /* Yielding: keep the working allocation for the next call. */ @@ -16013,7 +16025,6 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) #ifndef WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM sword32* s2 = NULL; #else - sword32* s2t = NULL; sword64* t64 = NULL; byte* h = NULL; #endif @@ -16073,10 +16084,9 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) /* Matrix A is expanded a polynomial at a time, so any cache is left * alone. */ if (ret == 0) { - /* s1-l, t-1, s2-1, a-1, [t64], h. h is last as its size may be - * odd. */ + /* s1-l, t-1, a-1, [t64], h. h is last as its size may be odd. */ allocSz = (unsigned int)params->s1Sz + - 3U * (unsigned int)MLDSA_POLY_SIZE + + 2U * (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 /* t64 */ @@ -16089,8 +16099,7 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) } else { t = s1 + params->s1Sz / sizeof(*s1); - s2t = t + MLDSA_N; - a = s2t + MLDSA_N; + a = t + MLDSA_N; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 t64 = (sword64*)(a + MLDSA_N); h = (byte*)(t64 + MLDSA_N); @@ -16172,8 +16181,8 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) for (r = 0; (ret == 0) && (r < params->k); r++) { /* Calculate row r of t = NTT-1(A o NTT(s1)) + s2 */ ret = mldsa_calc_t_row(&key->shake, aseed, (byte)r, s1, - params->l, s2p + (r * s2Stride), params->eta, t, s2t, a, - h, t64); + params->l, s2p + (r * s2Stride), params->eta, t, a, h, + t64); /* Encode row r of t1 into the public key and check row r * of the derived t0 against the private key's t0. */ if (ret == 0) { @@ -16209,8 +16218,7 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) /* s1, s2 and t at the front are secret; any A after is public. */ ForceZero(s1, (unsigned int)params->s1Sz + (2U * params->s2Sz)); #else - /* The public a and h sit between and after the secret s1, t, s2 and - * t64, so zeroize the entire allocation. */ + /* a also holds the secret s2 rows, so zeroize the entire allocation. */ ForceZero(s1, allocSz); #endif XFREE(s1, key->heap, DYNAMIC_TYPE_MLDSA); From 3ad7bb2f82e9eb853287be77cf9d669c49788719 Mon Sep 17 00:00:00 2001 From: Tobias Frauenschlaeger Date: Mon, 9 Nov 2026 07:51:24 +0000 Subject: [PATCH 5/7] Share dead buffers in the small memory ML-DSA signer The plain small memory signer allocated a hint polynomial w and an A sampling block of its own. w is only live in the hint loop, where the polynomial of A is dead, and the sampling block is only live while A is generated, when c is dead, and while c is sampled, when A is dead. Alias them. The PRECALC signers keep their layout, as their A is live throughout. Heap for ML-DSA-44/65/87 signing drops from 13897/16969/21321 to 12032/15104/19456 bytes. Signatures are unchanged. --- wolfcrypt/src/wc_mldsa.c | 46 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index a50642472ce..df75bbb1ecf 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -756,6 +756,10 @@ void print_data(const char* name, const byte* d, int len) /* The smallest memory signer expands the mask into a polynomial buffer. */ wc_static_assert(sizeof(sword32) * MLDSA_N >= MLDSA_MAX_V); #endif +#if !defined(WOLFSSL_MLDSA_NO_SIGN) && defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) +/* The small memory signers sample A into a polynomial buffer. */ +wc_static_assert(MLDSA_POLY_SIZE >= MLDSA_REJ_NTT_POLY_H_SIZE); +#endif /* 2 blocks, each block 136 bytes = 272 bytes. @@ -12917,6 +12921,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, sword32* z = NULL; sword32* ct0 = NULL; byte* blocks = NULL; + byte* cBlocks = NULL; byte* w1e = NULL; byte priv_rand_seed[MLDSA_Y_SEED_SZ]; byte* h = sig + params->lambda / 4 + params->zEncSz; @@ -12955,22 +12960,22 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Allocate memory for large intermediates. */ if (ret == 0) { - /* y-l, w0-k, w-1, c-1, z-1, A-1, w1e, blocks. - * Walking A a column at a time makes w0 the vector of accumulators, - * w1 is only kept encoded, and w is the scratch for the hints. */ + /* y-l, w0-k, c-1, z-1, A-1, w1e. w0 accumulates a column at a time; + * w shares A, and the sampling blocks share c then A. */ allocSz = (unsigned int)params->s1Sz + params->s2Sz + (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_POLY_SIZE + - (unsigned int)MLDSA_POLY_SIZE + - (unsigned int)params->w1EncSz + - (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE; + (unsigned int)params->w1EncSz; #ifdef WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC - allocSz += (unsigned int)params->s1Sz + params->s2Sz + params->s2Sz; + allocSz += (unsigned int)MLDSA_POLY_SIZE + + (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE + + (unsigned int)params->s1Sz + params->s2Sz + params->s2Sz; #elif defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A) /* w is decomposed and encoded as a full k vector regardless of how * many rows of A are pre-calculated, so it must hold k polynomials. */ - allocSz += (unsigned int)(params->k - 1) * + allocSz += (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE + + (unsigned int)params->k * (unsigned int)MLDSA_POLY_SIZE + (unsigned int)maxK * params->l * (unsigned int)MLDSA_POLY_SIZE; @@ -12999,8 +13004,8 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, y_check = y; #endif w0 = y + params->s1Sz / sizeof(*y_ntt); - w = w0 + params->s2Sz / sizeof(*w0); #if defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A) + w = w0 + params->s2Sz / sizeof(*w0); /* Only maxK rows of w are multiplied into, but the ML-DSA-44 * decompose kernels take no dimension and touch all k. One * zeroing covers every rejection round: decompose of zero is @@ -13010,8 +13015,11 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, (size_t)(params->k - maxK) * MLDSA_POLY_SIZE); } c = w + (unsigned int)params->k * MLDSA_N; - #else + #elif defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC) + w = w0 + params->s2Sz / sizeof(*w0); c = w + MLDSA_N; + #else + c = w0 + params->s2Sz / sizeof(*w0); #endif z = c + MLDSA_N; a = z + MLDSA_N; @@ -13023,6 +13031,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, t0 = z; w1e = (byte*)(a + (1 + maxK * params->l) * MLDSA_N); blocks = w1e + params->w1EncSz; + cBlocks = blocks; #elif defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC) y_ntt = z; s1 = a + MLDSA_N; @@ -13030,13 +13039,16 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, t0 = s2 + params->s2Sz / sizeof(*s2); w1e = (byte*)(t0 + params->s2Sz / sizeof(*t0)); blocks = w1e + params->w1EncSz; + cBlocks = blocks; #else y_ntt = z; s1 = z; s2 = z; t0 = z; + w = a; w1e = (byte*)(a + MLDSA_N); - blocks = w1e + params->w1EncSz; + blocks = (byte*)c; + cBlocks = (byte*)a; #endif } } @@ -13365,7 +13377,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Step 17: Compute c from first 256 bits of commit. */ ret = mldsa_sample_in_ball_ex(params->level, &key->shake, commit, params->lambda / 4, params->tau, c, - blocks); + cBlocks); } if (ret == 0) { /* Step 18: NTT(c). */ @@ -13585,10 +13597,14 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, } if (!valid) { - /* Wipe the rejected candidate; A and cached keys stay. */ + /* Wipe the rejected candidate; cached A and keys stay. */ + #if defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A) || \ + defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC) ForceZero(y, (size_t)((byte*)a - (byte*)y)); - ForceZero(w1e, (size_t)params->w1EncSz + - MLDSA_REJ_NTT_POLY_H_SIZE); + ForceZero(w1e, (size_t)((byte*)y + allocSz - w1e)); + #else + ForceZero(y, allocSz); + #endif ForceZero(sig, params->sigSz); /* Too many attempts - something wrong with implementation. */ if ((kappa > (word16)(kappa + params->l))) { From c22e8cb0b8166421ad3fb7cf8c85e514811317f1 Mon Sep 17 00:00:00 2001 From: Tobias Frauenschlaeger Date: Mon, 9 Nov 2026 07:51:24 +0000 Subject: [PATCH 6/7] Shrink the smallest memory ML-DSA signer and make it non-blocking The smallest memory signer now holds the k accumulators of w and three polynomials: y, A and c. - z is computed in place in the polynomial of A. - The A sampling block lives in c during the column walk, and y is the SampleInBall block. - w1 is not stored. Each row is encoded and absorbed into key->shake, which is idle once the column walk ends, and w is decomposed again in the hint loop instead of keeping w0. Heap for ML-DSA-44/65/87 signing drops from 9801/11849/14153 to 7168/9216/11264 bytes. Signatures are unchanged. WC_MLDSA_NONBLOCK now accepts WOLFSSL_MLDSA_SIGN_SMALLEST_MEM and WOLFSSL_MLDSA_VERIFY_SMALLEST_MEM. The signer yields after each column of the walk and after each rejected round, like the small memory signer. The accumulators persist in the working allocation; yChk and the result of the early y check move there too (40/48/64 bytes more heap), as a resumed call starts from a fresh stack. The w1 hash runs within one slice, since each call derives rho'' with key->shake first. The verifier already streamed z and needed only the guard lifted. A rejected attempt now wipes the signature buffer and the working polynomials before the next one, so a yield never exposes a rejected z. key->shake is now wiped rather than re-initialized on the way out: the re-initialization runs on every yield and dropped the key's heap and device id. A resumed call that fails before it picks up the working allocation still frees it, and an attempt the early y check rejected skips building the commitment. mldsa_nonblock_test now also signs eight messages with seeded key generation and signing through both paths and requires identical bytes; skipping one rejection candidate on resume passes the old test and fails this one. The yield counts must differ between messages, so at least one resume follows a rejection. Two pq-all rows build the smallest memory non-blocking paths, with and without the early checks and fault hardening. --- .github/configs/pq-all.json | 8 ++ wolfcrypt/src/wc_mldsa.c | 222 +++++++++++++++++++++++++----------- wolfcrypt/test/test.c | 96 ++++++++++++++++ 3 files changed, 262 insertions(+), 64 deletions(-) diff --git a/.github/configs/pq-all.json b/.github/configs/pq-all.json index a4482733d6c..00140c25fc2 100644 --- a/.github/configs/pq-all.json +++ b/.github/configs/pq-all.json @@ -25,6 +25,14 @@ "configure": ["--enable-dilithium=yes,small", "--enable-mlkem", "--enable-experimental", "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_LARGE_CODE -DWC_MLDSA_NONBLOCK -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWC_MLKEM_NONBLOCK"]}, +{"name": "pq-nonblock-smallest", "minutes": 2.0, + "comment": "Non-blocking ML-DSA on the smallest memory signer and verifier: mldsa_nonblock_test resumes the column walk and compares seeded output with the blocking path", + "configure": ["--enable-cryptonly", "--enable-mldsa", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -DWOLFSSL_MLDSA_SIGN_SMALLEST_MEM -DWOLFSSL_MLDSA_VERIFY_SMALLEST_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWC_MLDSA_NONBLOCK"]}, +{"name": "pq-nonblock-smallest-checks", "minutes": 2.0, + "comment": "Non-blocking smallest memory signer with the early y and w0 checks and fault hardening: the y check result is carried across yields", + "configure": ["--enable-cryptonly", "--enable-mldsa=yes,small", + "CPPFLAGS=-DWOLFSSL_MLDSA_SIGN_SMALLEST_MEM -DWOLFSSL_MLDSA_VERIFY_SMALLEST_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_SIGN_CHECK_Y -DWOLFSSL_MLDSA_SIGN_CHECK_W0 -DWC_MLDSA_FAULT_HARDEN -DWC_MLDSA_NONBLOCK"]}, {"name": "pq-nonblock-smallmem-dynkeys", "minutes": 2.5, "comment": "Non-blocking ML-KEM with the cached matrix A and dynamic key buffers: a resumed keygen must keep the vectors its first slice allocated, and encapsulation must read the cached matrix across yields", "configure": ["--enable-cryptonly", "--enable-mlkem", "--enable-smallstack", diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index df75bbb1ecf..8a68e25a457 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -272,9 +272,8 @@ #endif #if !defined(WOLFSSL_MLDSA_NO_SIGN) && \ (defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC) || \ - defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A) || \ - defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM)) - #error "WC_MLDSA_NONBLOCK sign needs plain SIGN_SMALL_MEM" + defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A)) + #error "WC_MLDSA_NONBLOCK sign does not support the PRECALC signers" #endif #if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) && \ !defined(WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM) @@ -289,9 +288,8 @@ #error "WC_MLDSA_NONBLOCK keygen unsupported with the keygen PCT" #endif #if !defined(WOLFSSL_MLDSA_NO_VERIFY) && \ - (defined(WOLFSSL_MLDSA_VERIFY_SMALLEST_MEM) || \ - defined(WOLFSSL_MLDSA_VERIFY_NO_MALLOC)) - #error "WC_MLDSA_NONBLOCK verify needs plain VERIFY_SMALL_MEM" + defined(WOLFSSL_MLDSA_VERIFY_NO_MALLOC) + #error "WC_MLDSA_NONBLOCK verify keeps its buffers on the heap" #endif #ifdef WOLFSSL_NO_MALLOC #error "WC_MLDSA_NONBLOCK keeps working vectors on the heap" @@ -3536,7 +3534,9 @@ WOLFSSL_TEST_VIS int wc_mldsa_encode_w1_32(const sword32* w1, byte* w1e) #endif #endif -#if !defined(WOLFSSL_MLDSA_NO_SIGN) && defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) +#if !defined(WOLFSSL_MLDSA_NO_SIGN) && \ + defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) && \ + !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM) /* Decode one polynomial of w1 from its encoded form. * * The small memory signing implementations keep w1 only in the encoded form @@ -13690,7 +13690,6 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, sword32* a = NULL; sword32* c = NULL; sword32* z = NULL; - byte* w1e = NULL; byte* blocks = NULL; byte priv_rand_seed[MLDSA_Y_SEED_SZ]; byte* h = sig + params->lambda / 4 + params->zEncSz; @@ -13700,21 +13699,31 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Bytes of encoded s1 or s2 per polynomial. */ unsigned int sStride = (unsigned int)params->s2EncSz / params->k; /* Checksum of each polynomial of y, to bind the two derivations. */ +#ifdef WC_MLDSA_NONBLOCK + word64* yChk = NULL; + int resume = 0; +#else word64 yChk[MLDSA_MAX_L_VECTOR_COUNT / MLDSA_N]; +#endif #ifdef WC_MLDSA_FAULT_HARDEN sword32* w_check; #endif - /* priv_rand_seed will hold the secret signing seed (rho'') derived below - * and yChk a checksum over each polynomial of the secret mask y; - * baseline-zero and register both up front (single-exit function) so any - * later exit before the ForceZero is covered. */ + /* rho'' and, in blocking builds, the yChk checksums of y are on the stack; + * register them up front so every exit before the ForceZero is covered. */ #ifdef WOLFSSL_CHECK_MEM_ZERO XMEMSET(priv_rand_seed, 0, sizeof(priv_rand_seed)); wc_MemZero_Add("mldsa sign priv_rand_seed", priv_rand_seed, sizeof(priv_rand_seed)); +#ifndef WC_MLDSA_NONBLOCK XMEMSET(yChk, 0, sizeof(yChk)); wc_MemZero_Add("mldsa sign yChk", yChk, sizeof(yChk)); +#endif +#endif +#ifdef WC_MLDSA_NONBLOCK + /* Resume only the sign of this rnd||mu into this buffer. */ + (void)mldsa_nb_begin(key, MLDSA_NB_OP_SIGN, seedMu, 0, + MLDSA_RND_SZ + MLDSA_MU_SZ, sig); #endif /* Check the signature buffer isn't too small. */ if (*sigLen < params->sigSz) { @@ -13727,14 +13736,30 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Allocate memory for large intermediates. */ if (ret == 0) { - /* w-k, y-1, a-1, c-1, z-1, w1e, blocks. - * w0 replaces w in place and w1 is only kept encoded, so w is the - * only vector held. */ + /* w-k, y-1, a-1, c-1, [yChk]. z shares a, the sampling blocks + * share c then y, and w1 is hashed a polynomial at a time. */ allocSz = (unsigned int)params->s2Sz + - 4U * (unsigned int)MLDSA_POLY_SIZE + - (unsigned int)params->w1EncSz + - (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE; - w = (sword32*)XMALLOC(allocSz, key->heap, DYNAMIC_TYPE_MLDSA); + 3U * (unsigned int)MLDSA_POLY_SIZE; + #ifdef WC_MLDSA_NONBLOCK + /* yChk and the column walk's validity carried across yields. */ + allocSz += ((unsigned int)params->l + 1U) * + (unsigned int)sizeof(word64); + if ((key->nb != NULL) && (key->nb->work != NULL)) { + /* Resume: reuse the working allocation from the previous call. */ + w = (sword32*)key->nb->work; + resume = 1; + } + else + #endif + { + w = (sword32*)XMALLOC(allocSz, key->heap, DYNAMIC_TYPE_MLDSA); + #ifdef WC_MLDSA_NONBLOCK + if ((w != NULL) && (key->nb != NULL)) { + key->nb->work = w; + key->nb->workSz = allocSz; + } + #endif + } if (w == NULL) { ret = MEMORY_E; } @@ -13745,9 +13770,11 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, y = w + params->s2Sz / sizeof(*w); a = y + MLDSA_N; c = a + MLDSA_N; - z = c + MLDSA_N; - w1e = (byte*)(z + MLDSA_N); - blocks = w1e + params->w1EncSz; + z = a; + blocks = (byte*)c; + #ifdef WC_MLDSA_NONBLOCK + yChk = (word64*)(c + MLDSA_N); + #endif } } @@ -13758,29 +13785,40 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, MLDSA_PRIV_RAND_SEED_SZ); } if (ret == 0) { + #ifdef WC_MLDSA_NONBLOCK + word16 kappa = (key->nb != NULL) ? key->nb->kappa : 0; + #else word16 kappa = 0; + #endif int valid; /* Step 11: Start rejection sampling loop */ do { byte aseed[MLDSA_GEN_A_SEED_SZ]; byte* commit = sig; - byte* w1et = w1e; const byte* sp; sword32* wt; unsigned int e; sword32 hi; byte r; - byte s; + byte s = 0; valid = 1; + #ifdef WC_MLDSA_NONBLOCK + /* Resume the column walk where the last yield left it. */ + if (resume) { + s = (byte)key->nb->row; + valid = (int)yChk[params->l]; + resume = 0; + } + #endif /* Copy the seed into a buffer that has space for s and r. */ XMEMCPY(aseed, pub_seed, MLDSA_PUB_SEED_SZ); /* Alg 26. Step 2: Loop over second dimension of matrix. Working * down the columns keeps one polynomial of y and transforms it * once rather than once per row. */ - for (s = 0; (ret == 0) && (s < params->l); s++) { + for (; (ret == 0) && (s < params->l); s++) { #ifdef WC_MLDSA_FAULT_HARDEN if (w_check != w) { valid = 0; @@ -13789,10 +13827,10 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, } #endif /* Step 12: Compute polynomial of y from seed and kappa. */ - /* z is not live yet, so it doubles as the expansion scratch + /* a is not live yet, so it doubles as the expansion scratch * rather than allocating one per polynomial. */ ret = mldsa_expand_mask_poly(&key->shake, priv_rand_seed, - (word16)(kappa + s), params->gamma1_bits, y, (byte*)z); + (word16)(kappa + s), params->gamma1_bits, y, (byte*)a); if (ret != 0) { break; } @@ -13861,10 +13899,28 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, #endif } } + #ifdef WC_MLDSA_NONBLOCK + /* Yield per column; w and yChk persist in the allocation. */ + if ((key->nb != NULL) && (ret == 0) && + (s + 1 < params->l)) { + key->nb->row = (word16)(s + 1); + yChk[params->l] = (word64)valid; + ret = MP_WOULDBLOCK; + } + #endif } + /* Step 15: Hash mu and the encoded w1 a polynomial at a time. An + * attempt the y check rejected needs no commitment. */ + if ((ret == 0) && valid) { + ret = wc_Shake256_Reset(&key->shake); + } + if ((ret == 0) && valid) { + ret = wc_Shake256_Update(&key->shake, mu, MLDSA_MU_SZ); + } wt = w; - for (r = 0; (ret == 0) && (r < params->k); r++) { + for (r = valid ? 0 : params->k; (ret == 0) && (r < params->k); + r++) { /* Step 13: w = NTT-1(A o NTT(y)) */ ret = mldsa_poly_red(wt); #ifdef WOLFSSL_MLDSA_CHECK_INVNTT_BOUND @@ -13876,7 +13932,8 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, if (ret == 0) { ret = mldsa_invntt_full(wt); } - /* Step 14, Step 22: Make values positive and decompose. */ + /* Step 14, Step 22: Make values positive and decompose. w is + * kept whole and decomposed again for the hints. */ if (ret == 0) { ret = mldsa_make_pos(wt); } @@ -13884,12 +13941,12 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, if (params->gamma2 == MLDSA_Q_LOW_88) { /* For each value of polynomial. */ for (e = 0; e < MLDSA_N; e++) { - /* w0 replaces w, w1 goes to the scratch polynomial. */ - mldsa_decompose_q88(wt[e], &wt[e], &a[e]); + /* w0 goes to a, w1 to y. */ + mldsa_decompose_q88(wt[e], &a[e], &y[e]); } /* Step 15: Encode this polynomial of w1. */ if (ret == 0) { - ret = mldsa_encode_w1_88(a, w1et); + ret = mldsa_encode_w1_88(y, blocks); } } #endif @@ -13897,35 +13954,36 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, if (params->gamma2 == MLDSA_Q_LOW_32) { /* For each value of polynomial. */ for (e = 0; e < MLDSA_N; e++) { - /* w0 replaces w, w1 goes to the scratch polynomial. */ - mldsa_decompose_q32(wt[e], &wt[e], &a[e]); + /* w0 goes to a, w1 to y. */ + mldsa_decompose_q32(wt[e], &a[e], &y[e]); } /* Step 15: Encode this polynomial of w1. */ if (ret == 0) { - ret = mldsa_encode_w1_32(a, w1et); + ret = mldsa_encode_w1_32(y, blocks); } } #endif #ifdef WOLFSSL_MLDSA_SIGN_CHECK_W0 /* Accumulate: w0 comes from the secret mask, so an early exit * would leak which row rejected. */ - valid &= mldsa_check_low(wt, + valid &= mldsa_check_low(a, params->gamma2 - params->beta); #endif + if (ret == 0) { + ret = wc_Shake256_Update(&key->shake, blocks, w1Stride); + } wt += MLDSA_N; - w1et += w1Stride; } if ((ret == 0) && valid) { - /* Step 15: Hash mu and encoded w1. - * Step 32: Hash is stored in signature. */ - ret = mldsa_hash256(&key->shake, mu, MLDSA_MU_SZ, w1e, - params->w1EncSz, commit, params->lambda / 4); + /* Step 32: Hash is stored in signature. */ + ret = wc_Shake256_Final(&key->shake, commit, + params->lambda / 4); } if ((ret == 0) && valid) { /* Step 17: Compute c from first 256 bits of commit. */ ret = mldsa_sample_in_ball_ex(params->level, &key->shake, - commit, params->lambda / 4, params->tau, c, blocks); + commit, params->lambda / 4, params->tau, c, (byte*)y); } if ((ret == 0) && valid) { /* Step 18: NTT(c). */ @@ -13941,10 +13999,8 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, * discarded round; y masks which one it was. */ for (s = 0; (ret == 0) && valid && (s < params->l); s++) { /* Vector y is not kept, so regenerate this polynomial. */ - /* z is overwritten by the multiply below, so it doubles - * as the expansion scratch. */ ret = mldsa_expand_mask_poly(&key->shake, priv_rand_seed, - (word16)(kappa + s), params->gamma1_bits, y, (byte*)z); + (word16)(kappa + s), params->gamma1_bits, y, (byte*)a); if (ret != 0) { break; } @@ -13958,7 +14014,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, if (ret == 0) { ret = mldsa_ntt_small(a); } - /* Step 19: cs1 = NTT-1(c o s1) */ + /* Step 19: cs1 = NTT-1(c o s1), in place as z shares a. */ if (ret == 0) { ret = mldsa_mul(z, c, a); } @@ -14003,8 +14059,23 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, sp = s2p; wt = w; - w1et = w1e; for (r = 0; (ret == 0) && valid && (r < params->k); r++) { + /* Decompose w again: w0 in place, w1 into y. */ + #ifndef WOLFSSL_NO_ML_DSA_44 + if (params->gamma2 == MLDSA_Q_LOW_88) { + for (e = 0; e < MLDSA_N; e++) { + mldsa_decompose_q88(wt[e], &wt[e], &y[e]); + } + } + #endif + #if !defined(WOLFSSL_NO_ML_DSA_65) || \ + !defined(WOLFSSL_NO_ML_DSA_87) + if (params->gamma2 == MLDSA_Q_LOW_32) { + for (e = 0; e < MLDSA_N; e++) { + mldsa_decompose_q32(wt[e], &wt[e], &y[e]); + } + } + #endif ret = mldsa_vec_decode_eta_bits(sp, params->eta, a, 1); if (ret == 0) { ret = mldsa_ntt_small(a); @@ -14049,10 +14120,6 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, ret = mldsa_poly_red(z); } - /* w1 is only kept encoded, so recover the polynomial - * the hint needs. */ - mldsa_decode_w1(w1et, params->gamma2, a); - /* Step 26, 27: Make hint from ct0 and w1 and check * number of hints is valid. * Step 32: h is encoded into signature. @@ -14060,7 +14127,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, #ifndef WOLFSSL_NO_ML_DSA_44 if ((ret == 0) && (params->gamma2 == MLDSA_Q_LOW_88)) { - ret = mldsa_make_hint_88(z, a, h, &idx, &valid); + ret = mldsa_make_hint_88(z, y, h, &idx, &valid); /* Alg 14, Step 10: Store count of hints for * polynomial at end of list. */ if (ret == 0) { @@ -14072,7 +14139,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, !defined(WOLFSSL_NO_ML_DSA_87) if ((ret == 0) && (params->gamma2 == MLDSA_Q_LOW_32)) { - ret = mldsa_make_hint_32(z, a, params->omega, + ret = mldsa_make_hint_32(z, y, params->omega, h, &idx, &valid); /* Alg 14, Step 10: Store count of hints for * polynomial at end of list. */ @@ -14086,7 +14153,6 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, sp += sStride; t0pt += MLDSA_D * MLDSA_N / 8; wt += MLDSA_N; - w1et += w1Stride; } valid &= ct0Valid; /* Set remaining hints to zero. Only for a valid attempt: a @@ -14097,7 +14163,10 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, } } - if (!valid) { + if ((ret == 0) && (!valid)) { + /* Wipe the rejected candidate; yChk is rebuilt per column. */ + ForceZero(w, allocSz); + ForceZero(sig, params->sigSz); /* Too many attempts - something wrong with implementation. */ if ((kappa > (word16)(kappa + params->l))) { ret = BAD_COND_E; @@ -14106,34 +14175,59 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Step 30: increment value to append to seed to unique value. */ kappa = (word16)(kappa + params->l); + #ifdef WC_MLDSA_NONBLOCK + /* Rejected attempt: restart the column walk and yield. */ + if ((key->nb != NULL) && (ret == 0)) { + key->nb->kappa = kappa; + key->nb->row = 0; + yChk[params->l] = 1; + ret = MP_WOULDBLOCK; + } + #endif } } /* Step 11: Check we have a valid signature. */ while ((ret == 0) && (!valid)); } - if ((ret != 0) && (*sigLen == params->sigSz)) { + if ((ret != 0) && +#ifdef WC_MLDSA_NONBLOCK + (ret != WC_NO_ERR_TRACE(MP_WOULDBLOCK)) && +#endif + (*sigLen == params->sigSz)) { /* FIPS 204 3.6.3: a failed sign leaves no part of a signature. */ ForceZero(sig, params->sigSz); } ForceZero(priv_rand_seed, sizeof(priv_rand_seed)); - /* The last expansion of y leaves rho'' recoverable from key->shake. - * Re-initializing clears the state but keeps what its free needs. */ - if (wc_InitShake256(&key->shake, NULL, INVALID_DEVID) != 0) { - ForceZero(&key->shake, sizeof(key->shake)); - } + /* The last expansion of y leaves rho'' recoverable from key->shake. */ + mldsa_shake_wipe(&key->shake); #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(priv_rand_seed, sizeof(priv_rand_seed)); #endif +#ifndef WC_MLDSA_NONBLOCK /* Checksums are derived from the secret mask y. */ ForceZero(yChk, sizeof(yChk)); #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(yChk, sizeof(yChk)); #endif - if (w != NULL) { - ForceZero(w, allocSz); +#endif +#ifdef WC_MLDSA_NONBLOCK + if ((key->nb != NULL) && (ret == WC_NO_ERR_TRACE(MP_WOULDBLOCK))) { + /* Yielding: keep the working allocation for the next call. */ + } + else if (key->nb != NULL) { + /* Final result: the context owns the allocation, even when this call + * failed before picking it up. */ + mldsa_nb_release(key); + } + else +#endif + { + if (w != NULL) { + ForceZero(w, allocSz); + } + XFREE(w, key->heap, DYNAMIC_TYPE_MLDSA); } - XFREE(w, key->heap, DYNAMIC_TYPE_MLDSA); return ret; #endif } diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 74d42b0a6d6..96667a25be5 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -78743,20 +78743,116 @@ static wc_test_ret_t mldsa_nonblock_test_level(int level, WC_RNG* rng) return ret; } +/* Seeded key generation and signing must give the same bytes whether run in + * one call or resumed across yields. */ +static wc_test_ret_t mldsa_nonblock_match_test_level(int level) +{ + wc_test_ret_t ret = 0; + int rc = 0; + wc_MlDsaKey* key = NULL; + wc_MlDsaKey* nbKey = NULL; + wc_MlDsaNonBlock nb; + byte* sig = NULL; + byte* nbSig = NULL; + int sigSz = 0; + word32 sigLen; + word32 nbSigLen; + int res; + int i; + int yields = 0; + int minYields = -1; + int maxYields = 0; + int nbInited = 0; + int keyInited = 0; + byte seed[MLDSA_SEED_SZ]; + byte msg[32]; + + XMEMSET(seed, 0x5a, sizeof(seed)); + XMEMSET(msg, 0xa5, sizeof(msg)); + + key = (wc_MlDsaKey*)XMALLOC(sizeof(*key), HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + nbKey = (wc_MlDsaKey*)XMALLOC(sizeof(*nbKey), HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if ((key == NULL) || (nbKey == NULL)) + ret = WC_TEST_RET_ENC_EC(MEMORY_E); + NB_STEP(wc_MlDsaKey_Init(key, HEAP_HINT, devId)); + keyInited = (ret == 0); + NB_STEP(wc_MlDsaKey_SetParams(key, (byte)level)); + NB_STEP(wc_MlDsaKey_Init(nbKey, HEAP_HINT, devId)); + nbInited = (ret == 0); + NB_STEP(wc_MlDsaKey_SetParams(nbKey, (byte)level)); + NB_STEP(wc_MlDsaKey_set_nonblock(nbKey, &nb)); + NB_STEP(wc_MlDsaKey_MakeKeyFromSeed(key, seed)); + NB_RUN(wc_MlDsaKey_MakeKeyFromSeed(nbKey, seed)); + NB_STEP(wc_MlDsaKey_GetSigLen(key, &sigSz)); + if (ret == 0) { + sig = (byte*)XMALLOC((size_t)sigSz, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + nbSig = (byte*)XMALLOC((size_t)sigSz, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if ((sig == NULL) || (nbSig == NULL)) + ret = WC_TEST_RET_ENC_EC(MEMORY_E); + } + for (i = 0; (ret == 0) && (i < 8); i++) { + msg[0] = (byte)i; + sigLen = (word32)sigSz; + nbSigLen = (word32)sigSz; + NB_STEP(wc_MlDsaKey_SignCtxWithSeed(key, NULL, 0, sig, &sigLen, msg, + (word32)sizeof(msg), seed)); + NB_DRIVE(wc_MlDsaKey_SignCtxWithSeed(nbKey, NULL, 0, nbSig, &nbSigLen, + msg, (word32)sizeof(msg), seed)); + if ((ret == 0) && (rc != 0)) + ret = WC_TEST_RET_ENC_EC(rc); + if (ret == 0) { + if ((minYields < 0) || (yields < minYields)) + minYields = yields; + if (yields > maxYields) + maxYields = yields; + } + if ((ret == 0) && ((nbSigLen != sigLen) || + (XMEMCMP(sig, nbSig, sigLen) != 0))) { + ret = WC_TEST_RET_ENC_NC; + } + res = 0; + NB_RUN(wc_MlDsaKey_VerifyCtx(nbKey, nbSig, nbSigLen, NULL, 0, msg, + (word32)sizeof(msg), &res)); + if ((ret == 0) && (res != 1)) + ret = WC_TEST_RET_ENC_NC; + } + if ((ret == 0) && (maxYields <= minYields)) + ret = WC_TEST_RET_ENC_NC; + + XFREE(nbSig, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(sig, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (nbInited) + wc_MlDsaKey_Free(nbKey); + if (keyInited) + wc_MlDsaKey_Free(key); + XFREE(nbKey, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(key, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return ret; +} + static wc_test_ret_t mldsa_nonblock_test(WC_RNG* rng) { wc_test_ret_t ret = 0; #ifndef WOLFSSL_NO_ML_DSA_44 if (ret == 0) ret = mldsa_nonblock_test_level(WC_ML_DSA_44, rng); + if (ret == 0) + ret = mldsa_nonblock_match_test_level(WC_ML_DSA_44); #endif #ifndef WOLFSSL_NO_ML_DSA_65 if (ret == 0) ret = mldsa_nonblock_test_level(WC_ML_DSA_65, rng); + if (ret == 0) + ret = mldsa_nonblock_match_test_level(WC_ML_DSA_65); #endif #ifndef WOLFSSL_NO_ML_DSA_87 if (ret == 0) ret = mldsa_nonblock_test_level(WC_ML_DSA_87, rng); + if (ret == 0) + ret = mldsa_nonblock_match_test_level(WC_ML_DSA_87); #endif return ret; } From 7902f3e8580391a03f574a34e27f4ddb14a553a5 Mon Sep 17 00:00:00 2001 From: Tobias Frauenschlaeger Date: Mon, 9 Nov 2026 07:51:25 +0000 Subject: [PATCH 7/7] Keep c sparse in the small memory ML-DSA verify c has only tau non-zero coefficients, yet the small memory verifier held NTT(c) as a full polynomial for the whole row loop. Sample c into the free w polynomial, keep its positions and signs (2 * tau bytes), and rebuild NTT(c) in the scratch polynomial of A at the start of each row, before A needs it. Heap for ML-DSA-44/65/87 verification drops from 8777/9801/12105 to 7831/8875/11201 bytes with VERIFY_SMALL_MEM and from 5705/5705/5961 to 4759/4779/5057 with VERIFY_SMALLEST_MEM. With VERIFY_NO_MALLOC the key pins the 120-byte sparse form instead of c, and the SampleInBall block shares key->h, so wc_MlDsaKey shrinks by 1040 bytes. The cost is one small NTT per row instead of one per verification, about 5 to 10% of the verify time in the C build. Subtracting c t1 directly after the inverse NTT saves the same memory but measured 12 to 20% slower. WC_MLDSA_CACHE_PUB_VECTORS keeps NTT(c) whole, since its t1 is cached in the NTT domain. The non-blocking verify no longer yields after the now empty NTT(c) phase. Two pq-all rows run the pinned sparse form, for all levels and for ML-DSA-44 alone; the existing VERIFY_NO_MALLOC row caches the public vectors and keeps NTT(c) whole. --- .github/configs/pq-all.json | 8 ++++ wolfcrypt/src/wc_mldsa.c | 90 +++++++++++++++++++++++++++++++++--- wolfssl/wolfcrypt/wc_mldsa.h | 14 +++++- 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/.github/configs/pq-all.json b/.github/configs/pq-all.json index 00140c25fc2..466b1903c12 100644 --- a/.github/configs/pq-all.json +++ b/.github/configs/pq-all.json @@ -429,6 +429,14 @@ "comment": "Cached public vectors alongside the pinned verify buffers, the combination whose key structure member clash stopped it building; the clashing member only exists when the small memory verify is selected too", "configure": ["--enable-cryptonly", "--enable-mldsa", "CPPFLAGS=-DWC_MLDSA_CACHE_PUB_VECTORS -DWOLFSSL_MLDSA_VERIFY_NO_MALLOC -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM"]}, +{"name": "mldsa-verify-no-malloc-sparse-c", "minutes": 2, + "comment": "Pinned verify buffers without cached public vectors, the only run of the sparse c verify that keeps c in key->cSparse and samples it with key->h as the block", + "configure": ["--enable-cryptonly", "--enable-mldsa", + "CPPFLAGS=-DWOLFSSL_MLDSA_VERIFY_NO_MALLOC -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM"]}, +{"name": "mldsa-44-verify-no-malloc-sparse-c", "minutes": 2, + "comment": "As mldsa-verify-no-malloc-sparse-c with ML-DSA-44 only, so key->cSparse is sized by the ML-DSA-44 tau", + "configure": ["--enable-cryptonly", "--enable-mldsa", + "CPPFLAGS=-DWOLFSSL_MLDSA_VERIFY_NO_MALLOC -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_NO_ML_DSA_65 -DWOLFSSL_NO_ML_DSA_87"]}, {"name": "mldsa-cache-matrix", "minutes": 2, "comment": "Cached matrix A on the default full vector signer, the only build where the signing cache allocation test's key->a and aSet assertions are compiled", "configure": ["--enable-cryptonly", "--enable-mldsa", diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index 8a68e25a457..16704cc8e88 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -47,6 +47,8 @@ * Compiles in only the verification and public key operations. * WOLFSSL_MLDSA_VERIFY_SMALL_MEM Default: OFF * Compiles verification implementation that uses smaller amounts of memory. + * Holds c as the positions and signs of its non-zero coefficients and + * rebuilds NTT(c) for each row, unless WC_MLDSA_CACHE_PUB_VECTORS is set. * WOLFSSL_MLDSA_VERIFY_SMALLEST_MEM Default: OFF * Implies WOLFSSL_MLDSA_VERIFY_SMALL_MEM. Decodes and transforms vector z a * polynomial at a time instead of holding the whole vector, k times rather @@ -758,6 +760,10 @@ wc_static_assert(sizeof(sword32) * MLDSA_N >= MLDSA_MAX_V); /* The small memory signers sample A into a polynomial buffer. */ wc_static_assert(MLDSA_POLY_SIZE >= MLDSA_REJ_NTT_POLY_H_SIZE); #endif +#if !defined(WOLFSSL_MLDSA_NO_VERIFY) && defined(WOLFSSL_MLDSA_VERIFY_NO_MALLOC) +/* The no-malloc verify samples c with key->h as its block. */ +wc_static_assert(MLDSA_REJ_NTT_POLY_H_SIZE >= MLDSA_GEN_C_BLOCK_BYTES); +#endif /* 2 blocks, each block 136 bytes = 272 bytes. @@ -14795,6 +14801,44 @@ static int mldsa_make_pub_vec(wc_MlDsaKey* key, sword32* t1) } #endif +#ifdef MLDSA_VERIFY_SPARSE_C +/* Record the positions and signs of the tau non-zero coefficients of c. + * + * @param [in] c Polynomial with tau coefficients of 1 or -1. + * @param [in] tau Number of non-zero coefficients. + * @param [out] cSparse Positions, then sign flags (1 for -1), tau bytes each. + */ +static void mldsa_make_sparse(const sword32* c, byte tau, byte* cSparse) +{ + unsigned int e; + unsigned int j = 0; + + for (e = 0; (e < MLDSA_N) && (j < tau); e++) { + if (c[e] != 0) { + cSparse[j] = (byte)e; + cSparse[tau + j] = (byte)(c[e] < 0); + j++; + } + } +} + +/* Rebuild c from its positions and signs. + * + * @param [in] cSparse Positions, then sign flags (1 for -1), tau bytes each. + * @param [in] tau Number of non-zero coefficients. + * @param [out] c Polynomial with tau coefficients of 1 or -1. + */ +static void mldsa_expand_sparse(const byte* cSparse, byte tau, sword32* c) +{ + unsigned int j; + + XMEMSET(c, 0, MLDSA_POLY_SIZE); + for (j = 0; j < tau; j++) { + c[cSparse[j]] = 1 - 2 * (sword32)cSparse[tau + j]; + } +} +#endif /* MLDSA_VERIFY_SPARSE_C */ + /* Verify signature of message using public key. * * FIPS 204 Section 5.3, Algorithm 3 ML-DSA.Verify(pk, M, sigma, ctx) @@ -15076,6 +15120,9 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, const sword32* a = NULL; #ifdef WOLFSSL_MLDSA_VERIFY_PRECOMP_A const sword32* aPre = mldsa_precomp_a(key); +#endif +#ifdef MLDSA_VERIFY_SPARSE_C + byte* cSparse = NULL; #endif sword32* c = NULL; sword32* z = NULL; @@ -15083,9 +15130,7 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 sword64* t64 = NULL; #endif -#ifndef WOLFSSL_MLDSA_VERIFY_NO_MALLOC byte* block = NULL; -#endif byte* w1e = NULL; byte commit_calc[MLDSA_TR_SZ]; int valid = 0; @@ -15138,13 +15183,18 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, #ifndef WOLFSSL_MLDSA_VERIFY_NO_MALLOC /* Allocate memory for large intermediates. */ if (ret == 0) { - /* z, c-1, w-1, t1-1, [t64], block, w1e */ + /* z, [c-1], w-1, t1-1, [t64], block, w1e, [cSparse] */ unsigned int allocSz; allocSz = zSz + - 3U * (unsigned int)MLDSA_POLY_SIZE + + 2U * (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE + params->w1EncSz; + #ifdef MLDSA_VERIFY_SPARSE_C + allocSz += 2U * (unsigned int)params->tau; + #else + allocSz += (unsigned int)MLDSA_POLY_SIZE; + #endif #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 allocSz += (unsigned int)MLDSA_POLY_SIZE * 2U; #endif @@ -15176,8 +15226,12 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, { XMEMSET(z, 0, allocSz); } + #ifdef MLDSA_VERIFY_SPARSE_C + w = z + zSz / sizeof(*t1); + #else c = z + zSz / sizeof(*t1); w = c + MLDSA_N; + #endif t1 = w + MLDSA_N; aBuf = t1; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 @@ -15187,16 +15241,25 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, block = (byte*)(t1 + MLDSA_N); #endif w1e = block + MLDSA_REJ_NTT_POLY_H_SIZE; + #ifdef MLDSA_VERIFY_SPARSE_C + cSparse = w1e + params->w1EncSz; + #endif } } #else if (ret == 0) { z = key->z; + #ifdef MLDSA_VERIFY_SPARSE_C + cSparse = key->cSparse; + #else c = key->c; + #endif w = key->w; t1 = key->vt1; w1e = key->w1e; aBuf = t1; + /* The sampling block is done with before A needs key->h. */ + block = key->h; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 t64 = key->t64; #endif @@ -15255,9 +15318,12 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, if ((ret == 0) && valid) { /* Step 9: Compute c from first 256 bits of commit. */ -#ifdef WOLFSSL_MLDSA_VERIFY_NO_MALLOC +#ifdef MLDSA_VERIFY_SPARSE_C ret = mldsa_sample_in_ball_ex(params->level, &key->shake, commit, - params->lambda / 4, params->tau, c, key->block); + params->lambda / 4, params->tau, w, block); + if (ret == 0) { + mldsa_make_sparse(w, params->tau, cSparse); + } #else ret = mldsa_sample_in_ball_ex(params->level, &key->shake, commit, params->lambda / 4, params->tau, c, block); @@ -15274,16 +15340,20 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, #endif FALL_THROUGH; case 1: +#ifndef MLDSA_VERIFY_SPARSE_C if ((ret == 0) && valid) { ret = mldsa_ntt_small_full(c); } +#endif #ifdef WC_MLDSA_NONBLOCK if ((key->nb != NULL) && (ret == 0) && valid) { key->nb->state = 2; key->nb->row = 0; key->nb->hintPos = 0; + #ifndef MLDSA_VERIFY_SPARSE_C ret = MP_WOULDBLOCK; break; + #endif } #endif FALL_THROUGH; @@ -15314,6 +15384,14 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, * result goes to w either way, so a cached polynomial is read in * place rather than copied. */ const sword32* t1v = w; +#ifdef MLDSA_VERIFY_SPARSE_C + /* Rebuild NTT(c) in the scratch of A, which is free until below. */ + c = aBuf; + if (ret == 0) { + mldsa_expand_sparse(cSparse, params->tau, c); + ret = mldsa_ntt_small_full(c); + } +#endif #ifdef WC_MLDSA_CACHE_PUB_VECTORS if (key->pubVecSet) { diff --git a/wolfssl/wolfcrypt/wc_mldsa.h b/wolfssl/wolfcrypt/wc_mldsa.h index d2aa0cb6204..e3a7849752c 100644 --- a/wolfssl/wolfcrypt/wc_mldsa.h +++ b/wolfssl/wolfcrypt/wc_mldsa.h @@ -437,6 +437,7 @@ #define MLDSA_MAX_W1_ENC_SZ PARAMS_ML_DSA_87_W1_ENC_SZ /* Maximum collision strength of c-tilde in bytes. */ #define MLDSA_MAX_LAMBDA PARAMS_ML_DSA_87_LAMBDA +#define MLDSA_MAX_TAU PARAMS_ML_DSA_87_TAU /* Maximum count of elements of a vector with dimension K. */ #define MLDSA_MAX_K_VECTOR_COUNT \ @@ -454,6 +455,7 @@ #define MLDSA_MAX_W1_ENC_SZ PARAMS_ML_DSA_65_W1_ENC_SZ /* Maximum collision strength of c-tilde in bytes. */ #define MLDSA_MAX_LAMBDA PARAMS_ML_DSA_65_LAMBDA +#define MLDSA_MAX_TAU PARAMS_ML_DSA_65_TAU /* Maximum count of elements of a vector with dimension K. */ #define MLDSA_MAX_K_VECTOR_COUNT \ @@ -471,6 +473,7 @@ #define MLDSA_MAX_W1_ENC_SZ PARAMS_ML_DSA_44_W1_ENC_SZ /* Maximum collision strength of c-tilde in bytes. */ #define MLDSA_MAX_LAMBDA PARAMS_ML_DSA_44_LAMBDA +#define MLDSA_MAX_TAU PARAMS_ML_DSA_44_TAU /* Maximum count of elements of a vector with dimension K. */ #define MLDSA_MAX_K_VECTOR_COUNT \ @@ -619,6 +622,12 @@ typedef struct wc_MlDsaParams { typedef struct wc_MlDsaParams MlDsaParams; #endif +/* Small memory verify keeps c sparse unless NTT(t1) is cached. */ +#if defined(WOLFSSL_MLDSA_VERIFY_SMALL_MEM) && \ + !defined(WC_MLDSA_CACHE_PUB_VECTORS) + #define MLDSA_VERIFY_SPARSE_C +#endif + #ifdef WC_MLDSA_NONBLOCK /* Resumable state for a non-blocking ML-DSA operation. Attach one to a key with * wc_MlDsaKey_set_nonblock(); the operation then runs one chunk per call, @@ -747,7 +756,11 @@ struct wc_MlDsaKey { #else sword32 z[MLDSA_MAX_L_VECTOR_COUNT]; #endif +#ifdef MLDSA_VERIFY_SPARSE_C + byte cSparse[2 * MLDSA_MAX_TAU]; /* positions then signs of c */ +#else sword32 c[MLDSA_N]; +#endif sword32 w[MLDSA_N]; /* One t1 polynomial, also used for a polynomial of A. Not named t1 as * WC_MLDSA_CACHE_PUB_VECTORS already has a member of that name. */ @@ -757,7 +770,6 @@ struct wc_MlDsaKey { sword64 t64[MLDSA_N]; #endif byte h[MLDSA_REJ_NTT_POLY_H_SIZE]; - byte block[MLDSA_GEN_C_BLOCK_BYTES]; #endif /* WOLFSSL_MLDSA_VERIFY_NO_MALLOC && * WOLFSSL_MLDSA_VERIFY_SMALL_MEM */ };