diff --git a/.github/configs/pq-all.json b/.github/configs/pq-all.json index 0bc09c3612f..466b1903c12 100644 --- a/.github/configs/pq-all.json +++ b/.github/configs/pq-all.json @@ -25,6 +25,14 @@ "configure": ["--enable-dilithium=yes,small", "--enable-mlkem", "--enable-experimental", "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_LARGE_CODE -DWC_MLDSA_NONBLOCK -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWC_MLKEM_NONBLOCK"]}, +{"name": "pq-nonblock-smallest", "minutes": 2.0, + "comment": "Non-blocking ML-DSA on the smallest memory signer and verifier: mldsa_nonblock_test resumes the column walk and compares seeded output with the blocking path", + "configure": ["--enable-cryptonly", "--enable-mldsa", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -DWOLFSSL_MLDSA_SIGN_SMALLEST_MEM -DWOLFSSL_MLDSA_VERIFY_SMALLEST_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWC_MLDSA_NONBLOCK"]}, +{"name": "pq-nonblock-smallest-checks", "minutes": 2.0, + "comment": "Non-blocking smallest memory signer with the early y and w0 checks and fault hardening: the y check result is carried across yields", + "configure": ["--enable-cryptonly", "--enable-mldsa=yes,small", + "CPPFLAGS=-DWOLFSSL_MLDSA_SIGN_SMALLEST_MEM -DWOLFSSL_MLDSA_VERIFY_SMALLEST_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_SIGN_CHECK_Y -DWOLFSSL_MLDSA_SIGN_CHECK_W0 -DWC_MLDSA_FAULT_HARDEN -DWC_MLDSA_NONBLOCK"]}, {"name": "pq-nonblock-smallmem-dynkeys", "minutes": 2.5, "comment": "Non-blocking ML-KEM with the cached matrix A and dynamic key buffers: a resumed keygen must keep the vectors its first slice allocated, and encapsulation must read the cached matrix across yields", "configure": ["--enable-cryptonly", "--enable-mlkem", "--enable-smallstack", @@ -361,6 +369,14 @@ "--disable-examples"], "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "QEMU_CPU=max", "./wolfcrypt/test/testwolfcrypt"]]}, +{"name": "mldsa-aarch64-armasm-smallest", "minutes": 0.5, "check": false, + "cc": "ccache aarch64-linux-gnu-gcc", + "comment": "AArch64 NEON with the smallest memory signer and verifier and the small memory key generation, which leave the batched gen_y and gen_s lanes out. sha3-crypto builds the SHA-3 instruction wrappers as functions, and without the key check matrix A expansion does not use them either", + "configure": ["--host=aarch64-linux-gnu", "--enable-cryptonly", + "--enable-armasm=sha3-crypto", "--enable-mldsa", "--disable-examples", + "CPPFLAGS=-DWOLFSSL_MLDSA_SIGN_SMALLEST_MEM -DWOLFSSL_MLDSA_VERIFY_SMALLEST_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_CHECK_KEY"], + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "QEMU_CPU=max", + "./wolfcrypt/test/testwolfcrypt"]]}, {"name": "mldsa-mlkem-aarch64-armasm-inline", "minutes": 0.5, "check": false, "cc": "ccache aarch64-linux-gnu-gcc", "comment": "Same, through the inline-asm backend.", @@ -413,6 +429,14 @@ "comment": "Cached public vectors alongside the pinned verify buffers, the combination whose key structure member clash stopped it building; the clashing member only exists when the small memory verify is selected too", "configure": ["--enable-cryptonly", "--enable-mldsa", "CPPFLAGS=-DWC_MLDSA_CACHE_PUB_VECTORS -DWOLFSSL_MLDSA_VERIFY_NO_MALLOC -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM"]}, +{"name": "mldsa-verify-no-malloc-sparse-c", "minutes": 2, + "comment": "Pinned verify buffers without cached public vectors, the only run of the sparse c verify that keeps c in key->cSparse and samples it with key->h as the block", + "configure": ["--enable-cryptonly", "--enable-mldsa", + "CPPFLAGS=-DWOLFSSL_MLDSA_VERIFY_NO_MALLOC -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM"]}, +{"name": "mldsa-44-verify-no-malloc-sparse-c", "minutes": 2, + "comment": "As mldsa-verify-no-malloc-sparse-c with ML-DSA-44 only, so key->cSparse is sized by the ML-DSA-44 tau", + "configure": ["--enable-cryptonly", "--enable-mldsa", + "CPPFLAGS=-DWOLFSSL_MLDSA_VERIFY_NO_MALLOC -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_NO_ML_DSA_65 -DWOLFSSL_NO_ML_DSA_87"]}, {"name": "mldsa-cache-matrix", "minutes": 2, "comment": "Cached matrix A on the default full vector signer, the only build where the signing cache allocation test's key->a and aSet assertions are compiled", "configure": ["--enable-cryptonly", "--enable-mldsa", diff --git a/tests/api/test_mldsa.c b/tests/api/test_mldsa.c index 6a89e617983..ea84c8b156e 100644 --- a/tests/api/test_mldsa.c +++ b/tests/api/test_mldsa.c @@ -31684,6 +31684,152 @@ int test_mldsa_pkcs8_export_import_wolfSSL_form(void) return EXPECT_RESULT(); } +#if defined(WOLFSSL_HAVE_MLDSA) && defined(WC_MLDSA_HAVE_NATIVE) && \ + !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) +#define MLDSA_TEST_ENC_MAX_K 8 +#define MLDSA_TEST_ENC_GUARD 32 +#define MLDSA_TEST_ENC_SZ \ + (MLDSA_TEST_ENC_MAX_K * MLDSA_N * MLDSA_D / 8 + MLDSA_TEST_ENC_GUARD) + +/* Pack n values of bits bits each, little-endian, as FIPS 204 does. */ +static void test_mldsa_pack_bits(const sword32* v, word32 n, word32 bits, + byte* out) +{ + word32 i; + word32 accBits = 0; + word64 acc = 0; + + for (i = 0; i < n; i++) { + acc |= (word64)(word32)v[i] << accBits; + accBits += bits; + while (accBits >= 8) { + *(out++) = (byte)acc; + acc >>= 8; + accBits -= 8; + } + } +} + +static int test_mldsa_guard_intact(const byte* p) +{ + int i; + + for (i = 0; i < MLDSA_TEST_ENC_GUARD; i++) { + if (p[i] != 0xa5) { + return 0; + } + } + return 1; +} + +static word32 test_mldsa_rand(word32* state) +{ + *state = *state * 1103515245U + 12345U; + return *state >> 8; +} +#endif + +/* The vector encoders write exactly the encoded length, for every count. */ +int test_mldsa_vec_encode_exact_len(void) +{ + EXPECT_DECLS; +#if defined(WOLFSSL_HAVE_MLDSA) && defined(WC_MLDSA_HAVE_NATIVE) && \ + !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) + static const byte etas[] = { + #if !defined(WOLFSSL_NO_ML_DSA_44) || !defined(WOLFSSL_NO_ML_DSA_87) + 2, + #endif + #ifndef WOLFSSL_NO_ML_DSA_65 + 4, + #endif + }; + sword32* s = NULL; + sword32* v = NULL; + byte* enc = NULL; + byte* enc1 = NULL; + byte* exp = NULL; + word32 rnd = 1; + word32 len; + word32 len1; + word32 i; + word32 e; + byte d; + + ExpectNotNull(s = (sword32*)XMALLOC(MLDSA_TEST_ENC_MAX_K * MLDSA_N * + sizeof(sword32), NULL, DYNAMIC_TYPE_TMP_BUFFER)); + ExpectNotNull(v = (sword32*)XMALLOC(MLDSA_TEST_ENC_MAX_K * MLDSA_N * + sizeof(sword32), NULL, DYNAMIC_TYPE_TMP_BUFFER)); + ExpectNotNull(enc = (byte*)XMALLOC(MLDSA_TEST_ENC_SZ, NULL, + DYNAMIC_TYPE_TMP_BUFFER)); + ExpectNotNull(enc1 = (byte*)XMALLOC(MLDSA_TEST_ENC_SZ, NULL, + DYNAMIC_TYPE_TMP_BUFFER)); + ExpectNotNull(exp = (byte*)XMALLOC(MLDSA_TEST_ENC_SZ, NULL, + DYNAMIC_TYPE_TMP_BUFFER)); + +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && \ + defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); +#endif + + /* Every count, so the last store of each vector length is checked. */ + for (e = 0; EXPECT_SUCCESS() && (e < sizeof(etas)); e++) { + byte eta = etas[e]; + word32 bits = (eta == 2) ? 3 : 4; + + for (d = 1; EXPECT_SUCCESS() && (d <= MLDSA_TEST_ENC_MAX_K); d++) { + len = (word32)d * MLDSA_N * bits / 8; + for (i = 0; i < (word32)d * MLDSA_N; i++) { + s[i] = (sword32)(test_mldsa_rand(&rnd) % (2U * eta + 1)) - + eta; + v[i] = eta - s[i]; + } + test_mldsa_pack_bits(v, (word32)d * MLDSA_N, bits, exp); + XMEMSET(enc, 0xa5, MLDSA_TEST_ENC_SZ); + ExpectIntEQ(wc_mldsa_vec_encode_eta(s, d, eta, enc), 0); + ExpectBufEQ(enc, exp, len); + ExpectIntEQ(test_mldsa_guard_intact(enc + len), 1); + } + } + + for (d = 1; EXPECT_SUCCESS() && (d <= MLDSA_TEST_ENC_MAX_K); d++) { + len = (word32)d * MLDSA_N * MLDSA_D / 8; + len1 = (word32)d * MLDSA_N * MLDSA_U / 8; + for (i = 0; i < (word32)d * MLDSA_N; i++) { + s[i] = (sword32)(test_mldsa_rand(&rnd) % MLDSA_Q); + } + /* Power2Round: t1 in the top bits, 2^(d-1) - t0 in the low ones. */ + for (i = 0; i < (word32)d * MLDSA_N; i++) { + sword32 t1 = (s[i] + (1 << (MLDSA_D - 1)) - 1) >> MLDSA_D; + v[i] = (1 << (MLDSA_D - 1)) - (s[i] - (t1 << MLDSA_D)); + } + test_mldsa_pack_bits(v, (word32)d * MLDSA_N, MLDSA_D, exp); + XMEMSET(enc, 0xa5, MLDSA_TEST_ENC_SZ); + XMEMSET(enc1, 0xa5, MLDSA_TEST_ENC_SZ); + ExpectIntEQ(wc_mldsa_vec_encode_t0_t1(s, d, enc, enc1), 0); + ExpectBufEQ(enc, exp, len); + ExpectIntEQ(test_mldsa_guard_intact(enc + len), 1); + for (i = 0; i < (word32)d * MLDSA_N; i++) { + v[i] = (s[i] + (1 << (MLDSA_D - 1)) - 1) >> MLDSA_D; + } + test_mldsa_pack_bits(v, (word32)d * MLDSA_N, MLDSA_U, exp); + ExpectBufEQ(enc1, exp, len1); + ExpectIntEQ(test_mldsa_guard_intact(enc1 + len1), 1); + } + +#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && \ + defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING) + WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0); +#endif + + XFREE(exp, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(enc1, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(enc, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(v, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(s, NULL, DYNAMIC_TYPE_TMP_BUFFER); +#endif + return EXPECT_RESULT(); +} + /* Exercise w1 encoding with adversarial inputs to catch undefined behavior. * * The word32-optimized paths in mldsa_encode_w1_88_c and diff --git a/tests/api/test_mldsa.h b/tests/api/test_mldsa.h index 74f212d9451..9fc042bab8b 100644 --- a/tests/api/test_mldsa.h +++ b/tests/api/test_mldsa.h @@ -70,6 +70,7 @@ int test_mldsa_verify_mu_kats(void); int test_mldsa_PrivateKeyDecode_OpenSSL_form(void); int test_mldsa_pkcs8_import_OpenSSL_form(void); int test_mldsa_pkcs8_export_import_wolfSSL_form(void); +int test_mldsa_vec_encode_exact_len(void); int test_mldsa_encode_w1_large_values(void); int test_mldsa_pkcs12(void); int test_mldsa_x509_pubkey_sigtype(void); @@ -132,6 +133,7 @@ int test_mldsa_make_key_id_dev(void); TEST_DECL_GROUP("mldsa", test_mldsa_PrivateKeyDecode_OpenSSL_form), \ TEST_DECL_GROUP("mldsa", test_mldsa_pkcs8_import_OpenSSL_form), \ TEST_DECL_GROUP("mldsa", test_mldsa_pkcs8_export_import_wolfSSL_form), \ + TEST_DECL_GROUP("mldsa", test_mldsa_vec_encode_exact_len), \ TEST_DECL_GROUP("mldsa", test_mldsa_encode_w1_large_values), \ TEST_DECL_GROUP("mldsa", test_mldsa_pkcs12), \ TEST_DECL_GROUP("mldsa", test_mldsa_x509_pubkey_sigtype), \ diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index d80f3ebae57..16704cc8e88 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -47,6 +47,8 @@ * Compiles in only the verification and public key operations. * WOLFSSL_MLDSA_VERIFY_SMALL_MEM Default: OFF * Compiles verification implementation that uses smaller amounts of memory. + * Holds c as the positions and signs of its non-zero coefficients and + * rebuilds NTT(c) for each row, unless WC_MLDSA_CACHE_PUB_VECTORS is set. * WOLFSSL_MLDSA_VERIFY_SMALLEST_MEM Default: OFF * Implies WOLFSSL_MLDSA_VERIFY_SMALL_MEM. Decodes and transforms vector z a * polynomial at a time instead of holding the whole vector, k times rather @@ -272,9 +274,8 @@ #endif #if !defined(WOLFSSL_MLDSA_NO_SIGN) && \ (defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC) || \ - defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A) || \ - defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM)) - #error "WC_MLDSA_NONBLOCK sign needs plain SIGN_SMALL_MEM" + defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A)) + #error "WC_MLDSA_NONBLOCK sign does not support the PRECALC signers" #endif #if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) && \ !defined(WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM) @@ -289,9 +290,8 @@ #error "WC_MLDSA_NONBLOCK keygen unsupported with the keygen PCT" #endif #if !defined(WOLFSSL_MLDSA_NO_VERIFY) && \ - (defined(WOLFSSL_MLDSA_VERIFY_SMALLEST_MEM) || \ - defined(WOLFSSL_MLDSA_VERIFY_NO_MALLOC)) - #error "WC_MLDSA_NONBLOCK verify needs plain VERIFY_SMALL_MEM" + defined(WOLFSSL_MLDSA_VERIFY_NO_MALLOC) + #error "WC_MLDSA_NONBLOCK verify keeps its buffers on the heap" #endif #ifdef WOLFSSL_NO_MALLOC #error "WC_MLDSA_NONBLOCK keeps working vectors on the heap" @@ -550,8 +550,11 @@ static WC_INLINE cpuid_flags_t mldsa_cpuid_flags(void) #endif /* Expanding a, s (make key) and y (sign) all squeeze more blocks. */ -#if defined(MLDSA_NEON_EXPAND_A) || !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ - !defined(WOLFSSL_MLDSA_NO_SIGN) +#if defined(MLDSA_NEON_EXPAND_A) || \ + (!defined(WOLFSSL_MLDSA_NO_MAKE_KEY) && \ + !defined(WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM)) || \ + (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ + !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM)) static void mlkem_sha3_blocksx3(word64* state) { if (IS_AARCH64_SHA3(mldsa_cpuid_flags())) { @@ -576,7 +579,10 @@ static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) #endif /* Only expanding s (make key) and y (sign) use the 64-byte seed. */ -#if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || !defined(WOLFSSL_MLDSA_NO_SIGN) +#if (!defined(WOLFSSL_MLDSA_NO_MAKE_KEY) && \ + !defined(WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM)) || \ + (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ + !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM)) static void mlkem_shake256_blocksx3_seed_64(word64* state, byte* seed) { if (IS_AARCH64_SHA3(mldsa_cpuid_flags())) { @@ -750,6 +756,14 @@ void print_data(const char* name, const byte* d, int len) /* The smallest memory signer expands the mask into a polynomial buffer. */ wc_static_assert(sizeof(sword32) * MLDSA_N >= MLDSA_MAX_V); #endif +#if !defined(WOLFSSL_MLDSA_NO_SIGN) && defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) +/* The small memory signers sample A into a polynomial buffer. */ +wc_static_assert(MLDSA_POLY_SIZE >= MLDSA_REJ_NTT_POLY_H_SIZE); +#endif +#if !defined(WOLFSSL_MLDSA_NO_VERIFY) && defined(WOLFSSL_MLDSA_VERIFY_NO_MALLOC) +/* The no-malloc verify samples c with key->h as its block. */ +wc_static_assert(MLDSA_REJ_NTT_POLY_H_SIZE >= MLDSA_GEN_C_BLOCK_BYTES); +#endif /* 2 blocks, each block 136 bytes = 272 bytes. @@ -1858,7 +1872,7 @@ static int mldsa_vec_encode_eta_bits(const sword32* s, byte d, byte eta, #endif #ifndef WOLFSSL_NO_ML_DSA_65 if (eta == MLDSA_ETA_4) { - wc_mldsa_vec_encode_eta_4_avx2(s, p); + wc_mldsa_vec_encode_eta_4_avx2(s, d, p); } #endif RESTORE_VECTOR_REGISTERS(); @@ -1870,6 +1884,12 @@ static int mldsa_vec_encode_eta_bits(const sword32* s, byte d, byte eta, } return ret; } + +WOLFSSL_TEST_VIS int wc_mldsa_vec_encode_eta(const sword32* s, byte d, + byte eta, byte* p) +{ + return mldsa_vec_encode_eta_bits(s, d, eta, p); +} #endif /* !WOLFSSL_MLDSA_NO_MAKE_KEY */ #if !defined(WOLFSSL_MLDSA_NO_SIGN) || defined(WOLFSSL_MLDSA_CHECK_KEY) || \ @@ -2340,6 +2360,12 @@ static int mldsa_vec_encode_t0_t1(const sword32* t, byte d, byte* t0, } return ret; } + +WOLFSSL_TEST_VIS int wc_mldsa_vec_encode_t0_t1(const sword32* t, byte d, + byte* t0, byte* t1) +{ + return mldsa_vec_encode_t0_t1(t, d, t0, t1); +} #endif /* !WOLFSSL_MLDSA_NO_MAKE_KEY */ #if !defined(WOLFSSL_MLDSA_NO_SIGN) || defined(WOLFSSL_MLDSA_CHECK_KEY) @@ -3514,7 +3540,9 @@ WOLFSSL_TEST_VIS int wc_mldsa_encode_w1_32(const sword32* w1, byte* w1e) #endif #endif -#if !defined(WOLFSSL_MLDSA_NO_SIGN) && defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) +#if !defined(WOLFSSL_MLDSA_NO_SIGN) && \ + defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) && \ + !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM) /* Decode one polynomial of w1 from its encoded form. * * The small memory signing implementations keep w1 only in the encoded form @@ -5347,6 +5375,7 @@ static int mldsa_rej_bound_poly(wc_Shake* shake256, byte* seed, sword32* s, #endif } +#ifndef WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifndef WOLFSSL_NO_ML_DSA_44 /* Deterministically generate a matrix (or transpose) of uniform integers mod q. @@ -6198,6 +6227,7 @@ static int wc_mldsa_gen_s_7_8_arm(sword32* s[2], byte* seed) } #endif #endif +#endif /* !WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM */ /* Expand private seed into vectors s1 and s2. * @@ -6266,6 +6296,7 @@ static int mldsa_expand_s_c(wc_Shake* shake256, byte* priv_seed, byte eta, return ret; } +#ifndef WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM /* Expand private seed into vectors s1 and s2. * * @param [in, out] shake256 SHAKE-256 object. @@ -6381,6 +6412,7 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, return ret; } +#endif /* !WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM */ #endif /* !WOLFSSL_MLDSA_NO_MAKE_KEY */ #ifndef WOLFSSL_MLDSA_NO_SIGN @@ -6734,7 +6766,7 @@ static int wc_mldsa_gen_y_avx512(sword32* y, byte* seed, word16 kappa, return 0; } #endif /* WOLFSSL_MLDSA_HAVE_INTEL_AVX512 */ -#elif defined(MLDSA_ASM_BLOCKSX3) +#elif defined(MLDSA_ASM_BLOCKSX3) && !defined(WOLFSSL_MLDSA_SIGN_SMALLEST_MEM) /* Decode polynomial with range -(GAMMA1-1)..GAMMA1 inside a gen_y lane. * * The lane's caller already holds the vector registers, so this calls the @@ -11541,7 +11573,7 @@ static void mldsa_shake_wipe(wc_Shake* shake) /* Calculate row r of t = NTT-1(A o NTT(s1)) + s2 and make it positive. * * Matrix A is expanded a polynomial at a time and the row of s2 is decoded - * from its encoding, so only single polynomial scratch is needed. + * from its encoding into the same scratch polynomial once A is done with. * * @param [in, out] shake SHAKE-128 object to expand A with. * @param [in, out] aseed Public seed followed by space for column and row. @@ -11551,8 +11583,8 @@ static void mldsa_shake_wipe(wc_Shake* shake) * @param [in] s2p Encoding of row r of s2. * @param [in] eta Range of s2 coefficients. * @param [out] tt Row r of t, made positive. - * @param [out] s2t Scratch polynomial for row r of s2. - * @param [out] a Scratch polynomial for an element of A. + * @param [out] a Scratch polynomial for an element of A and the row + * of s2. * @param [out] h Rejection sampling scratch. * @param [out] t64 Accumulator polynomial. Only used with * WOLFSSL_MLDSA_SMALL_MEM_POLY64. @@ -11561,7 +11593,7 @@ static void mldsa_shake_wipe(wc_Shake* shake) */ static int mldsa_calc_t_row(wc_Shake* shake, byte* aseed, byte r, const sword32* s1, byte l, const byte* s2p, byte eta, sword32* tt, - sword32* s2t, sword32* a, byte* h, sword64* t64) + sword32* a, byte* h, sword64* t64) { int ret = 0; const sword32* s1t = s1; @@ -11686,10 +11718,10 @@ static int mldsa_calc_t_row(wc_Shake* shake, byte* aseed, byte r, } /* Decode the row of s2 as the vector is not kept by the callers. */ if (ret == 0) { - ret = mldsa_vec_decode_eta_bits(s2p, eta, s2t, 1); + ret = mldsa_vec_decode_eta_bits(s2p, eta, a, 1); } if (ret == 0) { - ret = mldsa_add(tt, s2t); + ret = mldsa_add(tt, a); } /* Make positive for decomposing. */ if (ret == 0) { @@ -11951,7 +11983,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) const wc_MlDsaParams* params = key->params; sword32* a = NULL; sword32* s1 = NULL; - sword32* s2 = NULL; sword32* t = NULL; sword64* t64 = NULL; byte* h = NULL; @@ -11980,12 +12011,10 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) /* Allocate memory for large intermediates. */ if (ret == 0) { - /* s1-l, s2-k, a-1, [t64], h. t is encoded a polynomial at a time, - * so it and the one decoded s2 polynomial share the s2 vector, which - * is dead once s2 has been encoded into the private key. - * h is last as its size may be odd. */ - allocSz = (unsigned int)params->s1Sz + params->s2Sz + - (unsigned int)MLDSA_POLY_SIZE + + /* s1-l, t-1, a-1, [t64], h. s2 and t are encoded a polynomial at a + * time. h is last as its size may be odd. */ + allocSz = (unsigned int)params->s1Sz + + 2U * (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 /* t64 */ @@ -12013,9 +12042,8 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) ret = MEMORY_E; } else { - s2 = s1 + params->s1Sz / sizeof(*s1); - t = s2; - a = s2 + params->s2Sz / sizeof(*s2); + t = s1 + params->s1Sz / sizeof(*s1); + a = t + MLDSA_N; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 t64 = (sword64*)(a + MLDSA_N); h = (byte*)(t64 + MLDSA_N); @@ -12066,13 +12094,40 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) case 1: if (ret == 0) { byte* priv_seed = key->k + MLDSA_PUB_SEED_SZ; + byte* s2p = priv_seed + MLDSA_K_SZ + MLDSA_TR_SZ + params->s1EncSz; + word32 s2Stride = (word32)params->s2EncSz / params->k; + byte sseed[MLDSA_GEN_S_SEED_SZ]; /* Step 7; Alg 22 Step 1: Copy public seed into public key. */ XMEMCPY(key->p, pub_seed, MLDSA_PUB_SEED_SZ); - /* Step 4: Expand private seed into to vectors of polynomials. */ - ret = mldsa_expand_s(&key->shake, priv_seed, params->eta, s1, - params->l, s2, params->k, key->heap); + /* Step 4: Expand private seed into vector s1. */ + ret = mldsa_expand_s_c(&key->shake, priv_seed, params->eta, s1, + params->l, NULL, 0); + /* Step 9. Alg 24 Steps 2-4: Encode s1 into private key. */ + if (ret == 0) { + ret = mldsa_vec_encode_eta_bits(s1, params->l, params->eta, + s2p - params->s1EncSz); + } + /* Step 4, Step 9. Alg 24 Steps 5-7: Expand and encode s2 a polynomial + * at a time. The encoding lies past rho' and K, still in use. */ + XMEMCPY(sseed, priv_seed, MLDSA_PRIV_SEED_SZ); + #ifdef WOLFSSL_CHECK_MEM_ZERO + wc_MemZero_Add("mldsa make_key sseed", sseed, sizeof(sseed)); + #endif + sseed[MLDSA_PRIV_SEED_SZ + 1] = 0; + for (r = 0; (ret == 0) && (r < params->k); r++) { + sseed[MLDSA_PRIV_SEED_SZ] = (byte)(r + params->l); + ret = mldsa_rej_bound_poly(&key->shake, sseed, a, params->eta); + if (ret == 0) { + ret = mldsa_vec_encode_eta_bits(a, 1, params->eta, + s2p + (size_t)r * s2Stride); + } + } + ForceZero(sseed, sizeof(sseed)); + #ifdef WOLFSSL_CHECK_MEM_ZERO + wc_MemZero_Check(sseed, sizeof(sseed)); + #endif } #ifdef WC_MLDSA_NONBLOCK if ((key->nb != NULL) && (ret == 0)) { @@ -12085,8 +12140,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) case 2: if (ret == 0) { byte* k = pub_seed + MLDSA_PUB_SEED_SZ; - byte* s1p = k + MLDSA_K_SZ + MLDSA_TR_SZ; - byte* s2p = s1p + params->s1EncSz; byte kBuf[MLDSA_K_SZ]; #ifdef WOLFSSL_CHECK_MEM_ZERO @@ -12097,14 +12150,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) ForceZero(k, MLDSA_PRIV_SEED_SZ + MLDSA_K_SZ); XMEMCPY(k, kBuf, MLDSA_K_SZ); ForceZero(kBuf, sizeof(kBuf)); - /* Step 9. Alg 24 Steps 2-4: Encode s1 into private key. */ - if (ret == 0) { - ret = mldsa_vec_encode_eta_bits(s1, params->l, params->eta, s1p); - } - /* Step 9. Alg 24 Steps 5-7: Encode s2 into private key. */ - if (ret == 0) { - ret = mldsa_vec_encode_eta_bits(s2, params->k, params->eta, s2p); - } /* Step 5: NTT(s1) */ if (ret == 0) { ret = mldsa_vec_ntt_small_full(s1, params->l); @@ -12134,8 +12179,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) word32 s2Stride = (word32)params->s2EncSz / params->k; unsigned int r0 = 0; byte aseed[MLDSA_GEN_A_SEED_SZ]; - /* One decoded s2 polynomial, held after t in the dead s2 vector. */ - sword32* s2t = t + MLDSA_N; sword32* tt = t; const byte* s2pt; byte* t0; @@ -12153,7 +12196,7 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) XMEMCPY(aseed, pub_seed, MLDSA_PUB_SEED_SZ); for (r = r0; (ret == 0) && (r < params->k); r++) { ret = mldsa_calc_t_row(&key->shake, aseed, (byte)r, s1, - params->l, s2pt, params->eta, tt, s2t, a, h, t64); + params->l, s2pt, params->eta, tt, a, h, t64); /* Decompose t into t0 and t1 and encode into the keys, one poly. */ if (ret == 0) { ret = mldsa_vec_encode_t0_t1(tt, 1, t0, t1); @@ -12204,12 +12247,6 @@ static int mldsa_make_key_from_seed(wc_MlDsaKey* key, const byte* seed) break; } - /* Zeroize the whole buffer before freeing. It holds the private vectors - * s1, s2 and t at the front, then the public matrix polynomial a, then - * (POLY64 builds) the t64 accumulator holding A o NTT(s1) - from which - * s1 is recoverable - and finally the public rejection-sampling buffer - * h. As the secret material is not contiguous, zeroize the entire - * allocation rather than a sub-range. */ #ifdef WC_MLDSA_NONBLOCK if ((key->nb != NULL) && (ret == WC_NO_ERR_TRACE(MP_WOULDBLOCK))) { /* Yielding: keep the working allocation for the next call. */ @@ -12890,6 +12927,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, sword32* z = NULL; sword32* ct0 = NULL; byte* blocks = NULL; + byte* cBlocks = NULL; byte* w1e = NULL; byte priv_rand_seed[MLDSA_Y_SEED_SZ]; byte* h = sig + params->lambda / 4 + params->zEncSz; @@ -12928,22 +12966,22 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Allocate memory for large intermediates. */ if (ret == 0) { - /* y-l, w0-k, w-1, c-1, z-1, A-1, w1e, blocks. - * Walking A a column at a time makes w0 the vector of accumulators, - * w1 is only kept encoded, and w is the scratch for the hints. */ + /* y-l, w0-k, c-1, z-1, A-1, w1e. w0 accumulates a column at a time; + * w shares A, and the sampling blocks share c then A. */ allocSz = (unsigned int)params->s1Sz + params->s2Sz + (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_POLY_SIZE + - (unsigned int)MLDSA_POLY_SIZE + - (unsigned int)params->w1EncSz + - (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE; + (unsigned int)params->w1EncSz; #ifdef WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC - allocSz += (unsigned int)params->s1Sz + params->s2Sz + params->s2Sz; + allocSz += (unsigned int)MLDSA_POLY_SIZE + + (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE + + (unsigned int)params->s1Sz + params->s2Sz + params->s2Sz; #elif defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A) /* w is decomposed and encoded as a full k vector regardless of how * many rows of A are pre-calculated, so it must hold k polynomials. */ - allocSz += (unsigned int)(params->k - 1) * + allocSz += (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE + + (unsigned int)params->k * (unsigned int)MLDSA_POLY_SIZE + (unsigned int)maxK * params->l * (unsigned int)MLDSA_POLY_SIZE; @@ -12972,8 +13010,8 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, y_check = y; #endif w0 = y + params->s1Sz / sizeof(*y_ntt); - w = w0 + params->s2Sz / sizeof(*w0); #if defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A) + w = w0 + params->s2Sz / sizeof(*w0); /* Only maxK rows of w are multiplied into, but the ML-DSA-44 * decompose kernels take no dimension and touch all k. One * zeroing covers every rejection round: decompose of zero is @@ -12983,8 +13021,11 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, (size_t)(params->k - maxK) * MLDSA_POLY_SIZE); } c = w + (unsigned int)params->k * MLDSA_N; - #else + #elif defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC) + w = w0 + params->s2Sz / sizeof(*w0); c = w + MLDSA_N; + #else + c = w0 + params->s2Sz / sizeof(*w0); #endif z = c + MLDSA_N; a = z + MLDSA_N; @@ -12996,6 +13037,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, t0 = z; w1e = (byte*)(a + (1 + maxK * params->l) * MLDSA_N); blocks = w1e + params->w1EncSz; + cBlocks = blocks; #elif defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC) y_ntt = z; s1 = a + MLDSA_N; @@ -13003,18 +13045,21 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, t0 = s2 + params->s2Sz / sizeof(*s2); w1e = (byte*)(t0 + params->s2Sz / sizeof(*t0)); blocks = w1e + params->w1EncSz; + cBlocks = blocks; #else y_ntt = z; s1 = z; s2 = z; t0 = z; + w = a; w1e = (byte*)(a + MLDSA_N); - blocks = w1e + params->w1EncSz; + blocks = (byte*)c; + cBlocks = (byte*)a; #endif } } - if ((ret == 0) + if (ret == 0 #ifdef WC_MLDSA_NONBLOCK /* Resuming part way through an attempt: its y is still in the working * allocation, so the seed it is expanded from is not needed. */ @@ -13338,7 +13383,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Step 17: Compute c from first 256 bits of commit. */ ret = mldsa_sample_in_ball_ex(params->level, &key->shake, commit, params->lambda / 4, params->tau, c, - blocks); + cBlocks); } if (ret == 0) { /* Step 18: NTT(c). */ @@ -13558,10 +13603,14 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, } if (!valid) { - /* Wipe the rejected candidate; A and cached keys stay. */ + /* Wipe the rejected candidate; cached A and keys stay. */ + #if defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A) || \ + defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC) ForceZero(y, (size_t)((byte*)a - (byte*)y)); - ForceZero(w1e, (size_t)params->w1EncSz + - MLDSA_REJ_NTT_POLY_H_SIZE); + ForceZero(w1e, (size_t)((byte*)y + allocSz - w1e)); + #else + ForceZero(y, allocSz); + #endif ForceZero(sig, params->sigSz); /* Too many attempts - something wrong with implementation. */ if ((kappa > (word16)(kappa + params->l))) { @@ -13647,7 +13696,6 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, sword32* a = NULL; sword32* c = NULL; sword32* z = NULL; - byte* w1e = NULL; byte* blocks = NULL; byte priv_rand_seed[MLDSA_Y_SEED_SZ]; byte* h = sig + params->lambda / 4 + params->zEncSz; @@ -13657,21 +13705,31 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Bytes of encoded s1 or s2 per polynomial. */ unsigned int sStride = (unsigned int)params->s2EncSz / params->k; /* Checksum of each polynomial of y, to bind the two derivations. */ +#ifdef WC_MLDSA_NONBLOCK + word64* yChk = NULL; + int resume = 0; +#else word64 yChk[MLDSA_MAX_L_VECTOR_COUNT / MLDSA_N]; +#endif #ifdef WC_MLDSA_FAULT_HARDEN sword32* w_check; #endif - /* priv_rand_seed will hold the secret signing seed (rho'') derived below - * and yChk a checksum over each polynomial of the secret mask y; - * baseline-zero and register both up front (single-exit function) so any - * later exit before the ForceZero is covered. */ + /* rho'' and, in blocking builds, the yChk checksums of y are on the stack; + * register them up front so every exit before the ForceZero is covered. */ #ifdef WOLFSSL_CHECK_MEM_ZERO XMEMSET(priv_rand_seed, 0, sizeof(priv_rand_seed)); wc_MemZero_Add("mldsa sign priv_rand_seed", priv_rand_seed, sizeof(priv_rand_seed)); +#ifndef WC_MLDSA_NONBLOCK XMEMSET(yChk, 0, sizeof(yChk)); wc_MemZero_Add("mldsa sign yChk", yChk, sizeof(yChk)); +#endif +#endif +#ifdef WC_MLDSA_NONBLOCK + /* Resume only the sign of this rnd||mu into this buffer. */ + (void)mldsa_nb_begin(key, MLDSA_NB_OP_SIGN, seedMu, 0, + MLDSA_RND_SZ + MLDSA_MU_SZ, sig); #endif /* Check the signature buffer isn't too small. */ if (*sigLen < params->sigSz) { @@ -13684,14 +13742,30 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Allocate memory for large intermediates. */ if (ret == 0) { - /* w-k, y-1, a-1, c-1, z-1, w1e, blocks. - * w0 replaces w in place and w1 is only kept encoded, so w is the - * only vector held. */ + /* w-k, y-1, a-1, c-1, [yChk]. z shares a, the sampling blocks + * share c then y, and w1 is hashed a polynomial at a time. */ allocSz = (unsigned int)params->s2Sz + - 4U * (unsigned int)MLDSA_POLY_SIZE + - (unsigned int)params->w1EncSz + - (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE; - w = (sword32*)XMALLOC(allocSz, key->heap, DYNAMIC_TYPE_MLDSA); + 3U * (unsigned int)MLDSA_POLY_SIZE; + #ifdef WC_MLDSA_NONBLOCK + /* yChk and the column walk's validity carried across yields. */ + allocSz += ((unsigned int)params->l + 1U) * + (unsigned int)sizeof(word64); + if ((key->nb != NULL) && (key->nb->work != NULL)) { + /* Resume: reuse the working allocation from the previous call. */ + w = (sword32*)key->nb->work; + resume = 1; + } + else + #endif + { + w = (sword32*)XMALLOC(allocSz, key->heap, DYNAMIC_TYPE_MLDSA); + #ifdef WC_MLDSA_NONBLOCK + if ((w != NULL) && (key->nb != NULL)) { + key->nb->work = w; + key->nb->workSz = allocSz; + } + #endif + } if (w == NULL) { ret = MEMORY_E; } @@ -13702,9 +13776,11 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, y = w + params->s2Sz / sizeof(*w); a = y + MLDSA_N; c = a + MLDSA_N; - z = c + MLDSA_N; - w1e = (byte*)(z + MLDSA_N); - blocks = w1e + params->w1EncSz; + z = a; + blocks = (byte*)c; + #ifdef WC_MLDSA_NONBLOCK + yChk = (word64*)(c + MLDSA_N); + #endif } } @@ -13715,29 +13791,40 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, MLDSA_PRIV_RAND_SEED_SZ); } if (ret == 0) { + #ifdef WC_MLDSA_NONBLOCK + word16 kappa = (key->nb != NULL) ? key->nb->kappa : 0; + #else word16 kappa = 0; + #endif int valid; /* Step 11: Start rejection sampling loop */ do { byte aseed[MLDSA_GEN_A_SEED_SZ]; byte* commit = sig; - byte* w1et = w1e; const byte* sp; sword32* wt; unsigned int e; sword32 hi; byte r; - byte s; + byte s = 0; valid = 1; + #ifdef WC_MLDSA_NONBLOCK + /* Resume the column walk where the last yield left it. */ + if (resume) { + s = (byte)key->nb->row; + valid = (int)yChk[params->l]; + resume = 0; + } + #endif /* Copy the seed into a buffer that has space for s and r. */ XMEMCPY(aseed, pub_seed, MLDSA_PUB_SEED_SZ); /* Alg 26. Step 2: Loop over second dimension of matrix. Working * down the columns keeps one polynomial of y and transforms it * once rather than once per row. */ - for (s = 0; (ret == 0) && (s < params->l); s++) { + for (; (ret == 0) && (s < params->l); s++) { #ifdef WC_MLDSA_FAULT_HARDEN if (w_check != w) { valid = 0; @@ -13746,10 +13833,10 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, } #endif /* Step 12: Compute polynomial of y from seed and kappa. */ - /* z is not live yet, so it doubles as the expansion scratch + /* a is not live yet, so it doubles as the expansion scratch * rather than allocating one per polynomial. */ ret = mldsa_expand_mask_poly(&key->shake, priv_rand_seed, - (word16)(kappa + s), params->gamma1_bits, y, (byte*)z); + (word16)(kappa + s), params->gamma1_bits, y, (byte*)a); if (ret != 0) { break; } @@ -13818,10 +13905,28 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, #endif } } + #ifdef WC_MLDSA_NONBLOCK + /* Yield per column; w and yChk persist in the allocation. */ + if ((key->nb != NULL) && (ret == 0) && + (s + 1 < params->l)) { + key->nb->row = (word16)(s + 1); + yChk[params->l] = (word64)valid; + ret = MP_WOULDBLOCK; + } + #endif } + /* Step 15: Hash mu and the encoded w1 a polynomial at a time. An + * attempt the y check rejected needs no commitment. */ + if ((ret == 0) && valid) { + ret = wc_Shake256_Reset(&key->shake); + } + if ((ret == 0) && valid) { + ret = wc_Shake256_Update(&key->shake, mu, MLDSA_MU_SZ); + } wt = w; - for (r = 0; (ret == 0) && (r < params->k); r++) { + for (r = valid ? 0 : params->k; (ret == 0) && (r < params->k); + r++) { /* Step 13: w = NTT-1(A o NTT(y)) */ ret = mldsa_poly_red(wt); #ifdef WOLFSSL_MLDSA_CHECK_INVNTT_BOUND @@ -13833,7 +13938,8 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, if (ret == 0) { ret = mldsa_invntt_full(wt); } - /* Step 14, Step 22: Make values positive and decompose. */ + /* Step 14, Step 22: Make values positive and decompose. w is + * kept whole and decomposed again for the hints. */ if (ret == 0) { ret = mldsa_make_pos(wt); } @@ -13841,12 +13947,12 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, if (params->gamma2 == MLDSA_Q_LOW_88) { /* For each value of polynomial. */ for (e = 0; e < MLDSA_N; e++) { - /* w0 replaces w, w1 goes to the scratch polynomial. */ - mldsa_decompose_q88(wt[e], &wt[e], &a[e]); + /* w0 goes to a, w1 to y. */ + mldsa_decompose_q88(wt[e], &a[e], &y[e]); } /* Step 15: Encode this polynomial of w1. */ if (ret == 0) { - ret = mldsa_encode_w1_88(a, w1et); + ret = mldsa_encode_w1_88(y, blocks); } } #endif @@ -13854,35 +13960,36 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, if (params->gamma2 == MLDSA_Q_LOW_32) { /* For each value of polynomial. */ for (e = 0; e < MLDSA_N; e++) { - /* w0 replaces w, w1 goes to the scratch polynomial. */ - mldsa_decompose_q32(wt[e], &wt[e], &a[e]); + /* w0 goes to a, w1 to y. */ + mldsa_decompose_q32(wt[e], &a[e], &y[e]); } /* Step 15: Encode this polynomial of w1. */ if (ret == 0) { - ret = mldsa_encode_w1_32(a, w1et); + ret = mldsa_encode_w1_32(y, blocks); } } #endif #ifdef WOLFSSL_MLDSA_SIGN_CHECK_W0 /* Accumulate: w0 comes from the secret mask, so an early exit * would leak which row rejected. */ - valid &= mldsa_check_low(wt, + valid &= mldsa_check_low(a, params->gamma2 - params->beta); #endif + if (ret == 0) { + ret = wc_Shake256_Update(&key->shake, blocks, w1Stride); + } wt += MLDSA_N; - w1et += w1Stride; } if ((ret == 0) && valid) { - /* Step 15: Hash mu and encoded w1. - * Step 32: Hash is stored in signature. */ - ret = mldsa_hash256(&key->shake, mu, MLDSA_MU_SZ, w1e, - params->w1EncSz, commit, params->lambda / 4); + /* Step 32: Hash is stored in signature. */ + ret = wc_Shake256_Final(&key->shake, commit, + params->lambda / 4); } if ((ret == 0) && valid) { /* Step 17: Compute c from first 256 bits of commit. */ ret = mldsa_sample_in_ball_ex(params->level, &key->shake, - commit, params->lambda / 4, params->tau, c, blocks); + commit, params->lambda / 4, params->tau, c, (byte*)y); } if ((ret == 0) && valid) { /* Step 18: NTT(c). */ @@ -13898,10 +14005,8 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, * discarded round; y masks which one it was. */ for (s = 0; (ret == 0) && valid && (s < params->l); s++) { /* Vector y is not kept, so regenerate this polynomial. */ - /* z is overwritten by the multiply below, so it doubles - * as the expansion scratch. */ ret = mldsa_expand_mask_poly(&key->shake, priv_rand_seed, - (word16)(kappa + s), params->gamma1_bits, y, (byte*)z); + (word16)(kappa + s), params->gamma1_bits, y, (byte*)a); if (ret != 0) { break; } @@ -13915,7 +14020,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, if (ret == 0) { ret = mldsa_ntt_small(a); } - /* Step 19: cs1 = NTT-1(c o s1) */ + /* Step 19: cs1 = NTT-1(c o s1), in place as z shares a. */ if (ret == 0) { ret = mldsa_mul(z, c, a); } @@ -13960,8 +14065,23 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, sp = s2p; wt = w; - w1et = w1e; for (r = 0; (ret == 0) && valid && (r < params->k); r++) { + /* Decompose w again: w0 in place, w1 into y. */ + #ifndef WOLFSSL_NO_ML_DSA_44 + if (params->gamma2 == MLDSA_Q_LOW_88) { + for (e = 0; e < MLDSA_N; e++) { + mldsa_decompose_q88(wt[e], &wt[e], &y[e]); + } + } + #endif + #if !defined(WOLFSSL_NO_ML_DSA_65) || \ + !defined(WOLFSSL_NO_ML_DSA_87) + if (params->gamma2 == MLDSA_Q_LOW_32) { + for (e = 0; e < MLDSA_N; e++) { + mldsa_decompose_q32(wt[e], &wt[e], &y[e]); + } + } + #endif ret = mldsa_vec_decode_eta_bits(sp, params->eta, a, 1); if (ret == 0) { ret = mldsa_ntt_small(a); @@ -14006,10 +14126,6 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, ret = mldsa_poly_red(z); } - /* w1 is only kept encoded, so recover the polynomial - * the hint needs. */ - mldsa_decode_w1(w1et, params->gamma2, a); - /* Step 26, 27: Make hint from ct0 and w1 and check * number of hints is valid. * Step 32: h is encoded into signature. @@ -14017,7 +14133,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, #ifndef WOLFSSL_NO_ML_DSA_44 if ((ret == 0) && (params->gamma2 == MLDSA_Q_LOW_88)) { - ret = mldsa_make_hint_88(z, a, h, &idx, &valid); + ret = mldsa_make_hint_88(z, y, h, &idx, &valid); /* Alg 14, Step 10: Store count of hints for * polynomial at end of list. */ if (ret == 0) { @@ -14029,7 +14145,7 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, !defined(WOLFSSL_NO_ML_DSA_87) if ((ret == 0) && (params->gamma2 == MLDSA_Q_LOW_32)) { - ret = mldsa_make_hint_32(z, a, params->omega, + ret = mldsa_make_hint_32(z, y, params->omega, h, &idx, &valid); /* Alg 14, Step 10: Store count of hints for * polynomial at end of list. */ @@ -14043,7 +14159,6 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, sp += sStride; t0pt += MLDSA_D * MLDSA_N / 8; wt += MLDSA_N; - w1et += w1Stride; } valid &= ct0Valid; /* Set remaining hints to zero. Only for a valid attempt: a @@ -14054,7 +14169,10 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, } } - if (!valid) { + if ((ret == 0) && (!valid)) { + /* Wipe the rejected candidate; yChk is rebuilt per column. */ + ForceZero(w, allocSz); + ForceZero(sig, params->sigSz); /* Too many attempts - something wrong with implementation. */ if ((kappa > (word16)(kappa + params->l))) { ret = BAD_COND_E; @@ -14063,34 +14181,59 @@ static int mldsa_sign_with_seed_mu(wc_MlDsaKey* key, /* Step 30: increment value to append to seed to unique value. */ kappa = (word16)(kappa + params->l); + #ifdef WC_MLDSA_NONBLOCK + /* Rejected attempt: restart the column walk and yield. */ + if ((key->nb != NULL) && (ret == 0)) { + key->nb->kappa = kappa; + key->nb->row = 0; + yChk[params->l] = 1; + ret = MP_WOULDBLOCK; + } + #endif } } /* Step 11: Check we have a valid signature. */ while ((ret == 0) && (!valid)); } - if ((ret != 0) && (*sigLen == params->sigSz)) { + if ((ret != 0) && +#ifdef WC_MLDSA_NONBLOCK + (ret != WC_NO_ERR_TRACE(MP_WOULDBLOCK)) && +#endif + (*sigLen == params->sigSz)) { /* FIPS 204 3.6.3: a failed sign leaves no part of a signature. */ ForceZero(sig, params->sigSz); } ForceZero(priv_rand_seed, sizeof(priv_rand_seed)); - /* The last expansion of y leaves rho'' recoverable from key->shake. - * Re-initializing clears the state but keeps what its free needs. */ - if (wc_InitShake256(&key->shake, NULL, INVALID_DEVID) != 0) { - ForceZero(&key->shake, sizeof(key->shake)); - } + /* The last expansion of y leaves rho'' recoverable from key->shake. */ + mldsa_shake_wipe(&key->shake); #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(priv_rand_seed, sizeof(priv_rand_seed)); #endif +#ifndef WC_MLDSA_NONBLOCK /* Checksums are derived from the secret mask y. */ ForceZero(yChk, sizeof(yChk)); #ifdef WOLFSSL_CHECK_MEM_ZERO wc_MemZero_Check(yChk, sizeof(yChk)); #endif - if (w != NULL) { - ForceZero(w, allocSz); +#endif +#ifdef WC_MLDSA_NONBLOCK + if ((key->nb != NULL) && (ret == WC_NO_ERR_TRACE(MP_WOULDBLOCK))) { + /* Yielding: keep the working allocation for the next call. */ + } + else if (key->nb != NULL) { + /* Final result: the context owns the allocation, even when this call + * failed before picking it up. */ + mldsa_nb_release(key); + } + else +#endif + { + if (w != NULL) { + ForceZero(w, allocSz); + } + XFREE(w, key->heap, DYNAMIC_TYPE_MLDSA); } - XFREE(w, key->heap, DYNAMIC_TYPE_MLDSA); return ret; #endif } @@ -14658,6 +14801,44 @@ static int mldsa_make_pub_vec(wc_MlDsaKey* key, sword32* t1) } #endif +#ifdef MLDSA_VERIFY_SPARSE_C +/* Record the positions and signs of the tau non-zero coefficients of c. + * + * @param [in] c Polynomial with tau coefficients of 1 or -1. + * @param [in] tau Number of non-zero coefficients. + * @param [out] cSparse Positions, then sign flags (1 for -1), tau bytes each. + */ +static void mldsa_make_sparse(const sword32* c, byte tau, byte* cSparse) +{ + unsigned int e; + unsigned int j = 0; + + for (e = 0; (e < MLDSA_N) && (j < tau); e++) { + if (c[e] != 0) { + cSparse[j] = (byte)e; + cSparse[tau + j] = (byte)(c[e] < 0); + j++; + } + } +} + +/* Rebuild c from its positions and signs. + * + * @param [in] cSparse Positions, then sign flags (1 for -1), tau bytes each. + * @param [in] tau Number of non-zero coefficients. + * @param [out] c Polynomial with tau coefficients of 1 or -1. + */ +static void mldsa_expand_sparse(const byte* cSparse, byte tau, sword32* c) +{ + unsigned int j; + + XMEMSET(c, 0, MLDSA_POLY_SIZE); + for (j = 0; j < tau; j++) { + c[cSparse[j]] = 1 - 2 * (sword32)cSparse[tau + j]; + } +} +#endif /* MLDSA_VERIFY_SPARSE_C */ + /* Verify signature of message using public key. * * FIPS 204 Section 5.3, Algorithm 3 ML-DSA.Verify(pk, M, sigma, ctx) @@ -14939,6 +15120,9 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, const sword32* a = NULL; #ifdef WOLFSSL_MLDSA_VERIFY_PRECOMP_A const sword32* aPre = mldsa_precomp_a(key); +#endif +#ifdef MLDSA_VERIFY_SPARSE_C + byte* cSparse = NULL; #endif sword32* c = NULL; sword32* z = NULL; @@ -14946,9 +15130,7 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 sword64* t64 = NULL; #endif -#ifndef WOLFSSL_MLDSA_VERIFY_NO_MALLOC byte* block = NULL; -#endif byte* w1e = NULL; byte commit_calc[MLDSA_TR_SZ]; int valid = 0; @@ -15001,13 +15183,18 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, #ifndef WOLFSSL_MLDSA_VERIFY_NO_MALLOC /* Allocate memory for large intermediates. */ if (ret == 0) { - /* z, c-1, w-1, t1-1, [t64], block, w1e */ + /* z, [c-1], w-1, t1-1, [t64], block, w1e, [cSparse] */ unsigned int allocSz; allocSz = zSz + - 3U * (unsigned int)MLDSA_POLY_SIZE + + 2U * (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE + params->w1EncSz; + #ifdef MLDSA_VERIFY_SPARSE_C + allocSz += 2U * (unsigned int)params->tau; + #else + allocSz += (unsigned int)MLDSA_POLY_SIZE; + #endif #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 allocSz += (unsigned int)MLDSA_POLY_SIZE * 2U; #endif @@ -15039,8 +15226,12 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, { XMEMSET(z, 0, allocSz); } + #ifdef MLDSA_VERIFY_SPARSE_C + w = z + zSz / sizeof(*t1); + #else c = z + zSz / sizeof(*t1); w = c + MLDSA_N; + #endif t1 = w + MLDSA_N; aBuf = t1; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 @@ -15050,16 +15241,25 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, block = (byte*)(t1 + MLDSA_N); #endif w1e = block + MLDSA_REJ_NTT_POLY_H_SIZE; + #ifdef MLDSA_VERIFY_SPARSE_C + cSparse = w1e + params->w1EncSz; + #endif } } #else if (ret == 0) { z = key->z; + #ifdef MLDSA_VERIFY_SPARSE_C + cSparse = key->cSparse; + #else c = key->c; + #endif w = key->w; t1 = key->vt1; w1e = key->w1e; aBuf = t1; + /* The sampling block is done with before A needs key->h. */ + block = key->h; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 t64 = key->t64; #endif @@ -15118,9 +15318,12 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, if ((ret == 0) && valid) { /* Step 9: Compute c from first 256 bits of commit. */ -#ifdef WOLFSSL_MLDSA_VERIFY_NO_MALLOC +#ifdef MLDSA_VERIFY_SPARSE_C ret = mldsa_sample_in_ball_ex(params->level, &key->shake, commit, - params->lambda / 4, params->tau, c, key->block); + params->lambda / 4, params->tau, w, block); + if (ret == 0) { + mldsa_make_sparse(w, params->tau, cSparse); + } #else ret = mldsa_sample_in_ball_ex(params->level, &key->shake, commit, params->lambda / 4, params->tau, c, block); @@ -15137,16 +15340,20 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, #endif FALL_THROUGH; case 1: +#ifndef MLDSA_VERIFY_SPARSE_C if ((ret == 0) && valid) { ret = mldsa_ntt_small_full(c); } +#endif #ifdef WC_MLDSA_NONBLOCK if ((key->nb != NULL) && (ret == 0) && valid) { key->nb->state = 2; key->nb->row = 0; key->nb->hintPos = 0; + #ifndef MLDSA_VERIFY_SPARSE_C ret = MP_WOULDBLOCK; break; + #endif } #endif FALL_THROUGH; @@ -15177,6 +15384,14 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, * result goes to w either way, so a cached polynomial is read in * place rather than copied. */ const sword32* t1v = w; +#ifdef MLDSA_VERIFY_SPARSE_C + /* Rebuild NTT(c) in the scratch of A, which is free until below. */ + c = aBuf; + if (ret == 0) { + mldsa_expand_sparse(cSparse, params->tau, c); + ret = mldsa_ntt_small_full(c); + } +#endif #ifdef WC_MLDSA_CACHE_PUB_VECTORS if (key->pubVecSet) { @@ -15998,7 +16213,6 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) #ifndef WOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM sword32* s2 = NULL; #else - sword32* s2t = NULL; sword64* t64 = NULL; byte* h = NULL; #endif @@ -16058,10 +16272,9 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) /* Matrix A is expanded a polynomial at a time, so any cache is left * alone. */ if (ret == 0) { - /* s1-l, t-1, s2-1, a-1, [t64], h. h is last as its size may be - * odd. */ + /* s1-l, t-1, a-1, [t64], h. h is last as its size may be odd. */ allocSz = (unsigned int)params->s1Sz + - 3U * (unsigned int)MLDSA_POLY_SIZE + + 2U * (unsigned int)MLDSA_POLY_SIZE + (unsigned int)MLDSA_REJ_NTT_POLY_H_SIZE; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 /* t64 */ @@ -16074,8 +16287,7 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) } else { t = s1 + params->s1Sz / sizeof(*s1); - s2t = t + MLDSA_N; - a = s2t + MLDSA_N; + a = t + MLDSA_N; #ifdef WOLFSSL_MLDSA_SMALL_MEM_POLY64 t64 = (sword64*)(a + MLDSA_N); h = (byte*)(t64 + MLDSA_N); @@ -16157,8 +16369,8 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) for (r = 0; (ret == 0) && (r < params->k); r++) { /* Calculate row r of t = NTT-1(A o NTT(s1)) + s2 */ ret = mldsa_calc_t_row(&key->shake, aseed, (byte)r, s1, - params->l, s2p + (r * s2Stride), params->eta, t, s2t, a, - h, t64); + params->l, s2p + (r * s2Stride), params->eta, t, a, h, + t64); /* Encode row r of t1 into the public key and check row r * of the derived t0 against the private key's t0. */ if (ret == 0) { @@ -16194,8 +16406,7 @@ static int mldsa_make_pub_key(wc_MlDsaKey* key) /* s1, s2 and t at the front are secret; any A after is public. */ ForceZero(s1, (unsigned int)params->s1Sz + (2U * params->s2Sz)); #else - /* The public a and h sit between and after the secret s1, t, s2 and - * t64, so zeroize the entire allocation. */ + /* a also holds the secret s2 rows, so zeroize the entire allocation. */ ForceZero(s1, allocSz); #endif XFREE(s1, key->heap, DYNAMIC_TYPE_MLDSA); diff --git a/wolfcrypt/src/wc_mldsa_asm.S b/wolfcrypt/src/wc_mldsa_asm.S index 89e71b21a54..ad0ea56f15d 100644 --- a/wolfcrypt/src/wc_mldsa_asm.S +++ b/wolfcrypt/src/wc_mldsa_asm.S @@ -23685,887 +23685,68 @@ L_mldsa_encode_eta_2_avx2_loop: vpor %ymm4, %ymm0, %ymm0 vextracti128 $0x01, %ymm0, %xmm4 vpor %ymm4, %ymm0, %ymm0 - vmovdqu %xmm0, (%rdx) - addq $12, %rdx - addq $0x400, %rdi - decb %sil - jnz L_mldsa_encode_eta_2_avx2_loop - vzeroupper - repz retq -#ifndef __APPLE__ -.size wc_mldsa_vec_encode_eta_2_avx2,.-wc_mldsa_vec_encode_eta_2_avx2 -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.data -#else -.section __DATA,__data -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.align 16 -#else -.p2align 4 -#endif /* __APPLE__ */ -L_mldsa_encode_eta_4_avx2_four: -.long 0x00000004,0x00000004,0x00000004,0x00000004 -.long 0x00000004,0x00000004,0x00000004,0x00000004 -#ifndef __APPLE__ -.data -#else -.section __DATA,__data -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.align 16 -#else -.p2align 4 -#endif /* __APPLE__ */ -L_mldsa_encode_eta_4_avx2_vs_4: -.long 0x00000000,0x00000004,0x00000000,0x00000004 -.long 0x00000000,0x00000004,0x00000000,0x00000004 -#ifndef __APPLE__ -.text -.globl wc_mldsa_vec_encode_eta_4_avx2 -WC_ASM_ATT_HIDDEN(wc_mldsa_vec_encode_eta_4_avx2) -.type wc_mldsa_vec_encode_eta_4_avx2,@function -.align 16 -wc_mldsa_vec_encode_eta_4_avx2: -#else -.section __TEXT,__text -.globl _wc_mldsa_vec_encode_eta_4_avx2 -WC_ASM_ATT_HIDDEN(_wc_mldsa_vec_encode_eta_4_avx2) -.p2align 4 -_wc_mldsa_vec_encode_eta_4_avx2: -#endif /* __APPLE__ */ - vpxor %ymm8, %ymm8, %ymm8 - vmovdqu L_mldsa_encode_eta_4_avx2_four(%rip), %ymm8 - vmovdqu L_mldsa_encode_eta_4_avx2_vs_4(%rip), %ymm9 - vmovdqu (%rdi), %ymm0 - vmovdqu 32(%rdi), %ymm1 - vmovdqu 64(%rdi), %ymm2 - vmovdqu 96(%rdi), %ymm3 - vmovdqu 128(%rdi), %ymm4 - vmovdqu 160(%rdi), %ymm5 - vmovdqu 192(%rdi), %ymm6 - vmovdqu 224(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, (%rsi) - vmovdqu 256(%rdi), %ymm0 - vmovdqu 288(%rdi), %ymm1 - vmovdqu 320(%rdi), %ymm2 - vmovdqu 352(%rdi), %ymm3 - vmovdqu 384(%rdi), %ymm4 - vmovdqu 416(%rdi), %ymm5 - vmovdqu 448(%rdi), %ymm6 - vmovdqu 480(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 32(%rsi) - vmovdqu 512(%rdi), %ymm0 - vmovdqu 544(%rdi), %ymm1 - vmovdqu 576(%rdi), %ymm2 - vmovdqu 608(%rdi), %ymm3 - vmovdqu 640(%rdi), %ymm4 - vmovdqu 672(%rdi), %ymm5 - vmovdqu 704(%rdi), %ymm6 - vmovdqu 736(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 64(%rsi) - vmovdqu 768(%rdi), %ymm0 - vmovdqu 800(%rdi), %ymm1 - vmovdqu 832(%rdi), %ymm2 - vmovdqu 864(%rdi), %ymm3 - vmovdqu 896(%rdi), %ymm4 - vmovdqu 928(%rdi), %ymm5 - vmovdqu 960(%rdi), %ymm6 - vmovdqu 992(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 96(%rsi) - vmovdqu 1024(%rdi), %ymm0 - vmovdqu 1056(%rdi), %ymm1 - vmovdqu 1088(%rdi), %ymm2 - vmovdqu 1120(%rdi), %ymm3 - vmovdqu 1152(%rdi), %ymm4 - vmovdqu 1184(%rdi), %ymm5 - vmovdqu 1216(%rdi), %ymm6 - vmovdqu 1248(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 128(%rsi) - vmovdqu 1280(%rdi), %ymm0 - vmovdqu 1312(%rdi), %ymm1 - vmovdqu 1344(%rdi), %ymm2 - vmovdqu 1376(%rdi), %ymm3 - vmovdqu 1408(%rdi), %ymm4 - vmovdqu 1440(%rdi), %ymm5 - vmovdqu 1472(%rdi), %ymm6 - vmovdqu 1504(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 160(%rsi) - vmovdqu 1536(%rdi), %ymm0 - vmovdqu 1568(%rdi), %ymm1 - vmovdqu 1600(%rdi), %ymm2 - vmovdqu 1632(%rdi), %ymm3 - vmovdqu 1664(%rdi), %ymm4 - vmovdqu 1696(%rdi), %ymm5 - vmovdqu 1728(%rdi), %ymm6 - vmovdqu 1760(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 192(%rsi) - vmovdqu 1792(%rdi), %ymm0 - vmovdqu 1824(%rdi), %ymm1 - vmovdqu 1856(%rdi), %ymm2 - vmovdqu 1888(%rdi), %ymm3 - vmovdqu 1920(%rdi), %ymm4 - vmovdqu 1952(%rdi), %ymm5 - vmovdqu 1984(%rdi), %ymm6 - vmovdqu 2016(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 224(%rsi) - vmovdqu 2048(%rdi), %ymm0 - vmovdqu 2080(%rdi), %ymm1 - vmovdqu 2112(%rdi), %ymm2 - vmovdqu 2144(%rdi), %ymm3 - vmovdqu 2176(%rdi), %ymm4 - vmovdqu 2208(%rdi), %ymm5 - vmovdqu 2240(%rdi), %ymm6 - vmovdqu 2272(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 256(%rsi) - vmovdqu 2304(%rdi), %ymm0 - vmovdqu 2336(%rdi), %ymm1 - vmovdqu 2368(%rdi), %ymm2 - vmovdqu 2400(%rdi), %ymm3 - vmovdqu 2432(%rdi), %ymm4 - vmovdqu 2464(%rdi), %ymm5 - vmovdqu 2496(%rdi), %ymm6 - vmovdqu 2528(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 288(%rsi) - vmovdqu 2560(%rdi), %ymm0 - vmovdqu 2592(%rdi), %ymm1 - vmovdqu 2624(%rdi), %ymm2 - vmovdqu 2656(%rdi), %ymm3 - vmovdqu 2688(%rdi), %ymm4 - vmovdqu 2720(%rdi), %ymm5 - vmovdqu 2752(%rdi), %ymm6 - vmovdqu 2784(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 320(%rsi) - vmovdqu 2816(%rdi), %ymm0 - vmovdqu 2848(%rdi), %ymm1 - vmovdqu 2880(%rdi), %ymm2 - vmovdqu 2912(%rdi), %ymm3 - vmovdqu 2944(%rdi), %ymm4 - vmovdqu 2976(%rdi), %ymm5 - vmovdqu 3008(%rdi), %ymm6 - vmovdqu 3040(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 352(%rsi) - vmovdqu 3072(%rdi), %ymm0 - vmovdqu 3104(%rdi), %ymm1 - vmovdqu 3136(%rdi), %ymm2 - vmovdqu 3168(%rdi), %ymm3 - vmovdqu 3200(%rdi), %ymm4 - vmovdqu 3232(%rdi), %ymm5 - vmovdqu 3264(%rdi), %ymm6 - vmovdqu 3296(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 384(%rsi) - vmovdqu 3328(%rdi), %ymm0 - vmovdqu 3360(%rdi), %ymm1 - vmovdqu 3392(%rdi), %ymm2 - vmovdqu 3424(%rdi), %ymm3 - vmovdqu 3456(%rdi), %ymm4 - vmovdqu 3488(%rdi), %ymm5 - vmovdqu 3520(%rdi), %ymm6 - vmovdqu 3552(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 416(%rsi) - vmovdqu 3584(%rdi), %ymm0 - vmovdqu 3616(%rdi), %ymm1 - vmovdqu 3648(%rdi), %ymm2 - vmovdqu 3680(%rdi), %ymm3 - vmovdqu 3712(%rdi), %ymm4 - vmovdqu 3744(%rdi), %ymm5 - vmovdqu 3776(%rdi), %ymm6 - vmovdqu 3808(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 448(%rsi) - vmovdqu 3840(%rdi), %ymm0 - vmovdqu 3872(%rdi), %ymm1 - vmovdqu 3904(%rdi), %ymm2 - vmovdqu 3936(%rdi), %ymm3 - vmovdqu 3968(%rdi), %ymm4 - vmovdqu 4000(%rdi), %ymm5 - vmovdqu 4032(%rdi), %ymm6 - vmovdqu 4064(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 480(%rsi) - vmovdqu 4096(%rdi), %ymm0 - vmovdqu 4128(%rdi), %ymm1 - vmovdqu 4160(%rdi), %ymm2 - vmovdqu 4192(%rdi), %ymm3 - vmovdqu 4224(%rdi), %ymm4 - vmovdqu 4256(%rdi), %ymm5 - vmovdqu 4288(%rdi), %ymm6 - vmovdqu 4320(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 512(%rsi) - vmovdqu 4352(%rdi), %ymm0 - vmovdqu 4384(%rdi), %ymm1 - vmovdqu 4416(%rdi), %ymm2 - vmovdqu 4448(%rdi), %ymm3 - vmovdqu 4480(%rdi), %ymm4 - vmovdqu 4512(%rdi), %ymm5 - vmovdqu 4544(%rdi), %ymm6 - vmovdqu 4576(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 544(%rsi) - vmovdqu 4608(%rdi), %ymm0 - vmovdqu 4640(%rdi), %ymm1 - vmovdqu 4672(%rdi), %ymm2 - vmovdqu 4704(%rdi), %ymm3 - vmovdqu 4736(%rdi), %ymm4 - vmovdqu 4768(%rdi), %ymm5 - vmovdqu 4800(%rdi), %ymm6 - vmovdqu 4832(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 576(%rsi) - vmovdqu 4864(%rdi), %ymm0 - vmovdqu 4896(%rdi), %ymm1 - vmovdqu 4928(%rdi), %ymm2 - vmovdqu 4960(%rdi), %ymm3 - vmovdqu 4992(%rdi), %ymm4 - vmovdqu 5024(%rdi), %ymm5 - vmovdqu 5056(%rdi), %ymm6 - vmovdqu 5088(%rdi), %ymm7 - vpsubd %ymm0, %ymm8, %ymm0 - vpsubd %ymm1, %ymm8, %ymm1 - vpsubd %ymm2, %ymm8, %ymm2 - vpsubd %ymm3, %ymm8, %ymm3 - vpsubd %ymm4, %ymm8, %ymm4 - vpsubd %ymm5, %ymm8, %ymm5 - vpsubd %ymm6, %ymm8, %ymm6 - vpsubd %ymm7, %ymm8, %ymm7 - vpsllvd %ymm9, %ymm0, %ymm0 - vpsllvd %ymm9, %ymm1, %ymm1 - vpsllvd %ymm9, %ymm2, %ymm2 - vpsllvd %ymm9, %ymm3, %ymm3 - vpsllvd %ymm9, %ymm4, %ymm4 - vpsllvd %ymm9, %ymm5, %ymm5 - vpsllvd %ymm9, %ymm6, %ymm6 - vpsllvd %ymm9, %ymm7, %ymm7 - vpackusdw %ymm1, %ymm0, %ymm0 - vpackusdw %ymm3, %ymm2, %ymm1 - vpackusdw %ymm5, %ymm4, %ymm2 - vpackusdw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpermq $0xd8, %ymm2, %ymm2 - vpermq $0xd8, %ymm3, %ymm3 - vphaddw %ymm1, %ymm0, %ymm0 - vphaddw %ymm3, %ymm2, %ymm1 - vphaddw %ymm5, %ymm4, %ymm2 - vphaddw %ymm7, %ymm6, %ymm3 - vpermq $0xd8, %ymm0, %ymm0 - vpermq $0xd8, %ymm1, %ymm1 - vpackuswb %ymm1, %ymm0, %ymm0 - vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 608(%rsi) - vmovdqu 5120(%rdi), %ymm0 - vmovdqu 5152(%rdi), %ymm1 - vmovdqu 5184(%rdi), %ymm2 - vmovdqu 5216(%rdi), %ymm3 - vmovdqu 5248(%rdi), %ymm4 - vmovdqu 5280(%rdi), %ymm5 - vmovdqu 5312(%rdi), %ymm6 - vmovdqu 5344(%rdi), %ymm7 + vmovq %xmm0, (%rdx) + vpextrd $2, %xmm0, 8(%rdx) + addq $12, %rdx + addq $0x400, %rdi + decb %sil + jnz L_mldsa_encode_eta_2_avx2_loop + vzeroupper + repz retq +#ifndef __APPLE__ +.size wc_mldsa_vec_encode_eta_2_avx2,.-wc_mldsa_vec_encode_eta_2_avx2 +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.data +#else +.section __DATA,__data +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.align 16 +#else +.p2align 4 +#endif /* __APPLE__ */ +L_mldsa_encode_eta_4_avx2_four: +.long 0x00000004,0x00000004,0x00000004,0x00000004 +.long 0x00000004,0x00000004,0x00000004,0x00000004 +#ifndef __APPLE__ +.data +#else +.section __DATA,__data +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.align 16 +#else +.p2align 4 +#endif /* __APPLE__ */ +L_mldsa_encode_eta_4_avx2_vs_4: +.long 0x00000000,0x00000004,0x00000000,0x00000004 +.long 0x00000000,0x00000004,0x00000000,0x00000004 +#ifndef __APPLE__ +.text +.globl wc_mldsa_vec_encode_eta_4_avx2 +WC_ASM_ATT_HIDDEN(wc_mldsa_vec_encode_eta_4_avx2) +.type wc_mldsa_vec_encode_eta_4_avx2,@function +.align 16 +wc_mldsa_vec_encode_eta_4_avx2: +#else +.section __TEXT,__text +.globl _wc_mldsa_vec_encode_eta_4_avx2 +WC_ASM_ATT_HIDDEN(_wc_mldsa_vec_encode_eta_4_avx2) +.p2align 4 +_wc_mldsa_vec_encode_eta_4_avx2: +#endif /* __APPLE__ */ + vmovdqu L_mldsa_encode_eta_4_avx2_four(%rip), %ymm8 + vmovdqu L_mldsa_encode_eta_4_avx2_vs_4(%rip), %ymm9 +L_mldsa_encode_eta_4_avx2_loop: + vmovdqu (%rdi), %ymm0 + vmovdqu 32(%rdi), %ymm1 + vmovdqu 64(%rdi), %ymm2 + vmovdqu 96(%rdi), %ymm3 + vmovdqu 128(%rdi), %ymm4 + vmovdqu 160(%rdi), %ymm5 + vmovdqu 192(%rdi), %ymm6 + vmovdqu 224(%rdi), %ymm7 vpsubd %ymm0, %ymm8, %ymm0 vpsubd %ymm1, %ymm8, %ymm1 vpsubd %ymm2, %ymm8, %ymm2 @@ -24598,15 +23779,15 @@ _wc_mldsa_vec_encode_eta_4_avx2: vpermq $0xd8, %ymm1, %ymm1 vpackuswb %ymm1, %ymm0, %ymm0 vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 640(%rsi) - vmovdqu 5376(%rdi), %ymm0 - vmovdqu 5408(%rdi), %ymm1 - vmovdqu 5440(%rdi), %ymm2 - vmovdqu 5472(%rdi), %ymm3 - vmovdqu 5504(%rdi), %ymm4 - vmovdqu 5536(%rdi), %ymm5 - vmovdqu 5568(%rdi), %ymm6 - vmovdqu 5600(%rdi), %ymm7 + vmovdqu %ymm0, (%rdx) + vmovdqu 256(%rdi), %ymm0 + vmovdqu 288(%rdi), %ymm1 + vmovdqu 320(%rdi), %ymm2 + vmovdqu 352(%rdi), %ymm3 + vmovdqu 384(%rdi), %ymm4 + vmovdqu 416(%rdi), %ymm5 + vmovdqu 448(%rdi), %ymm6 + vmovdqu 480(%rdi), %ymm7 vpsubd %ymm0, %ymm8, %ymm0 vpsubd %ymm1, %ymm8, %ymm1 vpsubd %ymm2, %ymm8, %ymm2 @@ -24639,15 +23820,15 @@ _wc_mldsa_vec_encode_eta_4_avx2: vpermq $0xd8, %ymm1, %ymm1 vpackuswb %ymm1, %ymm0, %ymm0 vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 672(%rsi) - vmovdqu 5632(%rdi), %ymm0 - vmovdqu 5664(%rdi), %ymm1 - vmovdqu 5696(%rdi), %ymm2 - vmovdqu 5728(%rdi), %ymm3 - vmovdqu 5760(%rdi), %ymm4 - vmovdqu 5792(%rdi), %ymm5 - vmovdqu 5824(%rdi), %ymm6 - vmovdqu 5856(%rdi), %ymm7 + vmovdqu %ymm0, 32(%rdx) + vmovdqu 512(%rdi), %ymm0 + vmovdqu 544(%rdi), %ymm1 + vmovdqu 576(%rdi), %ymm2 + vmovdqu 608(%rdi), %ymm3 + vmovdqu 640(%rdi), %ymm4 + vmovdqu 672(%rdi), %ymm5 + vmovdqu 704(%rdi), %ymm6 + vmovdqu 736(%rdi), %ymm7 vpsubd %ymm0, %ymm8, %ymm0 vpsubd %ymm1, %ymm8, %ymm1 vpsubd %ymm2, %ymm8, %ymm2 @@ -24680,15 +23861,15 @@ _wc_mldsa_vec_encode_eta_4_avx2: vpermq $0xd8, %ymm1, %ymm1 vpackuswb %ymm1, %ymm0, %ymm0 vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 704(%rsi) - vmovdqu 5888(%rdi), %ymm0 - vmovdqu 5920(%rdi), %ymm1 - vmovdqu 5952(%rdi), %ymm2 - vmovdqu 5984(%rdi), %ymm3 - vmovdqu 6016(%rdi), %ymm4 - vmovdqu 6048(%rdi), %ymm5 - vmovdqu 6080(%rdi), %ymm6 - vmovdqu 6112(%rdi), %ymm7 + vmovdqu %ymm0, 64(%rdx) + vmovdqu 768(%rdi), %ymm0 + vmovdqu 800(%rdi), %ymm1 + vmovdqu 832(%rdi), %ymm2 + vmovdqu 864(%rdi), %ymm3 + vmovdqu 896(%rdi), %ymm4 + vmovdqu 928(%rdi), %ymm5 + vmovdqu 960(%rdi), %ymm6 + vmovdqu 992(%rdi), %ymm7 vpsubd %ymm0, %ymm8, %ymm0 vpsubd %ymm1, %ymm8, %ymm1 vpsubd %ymm2, %ymm8, %ymm2 @@ -24721,7 +23902,11 @@ _wc_mldsa_vec_encode_eta_4_avx2: vpermq $0xd8, %ymm1, %ymm1 vpackuswb %ymm1, %ymm0, %ymm0 vpermq $0xd8, %ymm0, %ymm0 - vmovdqu %ymm0, 736(%rsi) + vmovdqu %ymm0, 96(%rdx) + addq $0x400, %rdi + addq $0x80, %rdx + decb %sil + jnz L_mldsa_encode_eta_4_avx2_loop vzeroupper repz retq #ifndef __APPLE__ @@ -26808,7 +25993,9 @@ L_mldsa_encode_t0_t1_avx2_loop: vpor %ymm1, %ymm3, %ymm3 vmovdqu %xmm2, (%rdx) addq $13, %rdx - vmovdqu %xmm3, (%rdx) + vmovq %xmm3, (%rdx) + vpextrd $2, %xmm3, 8(%rdx) + vpextrb $12, %xmm3, 12(%rdx) addq $13, %rdx vpsllvd %ymm11, %ymm4, %ymm4 vpsllvd %ymm11, %ymm5, %ymm5 @@ -26824,7 +26011,8 @@ L_mldsa_encode_t0_t1_avx2_loop: vpor %ymm1, %ymm5, %ymm5 vmovdqu %xmm4, (%rcx) addq $10, %rcx - vmovdqu %xmm5, (%rcx) + vmovq %xmm5, (%rcx) + vpextrw $4, %xmm5, 8(%rcx) addq $10, %rcx addq $0x400, %rdi decb %sil diff --git a/wolfcrypt/src/wc_mldsa_asm.asm b/wolfcrypt/src/wc_mldsa_asm.asm index 9649525a62c..2198aec09f3 100644 --- a/wolfcrypt/src/wc_mldsa_asm.asm +++ b/wolfcrypt/src/wc_mldsa_asm.asm @@ -23567,872 +23567,53 @@ L_mldsa_encode_eta_2_avx2_loop: vpor ymm0, ymm0, ymm4 vextracti128 xmm4, ymm0, 1 vpor ymm0, ymm0, ymm4 - vmovdqu OWORD PTR [r8], xmm0 - add r8, 12 - add rcx, 1024 - dec dl - jnz L_mldsa_encode_eta_2_avx2_loop - vzeroupper - vmovdqu xmm6, OWORD PTR [rsp+8] - vmovdqu xmm7, OWORD PTR [rsp+24] - vmovdqu xmm8, OWORD PTR [rsp+40] - vmovdqu xmm9, OWORD PTR [rsp+56] - vmovdqu xmm10, OWORD PTR [rsp+72] - vmovdqu xmm11, OWORD PTR [rsp+88] - add rsp, 104 - ret -wc_mldsa_vec_encode_eta_2_avx2 ENDP -_TEXT ENDS -_DATA SEGMENT -ALIGN 16 -L_mldsa_encode_eta_4_avx2_four DWORD 00000004h, 00000004h, 00000004h, 00000004h - DWORD 00000004h, 00000004h, 00000004h, 00000004h -ptr_L_mldsa_encode_eta_4_avx2_four QWORD L_mldsa_encode_eta_4_avx2_four -_DATA ENDS -_DATA SEGMENT -ALIGN 16 -L_mldsa_encode_eta_4_avx2_vs_4 DWORD 00000000h, 00000004h, 00000000h, 00000004h - DWORD 00000000h, 00000004h, 00000000h, 00000004h -ptr_L_mldsa_encode_eta_4_avx2_vs_4 QWORD L_mldsa_encode_eta_4_avx2_vs_4 -_DATA ENDS -_TEXT SEGMENT READONLY PARA -wc_mldsa_vec_encode_eta_4_avx2 PROC - sub rsp, 72 - vmovdqu OWORD PTR [rsp+8], xmm6 - vmovdqu OWORD PTR [rsp+24], xmm7 - vmovdqu OWORD PTR [rsp+40], xmm8 - vmovdqu OWORD PTR [rsp+56], xmm9 - vpxor ymm8, ymm8, ymm8 - vmovdqu ymm8, YMMWORD PTR L_mldsa_encode_eta_4_avx2_four - vmovdqu ymm9, YMMWORD PTR L_mldsa_encode_eta_4_avx2_vs_4 - vmovdqu ymm0, YMMWORD PTR [rcx] - vmovdqu ymm1, YMMWORD PTR [rcx+32] - vmovdqu ymm2, YMMWORD PTR [rcx+64] - vmovdqu ymm3, YMMWORD PTR [rcx+96] - vmovdqu ymm4, YMMWORD PTR [rcx+128] - vmovdqu ymm5, YMMWORD PTR [rcx+160] - vmovdqu ymm6, YMMWORD PTR [rcx+192] - vmovdqu ymm7, YMMWORD PTR [rcx+224] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+256] - vmovdqu ymm1, YMMWORD PTR [rcx+288] - vmovdqu ymm2, YMMWORD PTR [rcx+320] - vmovdqu ymm3, YMMWORD PTR [rcx+352] - vmovdqu ymm4, YMMWORD PTR [rcx+384] - vmovdqu ymm5, YMMWORD PTR [rcx+416] - vmovdqu ymm6, YMMWORD PTR [rcx+448] - vmovdqu ymm7, YMMWORD PTR [rcx+480] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+32], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+512] - vmovdqu ymm1, YMMWORD PTR [rcx+544] - vmovdqu ymm2, YMMWORD PTR [rcx+576] - vmovdqu ymm3, YMMWORD PTR [rcx+608] - vmovdqu ymm4, YMMWORD PTR [rcx+640] - vmovdqu ymm5, YMMWORD PTR [rcx+672] - vmovdqu ymm6, YMMWORD PTR [rcx+704] - vmovdqu ymm7, YMMWORD PTR [rcx+736] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+64], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+768] - vmovdqu ymm1, YMMWORD PTR [rcx+800] - vmovdqu ymm2, YMMWORD PTR [rcx+832] - vmovdqu ymm3, YMMWORD PTR [rcx+864] - vmovdqu ymm4, YMMWORD PTR [rcx+896] - vmovdqu ymm5, YMMWORD PTR [rcx+928] - vmovdqu ymm6, YMMWORD PTR [rcx+960] - vmovdqu ymm7, YMMWORD PTR [rcx+992] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+96], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+1024] - vmovdqu ymm1, YMMWORD PTR [rcx+1056] - vmovdqu ymm2, YMMWORD PTR [rcx+1088] - vmovdqu ymm3, YMMWORD PTR [rcx+1120] - vmovdqu ymm4, YMMWORD PTR [rcx+1152] - vmovdqu ymm5, YMMWORD PTR [rcx+1184] - vmovdqu ymm6, YMMWORD PTR [rcx+1216] - vmovdqu ymm7, YMMWORD PTR [rcx+1248] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+128], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+1280] - vmovdqu ymm1, YMMWORD PTR [rcx+1312] - vmovdqu ymm2, YMMWORD PTR [rcx+1344] - vmovdqu ymm3, YMMWORD PTR [rcx+1376] - vmovdqu ymm4, YMMWORD PTR [rcx+1408] - vmovdqu ymm5, YMMWORD PTR [rcx+1440] - vmovdqu ymm6, YMMWORD PTR [rcx+1472] - vmovdqu ymm7, YMMWORD PTR [rcx+1504] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+160], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+1536] - vmovdqu ymm1, YMMWORD PTR [rcx+1568] - vmovdqu ymm2, YMMWORD PTR [rcx+1600] - vmovdqu ymm3, YMMWORD PTR [rcx+1632] - vmovdqu ymm4, YMMWORD PTR [rcx+1664] - vmovdqu ymm5, YMMWORD PTR [rcx+1696] - vmovdqu ymm6, YMMWORD PTR [rcx+1728] - vmovdqu ymm7, YMMWORD PTR [rcx+1760] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+192], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+1792] - vmovdqu ymm1, YMMWORD PTR [rcx+1824] - vmovdqu ymm2, YMMWORD PTR [rcx+1856] - vmovdqu ymm3, YMMWORD PTR [rcx+1888] - vmovdqu ymm4, YMMWORD PTR [rcx+1920] - vmovdqu ymm5, YMMWORD PTR [rcx+1952] - vmovdqu ymm6, YMMWORD PTR [rcx+1984] - vmovdqu ymm7, YMMWORD PTR [rcx+2016] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+224], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+2048] - vmovdqu ymm1, YMMWORD PTR [rcx+2080] - vmovdqu ymm2, YMMWORD PTR [rcx+2112] - vmovdqu ymm3, YMMWORD PTR [rcx+2144] - vmovdqu ymm4, YMMWORD PTR [rcx+2176] - vmovdqu ymm5, YMMWORD PTR [rcx+2208] - vmovdqu ymm6, YMMWORD PTR [rcx+2240] - vmovdqu ymm7, YMMWORD PTR [rcx+2272] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+256], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+2304] - vmovdqu ymm1, YMMWORD PTR [rcx+2336] - vmovdqu ymm2, YMMWORD PTR [rcx+2368] - vmovdqu ymm3, YMMWORD PTR [rcx+2400] - vmovdqu ymm4, YMMWORD PTR [rcx+2432] - vmovdqu ymm5, YMMWORD PTR [rcx+2464] - vmovdqu ymm6, YMMWORD PTR [rcx+2496] - vmovdqu ymm7, YMMWORD PTR [rcx+2528] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+288], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+2560] - vmovdqu ymm1, YMMWORD PTR [rcx+2592] - vmovdqu ymm2, YMMWORD PTR [rcx+2624] - vmovdqu ymm3, YMMWORD PTR [rcx+2656] - vmovdqu ymm4, YMMWORD PTR [rcx+2688] - vmovdqu ymm5, YMMWORD PTR [rcx+2720] - vmovdqu ymm6, YMMWORD PTR [rcx+2752] - vmovdqu ymm7, YMMWORD PTR [rcx+2784] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+320], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+2816] - vmovdqu ymm1, YMMWORD PTR [rcx+2848] - vmovdqu ymm2, YMMWORD PTR [rcx+2880] - vmovdqu ymm3, YMMWORD PTR [rcx+2912] - vmovdqu ymm4, YMMWORD PTR [rcx+2944] - vmovdqu ymm5, YMMWORD PTR [rcx+2976] - vmovdqu ymm6, YMMWORD PTR [rcx+3008] - vmovdqu ymm7, YMMWORD PTR [rcx+3040] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+352], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+3072] - vmovdqu ymm1, YMMWORD PTR [rcx+3104] - vmovdqu ymm2, YMMWORD PTR [rcx+3136] - vmovdqu ymm3, YMMWORD PTR [rcx+3168] - vmovdqu ymm4, YMMWORD PTR [rcx+3200] - vmovdqu ymm5, YMMWORD PTR [rcx+3232] - vmovdqu ymm6, YMMWORD PTR [rcx+3264] - vmovdqu ymm7, YMMWORD PTR [rcx+3296] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+384], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+3328] - vmovdqu ymm1, YMMWORD PTR [rcx+3360] - vmovdqu ymm2, YMMWORD PTR [rcx+3392] - vmovdqu ymm3, YMMWORD PTR [rcx+3424] - vmovdqu ymm4, YMMWORD PTR [rcx+3456] - vmovdqu ymm5, YMMWORD PTR [rcx+3488] - vmovdqu ymm6, YMMWORD PTR [rcx+3520] - vmovdqu ymm7, YMMWORD PTR [rcx+3552] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+416], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+3584] - vmovdqu ymm1, YMMWORD PTR [rcx+3616] - vmovdqu ymm2, YMMWORD PTR [rcx+3648] - vmovdqu ymm3, YMMWORD PTR [rcx+3680] - vmovdqu ymm4, YMMWORD PTR [rcx+3712] - vmovdqu ymm5, YMMWORD PTR [rcx+3744] - vmovdqu ymm6, YMMWORD PTR [rcx+3776] - vmovdqu ymm7, YMMWORD PTR [rcx+3808] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+448], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+3840] - vmovdqu ymm1, YMMWORD PTR [rcx+3872] - vmovdqu ymm2, YMMWORD PTR [rcx+3904] - vmovdqu ymm3, YMMWORD PTR [rcx+3936] - vmovdqu ymm4, YMMWORD PTR [rcx+3968] - vmovdqu ymm5, YMMWORD PTR [rcx+4000] - vmovdqu ymm6, YMMWORD PTR [rcx+4032] - vmovdqu ymm7, YMMWORD PTR [rcx+4064] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+480], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+4096] - vmovdqu ymm1, YMMWORD PTR [rcx+4128] - vmovdqu ymm2, YMMWORD PTR [rcx+4160] - vmovdqu ymm3, YMMWORD PTR [rcx+4192] - vmovdqu ymm4, YMMWORD PTR [rcx+4224] - vmovdqu ymm5, YMMWORD PTR [rcx+4256] - vmovdqu ymm6, YMMWORD PTR [rcx+4288] - vmovdqu ymm7, YMMWORD PTR [rcx+4320] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+512], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+4352] - vmovdqu ymm1, YMMWORD PTR [rcx+4384] - vmovdqu ymm2, YMMWORD PTR [rcx+4416] - vmovdqu ymm3, YMMWORD PTR [rcx+4448] - vmovdqu ymm4, YMMWORD PTR [rcx+4480] - vmovdqu ymm5, YMMWORD PTR [rcx+4512] - vmovdqu ymm6, YMMWORD PTR [rcx+4544] - vmovdqu ymm7, YMMWORD PTR [rcx+4576] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+544], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+4608] - vmovdqu ymm1, YMMWORD PTR [rcx+4640] - vmovdqu ymm2, YMMWORD PTR [rcx+4672] - vmovdqu ymm3, YMMWORD PTR [rcx+4704] - vmovdqu ymm4, YMMWORD PTR [rcx+4736] - vmovdqu ymm5, YMMWORD PTR [rcx+4768] - vmovdqu ymm6, YMMWORD PTR [rcx+4800] - vmovdqu ymm7, YMMWORD PTR [rcx+4832] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+576], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+4864] - vmovdqu ymm1, YMMWORD PTR [rcx+4896] - vmovdqu ymm2, YMMWORD PTR [rcx+4928] - vmovdqu ymm3, YMMWORD PTR [rcx+4960] - vmovdqu ymm4, YMMWORD PTR [rcx+4992] - vmovdqu ymm5, YMMWORD PTR [rcx+5024] - vmovdqu ymm6, YMMWORD PTR [rcx+5056] - vmovdqu ymm7, YMMWORD PTR [rcx+5088] - vpsubd ymm0, ymm8, ymm0 - vpsubd ymm1, ymm8, ymm1 - vpsubd ymm2, ymm8, ymm2 - vpsubd ymm3, ymm8, ymm3 - vpsubd ymm4, ymm8, ymm4 - vpsubd ymm5, ymm8, ymm5 - vpsubd ymm6, ymm8, ymm6 - vpsubd ymm7, ymm8, ymm7 - vpsllvd ymm0, ymm0, ymm9 - vpsllvd ymm1, ymm1, ymm9 - vpsllvd ymm2, ymm2, ymm9 - vpsllvd ymm3, ymm3, ymm9 - vpsllvd ymm4, ymm4, ymm9 - vpsllvd ymm5, ymm5, ymm9 - vpsllvd ymm6, ymm6, ymm9 - vpsllvd ymm7, ymm7, ymm9 - vpackusdw ymm0, ymm0, ymm1 - vpackusdw ymm1, ymm2, ymm3 - vpackusdw ymm2, ymm4, ymm5 - vpackusdw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpermq ymm2, ymm2, 216 - vpermq ymm3, ymm3, 216 - vphaddw ymm0, ymm0, ymm1 - vphaddw ymm1, ymm2, ymm3 - vphaddw ymm2, ymm4, ymm5 - vphaddw ymm3, ymm6, ymm7 - vpermq ymm0, ymm0, 216 - vpermq ymm1, ymm1, 216 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+608], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+5120] - vmovdqu ymm1, YMMWORD PTR [rcx+5152] - vmovdqu ymm2, YMMWORD PTR [rcx+5184] - vmovdqu ymm3, YMMWORD PTR [rcx+5216] - vmovdqu ymm4, YMMWORD PTR [rcx+5248] - vmovdqu ymm5, YMMWORD PTR [rcx+5280] - vmovdqu ymm6, YMMWORD PTR [rcx+5312] - vmovdqu ymm7, YMMWORD PTR [rcx+5344] + vmovq QWORD PTR [r8], xmm0 + vpextrd DWORD PTR [r8+8], xmm0, 2 + add r8, 12 + add rcx, 1024 + dec dl + jnz L_mldsa_encode_eta_2_avx2_loop + vzeroupper + vmovdqu xmm6, OWORD PTR [rsp+8] + vmovdqu xmm7, OWORD PTR [rsp+24] + vmovdqu xmm8, OWORD PTR [rsp+40] + vmovdqu xmm9, OWORD PTR [rsp+56] + vmovdqu xmm10, OWORD PTR [rsp+72] + vmovdqu xmm11, OWORD PTR [rsp+88] + add rsp, 104 + ret +wc_mldsa_vec_encode_eta_2_avx2 ENDP +_TEXT ENDS +_DATA SEGMENT +ALIGN 16 +L_mldsa_encode_eta_4_avx2_four DWORD 00000004h, 00000004h, 00000004h, 00000004h + DWORD 00000004h, 00000004h, 00000004h, 00000004h +ptr_L_mldsa_encode_eta_4_avx2_four QWORD L_mldsa_encode_eta_4_avx2_four +_DATA ENDS +_DATA SEGMENT +ALIGN 16 +L_mldsa_encode_eta_4_avx2_vs_4 DWORD 00000000h, 00000004h, 00000000h, 00000004h + DWORD 00000000h, 00000004h, 00000000h, 00000004h +ptr_L_mldsa_encode_eta_4_avx2_vs_4 QWORD L_mldsa_encode_eta_4_avx2_vs_4 +_DATA ENDS +_TEXT SEGMENT READONLY PARA +wc_mldsa_vec_encode_eta_4_avx2 PROC + sub rsp, 72 + vmovdqu OWORD PTR [rsp+8], xmm6 + vmovdqu OWORD PTR [rsp+24], xmm7 + vmovdqu OWORD PTR [rsp+40], xmm8 + vmovdqu OWORD PTR [rsp+56], xmm9 + vmovdqu ymm8, YMMWORD PTR L_mldsa_encode_eta_4_avx2_four + vmovdqu ymm9, YMMWORD PTR L_mldsa_encode_eta_4_avx2_vs_4 +L_mldsa_encode_eta_4_avx2_loop: + vmovdqu ymm0, YMMWORD PTR [rcx] + vmovdqu ymm1, YMMWORD PTR [rcx+32] + vmovdqu ymm2, YMMWORD PTR [rcx+64] + vmovdqu ymm3, YMMWORD PTR [rcx+96] + vmovdqu ymm4, YMMWORD PTR [rcx+128] + vmovdqu ymm5, YMMWORD PTR [rcx+160] + vmovdqu ymm6, YMMWORD PTR [rcx+192] + vmovdqu ymm7, YMMWORD PTR [rcx+224] vpsubd ymm0, ymm8, ymm0 vpsubd ymm1, ymm8, ymm1 vpsubd ymm2, ymm8, ymm2 @@ -24465,15 +23646,15 @@ wc_mldsa_vec_encode_eta_4_avx2 PROC vpermq ymm1, ymm1, 216 vpackuswb ymm0, ymm0, ymm1 vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+640], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+5376] - vmovdqu ymm1, YMMWORD PTR [rcx+5408] - vmovdqu ymm2, YMMWORD PTR [rcx+5440] - vmovdqu ymm3, YMMWORD PTR [rcx+5472] - vmovdqu ymm4, YMMWORD PTR [rcx+5504] - vmovdqu ymm5, YMMWORD PTR [rcx+5536] - vmovdqu ymm6, YMMWORD PTR [rcx+5568] - vmovdqu ymm7, YMMWORD PTR [rcx+5600] + vmovdqu YMMWORD PTR [r8], ymm0 + vmovdqu ymm0, YMMWORD PTR [rcx+256] + vmovdqu ymm1, YMMWORD PTR [rcx+288] + vmovdqu ymm2, YMMWORD PTR [rcx+320] + vmovdqu ymm3, YMMWORD PTR [rcx+352] + vmovdqu ymm4, YMMWORD PTR [rcx+384] + vmovdqu ymm5, YMMWORD PTR [rcx+416] + vmovdqu ymm6, YMMWORD PTR [rcx+448] + vmovdqu ymm7, YMMWORD PTR [rcx+480] vpsubd ymm0, ymm8, ymm0 vpsubd ymm1, ymm8, ymm1 vpsubd ymm2, ymm8, ymm2 @@ -24506,15 +23687,15 @@ wc_mldsa_vec_encode_eta_4_avx2 PROC vpermq ymm1, ymm1, 216 vpackuswb ymm0, ymm0, ymm1 vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+672], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+5632] - vmovdqu ymm1, YMMWORD PTR [rcx+5664] - vmovdqu ymm2, YMMWORD PTR [rcx+5696] - vmovdqu ymm3, YMMWORD PTR [rcx+5728] - vmovdqu ymm4, YMMWORD PTR [rcx+5760] - vmovdqu ymm5, YMMWORD PTR [rcx+5792] - vmovdqu ymm6, YMMWORD PTR [rcx+5824] - vmovdqu ymm7, YMMWORD PTR [rcx+5856] + vmovdqu YMMWORD PTR [r8+32], ymm0 + vmovdqu ymm0, YMMWORD PTR [rcx+512] + vmovdqu ymm1, YMMWORD PTR [rcx+544] + vmovdqu ymm2, YMMWORD PTR [rcx+576] + vmovdqu ymm3, YMMWORD PTR [rcx+608] + vmovdqu ymm4, YMMWORD PTR [rcx+640] + vmovdqu ymm5, YMMWORD PTR [rcx+672] + vmovdqu ymm6, YMMWORD PTR [rcx+704] + vmovdqu ymm7, YMMWORD PTR [rcx+736] vpsubd ymm0, ymm8, ymm0 vpsubd ymm1, ymm8, ymm1 vpsubd ymm2, ymm8, ymm2 @@ -24547,15 +23728,15 @@ wc_mldsa_vec_encode_eta_4_avx2 PROC vpermq ymm1, ymm1, 216 vpackuswb ymm0, ymm0, ymm1 vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+704], ymm0 - vmovdqu ymm0, YMMWORD PTR [rcx+5888] - vmovdqu ymm1, YMMWORD PTR [rcx+5920] - vmovdqu ymm2, YMMWORD PTR [rcx+5952] - vmovdqu ymm3, YMMWORD PTR [rcx+5984] - vmovdqu ymm4, YMMWORD PTR [rcx+6016] - vmovdqu ymm5, YMMWORD PTR [rcx+6048] - vmovdqu ymm6, YMMWORD PTR [rcx+6080] - vmovdqu ymm7, YMMWORD PTR [rcx+6112] + vmovdqu YMMWORD PTR [r8+64], ymm0 + vmovdqu ymm0, YMMWORD PTR [rcx+768] + vmovdqu ymm1, YMMWORD PTR [rcx+800] + vmovdqu ymm2, YMMWORD PTR [rcx+832] + vmovdqu ymm3, YMMWORD PTR [rcx+864] + vmovdqu ymm4, YMMWORD PTR [rcx+896] + vmovdqu ymm5, YMMWORD PTR [rcx+928] + vmovdqu ymm6, YMMWORD PTR [rcx+960] + vmovdqu ymm7, YMMWORD PTR [rcx+992] vpsubd ymm0, ymm8, ymm0 vpsubd ymm1, ymm8, ymm1 vpsubd ymm2, ymm8, ymm2 @@ -24588,7 +23769,11 @@ wc_mldsa_vec_encode_eta_4_avx2 PROC vpermq ymm1, ymm1, 216 vpackuswb ymm0, ymm0, ymm1 vpermq ymm0, ymm0, 216 - vmovdqu YMMWORD PTR [rdx+736], ymm0 + vmovdqu YMMWORD PTR [r8+96], ymm0 + add rcx, 1024 + add r8, 128 + dec dl + jnz L_mldsa_encode_eta_4_avx2_loop vzeroupper vmovdqu xmm6, OWORD PTR [rsp+8] vmovdqu xmm7, OWORD PTR [rsp+24] @@ -26420,7 +25605,9 @@ L_mldsa_encode_t0_t1_avx2_loop: vpor ymm3, ymm3, ymm1 vmovdqu OWORD PTR [r8], xmm2 add r8, 13 - vmovdqu OWORD PTR [r8], xmm3 + vmovq QWORD PTR [r8], xmm3 + vpextrd DWORD PTR [r8+8], xmm3, 2 + vpextrb BYTE PTR [r8+12], xmm3, 12 add r8, 13 vpsllvd ymm4, ymm4, ymm11 vpsllvd ymm5, ymm5, ymm11 @@ -26436,7 +25623,8 @@ L_mldsa_encode_t0_t1_avx2_loop: vpor ymm5, ymm5, ymm1 vmovdqu OWORD PTR [r9], xmm4 add r9, 10 - vmovdqu OWORD PTR [r9], xmm5 + vmovq QWORD PTR [r9], xmm5 + vpextrw WORD PTR [r9+8], xmm5, 4 add r9, 10 add rcx, 1024 dec dl diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 74d42b0a6d6..96667a25be5 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -78743,20 +78743,116 @@ static wc_test_ret_t mldsa_nonblock_test_level(int level, WC_RNG* rng) return ret; } +/* Seeded key generation and signing must give the same bytes whether run in + * one call or resumed across yields. */ +static wc_test_ret_t mldsa_nonblock_match_test_level(int level) +{ + wc_test_ret_t ret = 0; + int rc = 0; + wc_MlDsaKey* key = NULL; + wc_MlDsaKey* nbKey = NULL; + wc_MlDsaNonBlock nb; + byte* sig = NULL; + byte* nbSig = NULL; + int sigSz = 0; + word32 sigLen; + word32 nbSigLen; + int res; + int i; + int yields = 0; + int minYields = -1; + int maxYields = 0; + int nbInited = 0; + int keyInited = 0; + byte seed[MLDSA_SEED_SZ]; + byte msg[32]; + + XMEMSET(seed, 0x5a, sizeof(seed)); + XMEMSET(msg, 0xa5, sizeof(msg)); + + key = (wc_MlDsaKey*)XMALLOC(sizeof(*key), HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + nbKey = (wc_MlDsaKey*)XMALLOC(sizeof(*nbKey), HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if ((key == NULL) || (nbKey == NULL)) + ret = WC_TEST_RET_ENC_EC(MEMORY_E); + NB_STEP(wc_MlDsaKey_Init(key, HEAP_HINT, devId)); + keyInited = (ret == 0); + NB_STEP(wc_MlDsaKey_SetParams(key, (byte)level)); + NB_STEP(wc_MlDsaKey_Init(nbKey, HEAP_HINT, devId)); + nbInited = (ret == 0); + NB_STEP(wc_MlDsaKey_SetParams(nbKey, (byte)level)); + NB_STEP(wc_MlDsaKey_set_nonblock(nbKey, &nb)); + NB_STEP(wc_MlDsaKey_MakeKeyFromSeed(key, seed)); + NB_RUN(wc_MlDsaKey_MakeKeyFromSeed(nbKey, seed)); + NB_STEP(wc_MlDsaKey_GetSigLen(key, &sigSz)); + if (ret == 0) { + sig = (byte*)XMALLOC((size_t)sigSz, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + nbSig = (byte*)XMALLOC((size_t)sigSz, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if ((sig == NULL) || (nbSig == NULL)) + ret = WC_TEST_RET_ENC_EC(MEMORY_E); + } + for (i = 0; (ret == 0) && (i < 8); i++) { + msg[0] = (byte)i; + sigLen = (word32)sigSz; + nbSigLen = (word32)sigSz; + NB_STEP(wc_MlDsaKey_SignCtxWithSeed(key, NULL, 0, sig, &sigLen, msg, + (word32)sizeof(msg), seed)); + NB_DRIVE(wc_MlDsaKey_SignCtxWithSeed(nbKey, NULL, 0, nbSig, &nbSigLen, + msg, (word32)sizeof(msg), seed)); + if ((ret == 0) && (rc != 0)) + ret = WC_TEST_RET_ENC_EC(rc); + if (ret == 0) { + if ((minYields < 0) || (yields < minYields)) + minYields = yields; + if (yields > maxYields) + maxYields = yields; + } + if ((ret == 0) && ((nbSigLen != sigLen) || + (XMEMCMP(sig, nbSig, sigLen) != 0))) { + ret = WC_TEST_RET_ENC_NC; + } + res = 0; + NB_RUN(wc_MlDsaKey_VerifyCtx(nbKey, nbSig, nbSigLen, NULL, 0, msg, + (word32)sizeof(msg), &res)); + if ((ret == 0) && (res != 1)) + ret = WC_TEST_RET_ENC_NC; + } + if ((ret == 0) && (maxYields <= minYields)) + ret = WC_TEST_RET_ENC_NC; + + XFREE(nbSig, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(sig, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (nbInited) + wc_MlDsaKey_Free(nbKey); + if (keyInited) + wc_MlDsaKey_Free(key); + XFREE(nbKey, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(key, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return ret; +} + static wc_test_ret_t mldsa_nonblock_test(WC_RNG* rng) { wc_test_ret_t ret = 0; #ifndef WOLFSSL_NO_ML_DSA_44 if (ret == 0) ret = mldsa_nonblock_test_level(WC_ML_DSA_44, rng); + if (ret == 0) + ret = mldsa_nonblock_match_test_level(WC_ML_DSA_44); #endif #ifndef WOLFSSL_NO_ML_DSA_65 if (ret == 0) ret = mldsa_nonblock_test_level(WC_ML_DSA_65, rng); + if (ret == 0) + ret = mldsa_nonblock_match_test_level(WC_ML_DSA_65); #endif #ifndef WOLFSSL_NO_ML_DSA_87 if (ret == 0) ret = mldsa_nonblock_test_level(WC_ML_DSA_87, rng); + if (ret == 0) + ret = mldsa_nonblock_match_test_level(WC_ML_DSA_87); #endif return ret; } diff --git a/wolfssl/wolfcrypt/wc_mldsa.h b/wolfssl/wolfcrypt/wc_mldsa.h index 17f9d9135f3..e3a7849752c 100644 --- a/wolfssl/wolfcrypt/wc_mldsa.h +++ b/wolfssl/wolfcrypt/wc_mldsa.h @@ -437,6 +437,7 @@ #define MLDSA_MAX_W1_ENC_SZ PARAMS_ML_DSA_87_W1_ENC_SZ /* Maximum collision strength of c-tilde in bytes. */ #define MLDSA_MAX_LAMBDA PARAMS_ML_DSA_87_LAMBDA +#define MLDSA_MAX_TAU PARAMS_ML_DSA_87_TAU /* Maximum count of elements of a vector with dimension K. */ #define MLDSA_MAX_K_VECTOR_COUNT \ @@ -454,6 +455,7 @@ #define MLDSA_MAX_W1_ENC_SZ PARAMS_ML_DSA_65_W1_ENC_SZ /* Maximum collision strength of c-tilde in bytes. */ #define MLDSA_MAX_LAMBDA PARAMS_ML_DSA_65_LAMBDA +#define MLDSA_MAX_TAU PARAMS_ML_DSA_65_TAU /* Maximum count of elements of a vector with dimension K. */ #define MLDSA_MAX_K_VECTOR_COUNT \ @@ -471,6 +473,7 @@ #define MLDSA_MAX_W1_ENC_SZ PARAMS_ML_DSA_44_W1_ENC_SZ /* Maximum collision strength of c-tilde in bytes. */ #define MLDSA_MAX_LAMBDA PARAMS_ML_DSA_44_LAMBDA +#define MLDSA_MAX_TAU PARAMS_ML_DSA_44_TAU /* Maximum count of elements of a vector with dimension K. */ #define MLDSA_MAX_K_VECTOR_COUNT \ @@ -619,6 +622,12 @@ typedef struct wc_MlDsaParams { typedef struct wc_MlDsaParams MlDsaParams; #endif +/* Small memory verify keeps c sparse unless NTT(t1) is cached. */ +#if defined(WOLFSSL_MLDSA_VERIFY_SMALL_MEM) && \ + !defined(WC_MLDSA_CACHE_PUB_VECTORS) + #define MLDSA_VERIFY_SPARSE_C +#endif + #ifdef WC_MLDSA_NONBLOCK /* Resumable state for a non-blocking ML-DSA operation. Attach one to a key with * wc_MlDsaKey_set_nonblock(); the operation then runs one chunk per call, @@ -747,7 +756,11 @@ struct wc_MlDsaKey { #else sword32 z[MLDSA_MAX_L_VECTOR_COUNT]; #endif +#ifdef MLDSA_VERIFY_SPARSE_C + byte cSparse[2 * MLDSA_MAX_TAU]; /* positions then signs of c */ +#else sword32 c[MLDSA_N]; +#endif sword32 w[MLDSA_N]; /* One t1 polynomial, also used for a polynomial of A. Not named t1 as * WC_MLDSA_CACHE_PUB_VECTORS already has a member of that name. */ @@ -757,7 +770,6 @@ struct wc_MlDsaKey { sword64 t64[MLDSA_N]; #endif byte h[MLDSA_REJ_NTT_POLY_H_SIZE]; - byte block[MLDSA_GEN_C_BLOCK_BYTES]; #endif /* WOLFSSL_MLDSA_VERIFY_NO_MALLOC && * WOLFSSL_MLDSA_VERIFY_SMALL_MEM */ }; @@ -1029,7 +1041,8 @@ WOLFSSL_LOCAL void wc_mldsa_extract_coeffs_eta4_avx2(const byte* z, WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_2_avx2(const sword32* s, byte d, byte* p); -WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_4_avx2(const sword32* t, byte* p); +WOLFSSL_LOCAL void wc_mldsa_vec_encode_eta_4_avx2(const sword32* s, byte d, + byte* p); WOLFSSL_LOCAL void wc_mldsa_decode_eta_2_avx2(const byte* p, sword32* s); WOLFSSL_LOCAL void wc_mldsa_decode_eta_4_avx2(const byte* p, sword32* s); @@ -1326,6 +1339,13 @@ WOLFSSL_TEST_VIS int wc_mldsa_encode_w1_32(const sword32* w1, byte* w1e); #endif #endif +#ifndef WOLFSSL_MLDSA_NO_MAKE_KEY +WOLFSSL_TEST_VIS int wc_mldsa_vec_encode_eta(const sword32* s, byte d, + byte eta, byte* p); +WOLFSSL_TEST_VIS int wc_mldsa_vec_encode_t0_t1(const sword32* t, byte d, + byte* t0, byte* t1); +#endif + #if defined(WOLFSSL_MLDSA_TEST_REJ_NTT_FAIL) && \ !defined(WOLF_CRYPTO_CB_ONLY_MLDSA) WOLFSSL_TEST_VIS int wc_mldsa_test_rej_ntt_fail(int n);