Skip to content

Commit a229d83

Browse files
author
Eric Biggers
committed
lib/crypto: x86/nh: Migrate optimized code into library
Migrate the x86_64 implementations of NH into lib/crypto/. This makes the nh() function be optimized on x86_64 kernels. Note: this temporarily makes the adiantum template not utilize the x86_64 optimized NH code. This is resolved in a later commit that converts the adiantum template to use nh() instead of "nhpoly1305". Link: https://lore.kernel.org/r/20251211011846.8179-6-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
1 parent b4a8528 commit a229d83

9 files changed

Lines changed: 49 additions & 190 deletions

File tree

‎arch/x86/crypto/Kconfig‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -333,26 +333,6 @@ config CRYPTO_AEGIS128_AESNI_SSE2
333333
- AES-NI (AES New Instructions)
334334
- SSE4.1 (Streaming SIMD Extensions 4.1)
335335

336-
config CRYPTO_NHPOLY1305_SSE2
337-
tristate "Hash functions: NHPoly1305 (SSE2)"
338-
depends on 64BIT
339-
select CRYPTO_NHPOLY1305
340-
help
341-
NHPoly1305 hash function for Adiantum
342-
343-
Architecture: x86_64 using:
344-
- SSE2 (Streaming SIMD Extensions 2)
345-
346-
config CRYPTO_NHPOLY1305_AVX2
347-
tristate "Hash functions: NHPoly1305 (AVX2)"
348-
depends on 64BIT
349-
select CRYPTO_NHPOLY1305
350-
help
351-
NHPoly1305 hash function for Adiantum
352-
353-
Architecture: x86_64 using:
354-
- AVX2 (Advanced Vector Extensions 2)
355-
356336
config CRYPTO_SM3_AVX_X86_64
357337
tristate "Hash functions: SM3 (AVX)"
358338
depends on 64BIT

‎arch/x86/crypto/Makefile‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \
5353
obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
5454
ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
5555

56-
obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o
57-
nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o
58-
obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o
59-
nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o
60-
6156
obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
6257
sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
6358

‎arch/x86/crypto/nhpoly1305-avx2-glue.c‎

Lines changed: 0 additions & 81 deletions
This file was deleted.

‎arch/x86/crypto/nhpoly1305-sse2-glue.c‎

Lines changed: 0 additions & 80 deletions
This file was deleted.

‎lib/crypto/Kconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ config CRYPTO_LIB_NH_ARCH
119119
depends on CRYPTO_LIB_NH && !UML
120120
default y if ARM && KERNEL_MODE_NEON
121121
default y if ARM64 && KERNEL_MODE_NEON
122+
default y if X86_64
122123

123124
config CRYPTO_LIB_POLY1305
124125
tristate

‎lib/crypto/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ ifeq ($(CONFIG_CRYPTO_LIB_NH_ARCH),y)
137137
CFLAGS_nh.o += -I$(src)/$(SRCARCH)
138138
libnh-$(CONFIG_ARM) += arm/nh-neon-core.o
139139
libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o
140+
libnh-$(CONFIG_X86) += x86/nh-sse2.o x86/nh-avx2.o
140141
endif
141142

142143
################################################################################

arch/x86/crypto/nh-avx2-x86_64.S renamed to lib/crypto/x86/nh-avx2.S

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
*/
99

1010
#include <linux/linkage.h>
11-
#include <linux/cfi_types.h>
1211

1312
#define PASS0_SUMS %ymm0
1413
#define PASS1_SUMS %ymm1
@@ -70,7 +69,7 @@
7069
*
7170
* It's guaranteed that message_len % 16 == 0.
7271
*/
73-
SYM_TYPED_FUNC_START(nh_avx2)
72+
SYM_FUNC_START(nh_avx2)
7473

7574
vmovdqu 0x00(KEY), K0
7675
vmovdqu 0x10(KEY), K1

arch/x86/crypto/nh-sse2-x86_64.S renamed to lib/crypto/x86/nh-sse2.S

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
*/
99

1010
#include <linux/linkage.h>
11-
#include <linux/cfi_types.h>
1211

1312
#define PASS0_SUMS %xmm0
1413
#define PASS1_SUMS %xmm1
@@ -72,7 +71,7 @@
7271
*
7372
* It's guaranteed that message_len % 16 == 0.
7473
*/
75-
SYM_TYPED_FUNC_START(nh_sse2)
74+
SYM_FUNC_START(nh_sse2)
7675

7776
movdqu 0x00(KEY), K0
7877
movdqu 0x10(KEY), K1

‎lib/crypto/x86/nh.h‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* x86_64 accelerated implementation of NH
4+
*
5+
* Copyright 2018 Google LLC
6+
*/
7+
8+
#include <asm/fpu/api.h>
9+
#include <linux/static_call.h>
10+
11+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2);
12+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2);
13+
14+
asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
15+
__le64 hash[NH_NUM_PASSES]);
16+
asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
17+
__le64 hash[NH_NUM_PASSES]);
18+
19+
static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
20+
__le64 hash[NH_NUM_PASSES])
21+
{
22+
if (message_len >= 64 && static_branch_likely(&have_sse2) &&
23+
irq_fpu_usable()) {
24+
kernel_fpu_begin();
25+
if (static_branch_likely(&have_avx2))
26+
nh_avx2(key, message, message_len, hash);
27+
else
28+
nh_sse2(key, message, message_len, hash);
29+
kernel_fpu_end();
30+
return true;
31+
}
32+
return false;
33+
}
34+
35+
#define nh_mod_init_arch nh_mod_init_arch
36+
static void nh_mod_init_arch(void)
37+
{
38+
if (boot_cpu_has(X86_FEATURE_XMM2)) {
39+
static_branch_enable(&have_sse2);
40+
if (boot_cpu_has(X86_FEATURE_AVX2) &&
41+
cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
42+
NULL))
43+
static_branch_enable(&have_avx2);
44+
}
45+
}

0 commit comments

Comments
 (0)