Skip to content

Commit 2b7531b

Browse files
author
Eric Biggers
committed
lib/crc: arm64: Migrate optimized CRC code into lib/crc/
Move the arm64-optimized CRC code from arch/arm64/lib/crc* into its new location in lib/crc/arm64/, and wire it up in the new way. This new way of organizing the CRC code eliminates the need to artificially split the code for each CRC variant into separate arch and generic modules, enabling better inlining and dead code elimination. For more details, see "lib/crc: Prepare for arch-optimized code in subdirs of lib/crc/". Reviewed-by: "Martin K. Petersen" <martin.petersen@oracle.com> Acked-by: Ingo Molnar <mingo@kernel.org> Acked-by: "Jason A. Donenfeld" <Jason@zx2c4.com> Link: https://lore.kernel.org/r/20250607200454.73587-5-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
1 parent 530b304 commit 2b7531b

8 files changed

Lines changed: 11 additions & 42 deletions

File tree

‎arch/arm64/Kconfig‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@ config ARM64
2121
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
2222
select ARCH_HAS_CACHE_LINE_SIZE
2323
select ARCH_HAS_CC_PLATFORM
24-
select ARCH_HAS_CRC32
25-
select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON
2624
select ARCH_HAS_CURRENT_STACK_POINTER
2725
select ARCH_HAS_DEBUG_VIRTUAL
2826
select ARCH_HAS_DEBUG_VM_PGTABLE

‎arch/arm64/lib/Makefile‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,6 @@ endif
1616

1717
lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
1818

19-
obj-$(CONFIG_CRC32_ARCH) += crc32-arm64.o
20-
crc32-arm64-y := crc32.o crc32-core.o
21-
22-
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm64.o
23-
crc-t10dif-arm64-y := crc-t10dif.o crc-t10dif-core.o
24-
2519
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
2620

2721
obj-$(CONFIG_ARM64_MTE) += mte.o

‎lib/crc/Kconfig‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ config CRC_T10DIF_ARCH
5151
bool
5252
depends on CRC_T10DIF && CRC_OPTIMIZATIONS
5353
default y if ARM && KERNEL_MODE_NEON
54+
default y if ARM64 && KERNEL_MODE_NEON
5455

5556
config CRC32
5657
tristate
@@ -66,6 +67,7 @@ config CRC32_ARCH
6667
bool
6768
depends on CRC32 && CRC_OPTIMIZATIONS
6869
default y if ARM && KERNEL_MODE_NEON
70+
default y if ARM64
6971

7072
config CRC64
7173
tristate

‎lib/crc/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,15 @@ crc-t10dif-y := crc-t10dif-main.o
1414
ifeq ($(CONFIG_CRC_T10DIF_ARCH),y)
1515
CFLAGS_crc-t10dif-main.o += -I$(src)/$(SRCARCH)
1616
crc-t10dif-$(CONFIG_ARM) += arm/crc-t10dif-core.o
17+
crc-t10dif-$(CONFIG_ARM64) += arm64/crc-t10dif-core.o
1718
endif
1819

1920
obj-$(CONFIG_CRC32) += crc32.o
2021
crc32-y := crc32-main.o
2122
ifeq ($(CONFIG_CRC32_ARCH),y)
2223
CFLAGS_crc32-main.o += -I$(src)/$(SRCARCH)
2324
crc32-$(CONFIG_ARM) += arm/crc32-core.o
25+
crc32-$(CONFIG_ARM64) += arm64/crc32-core.o
2426
endif
2527

2628
obj-$(CONFIG_CRC64) += crc64.o

arch/arm64/lib/crc-t10dif-core.S renamed to lib/crc/arm64/crc-t10dif-core.S

File renamed without changes.

arch/arm64/lib/crc-t10dif.c renamed to lib/crc/arm64/crc-t10dif.h

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,6 @@
66
*/
77

88
#include <linux/cpufeature.h>
9-
#include <linux/crc-t10dif.h>
10-
#include <linux/init.h>
11-
#include <linux/kernel.h>
12-
#include <linux/module.h>
13-
#include <linux/string.h>
149

1510
#include <crypto/internal/simd.h>
1611

@@ -26,7 +21,7 @@ asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len,
2621
u8 out[16]);
2722
asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
2823

29-
u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
24+
static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
3025
{
3126
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
3227
if (static_branch_likely(&have_pmull)) {
@@ -50,24 +45,13 @@ u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
5045
}
5146
return crc_t10dif_generic(crc, data, length);
5247
}
53-
EXPORT_SYMBOL(crc_t10dif_arch);
5448

55-
static int __init crc_t10dif_arm64_init(void)
49+
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
50+
static inline void crc_t10dif_mod_init_arch(void)
5651
{
5752
if (cpu_have_named_feature(ASIMD)) {
5853
static_branch_enable(&have_asimd);
5954
if (cpu_have_named_feature(PMULL))
6055
static_branch_enable(&have_pmull);
6156
}
62-
return 0;
6357
}
64-
subsys_initcall(crc_t10dif_arm64_init);
65-
66-
static void __exit crc_t10dif_arm64_exit(void)
67-
{
68-
}
69-
module_exit(crc_t10dif_arm64_exit);
70-
71-
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
72-
MODULE_DESCRIPTION("CRC-T10DIF using arm64 NEON and Crypto Extensions");
73-
MODULE_LICENSE("GPL v2");

arch/arm64/lib/crc32-core.S renamed to lib/crc/arm64/crc32-core.S

File renamed without changes.

arch/arm64/lib/crc32.c renamed to lib/crc/arm64/crc32.h

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
// SPDX-License-Identifier: GPL-2.0-only
22

3-
#include <linux/crc32.h>
4-
#include <linux/linkage.h>
5-
#include <linux/module.h>
6-
73
#include <asm/alternative.h>
84
#include <asm/cpufeature.h>
95
#include <asm/neon.h>
@@ -22,7 +18,7 @@ asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
2218
asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
2319
asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len);
2420

25-
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
21+
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
2622
{
2723
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
2824
return crc32_le_base(crc, p, len);
@@ -41,9 +37,8 @@ u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
4137

4238
return crc32_le_arm64(crc, p, len);
4339
}
44-
EXPORT_SYMBOL(crc32_le_arch);
4540

46-
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
41+
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
4742
{
4843
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
4944
return crc32c_base(crc, p, len);
@@ -62,9 +57,8 @@ u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
6257

6358
return crc32c_le_arm64(crc, p, len);
6459
}
65-
EXPORT_SYMBOL(crc32c_arch);
6660

67-
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
61+
static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
6862
{
6963
if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
7064
return crc32_be_base(crc, p, len);
@@ -83,17 +77,12 @@ u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
8377

8478
return crc32_be_arm64(crc, p, len);
8579
}
86-
EXPORT_SYMBOL(crc32_be_arch);
8780

88-
u32 crc32_optimizations(void)
81+
static inline u32 crc32_optimizations_arch(void)
8982
{
9083
if (alternative_has_cap_likely(ARM64_HAS_CRC32))
9184
return CRC32_LE_OPTIMIZATION |
9285
CRC32_BE_OPTIMIZATION |
9386
CRC32C_OPTIMIZATION;
9487
return 0;
9588
}
96-
EXPORT_SYMBOL(crc32_optimizations);
97-
98-
MODULE_LICENSE("GPL");
99-
MODULE_DESCRIPTION("arm64-optimized CRC32 functions");

0 commit comments

Comments
 (0)