Skip to content

Commit 530b304

Browse files
author
Eric Biggers
committed
lib/crc: arm: Migrate optimized CRC code into lib/crc/
Move the arm-optimized CRC code from arch/arm/lib/crc* into its new location in lib/crc/arm/, and wire it up in the new way. This new way of organizing the CRC code eliminates the need to artificially split the code for each CRC variant into separate arch and generic modules, enabling better inlining and dead code elimination. For more details, see "lib/crc: Prepare for arch-optimized code in subdirs of lib/crc/". Reviewed-by: "Martin K. Petersen" <martin.petersen@oracle.com> Acked-by: Ingo Molnar <mingo@kernel.org> Acked-by: "Jason A. Donenfeld" <Jason@zx2c4.com> Link: https://lore.kernel.org/r/20250607200454.73587-4-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
1 parent 0bcfca5 commit 530b304

8 files changed

Lines changed: 15 additions & 58 deletions

File tree

‎arch/arm/Kconfig‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ config ARM
88
select ARCH_HAS_CACHE_LINE_SIZE if OF
99
select ARCH_HAS_CPU_CACHE_ALIASING
1010
select ARCH_HAS_CPU_FINALIZE_INIT if MMU
11-
select ARCH_HAS_CRC32 if KERNEL_MODE_NEON
12-
select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON
1311
select ARCH_HAS_CURRENT_STACK_POINTER
1412
select ARCH_HAS_DEBUG_VIRTUAL if MMU
1513
select ARCH_HAS_DMA_ALLOC if MMU

‎arch/arm/lib/Makefile‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,3 @@ ifeq ($(CONFIG_KERNEL_MODE_NEON),y)
4747
endif
4848

4949
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
50-
51-
obj-$(CONFIG_CRC32_ARCH) += crc32-arm.o
52-
crc32-arm-y := crc32.o crc32-core.o
53-
54-
obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm.o
55-
crc-t10dif-arm-y := crc-t10dif.o crc-t10dif-core.o

‎lib/crc/Kconfig‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ config ARCH_HAS_CRC_T10DIF
5050
config CRC_T10DIF_ARCH
5151
bool
5252
depends on CRC_T10DIF && CRC_OPTIMIZATIONS
53+
default y if ARM && KERNEL_MODE_NEON
5354

5455
config CRC32
5556
tristate
@@ -64,6 +65,7 @@ config ARCH_HAS_CRC32
6465
config CRC32_ARCH
6566
bool
6667
depends on CRC32 && CRC_OPTIMIZATIONS
68+
default y if ARM && KERNEL_MODE_NEON
6769

6870
config CRC64
6971
tristate

‎lib/crc/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@ obj-$(CONFIG_CRC_T10DIF) += crc-t10dif.o
1313
crc-t10dif-y := crc-t10dif-main.o
1414
ifeq ($(CONFIG_CRC_T10DIF_ARCH),y)
1515
CFLAGS_crc-t10dif-main.o += -I$(src)/$(SRCARCH)
16+
crc-t10dif-$(CONFIG_ARM) += arm/crc-t10dif-core.o
1617
endif
1718

1819
obj-$(CONFIG_CRC32) += crc32.o
1920
crc32-y := crc32-main.o
2021
ifeq ($(CONFIG_CRC32_ARCH),y)
2122
CFLAGS_crc32-main.o += -I$(src)/$(SRCARCH)
23+
crc32-$(CONFIG_ARM) += arm/crc32-core.o
2224
endif
2325

2426
obj-$(CONFIG_CRC64) += crc64.o

arch/arm/lib/crc-t10dif-core.S renamed to lib/crc/arm/crc-t10dif-core.S

File renamed without changes.

arch/arm/lib/crc-t10dif.c renamed to lib/crc/arm/crc-t10dif.h

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,6 @@
55
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
66
*/
77

8-
#include <linux/crc-t10dif.h>
9-
#include <linux/init.h>
10-
#include <linux/kernel.h>
11-
#include <linux/module.h>
12-
#include <linux/string.h>
13-
148
#include <crypto/internal/simd.h>
159

1610
#include <asm/neon.h>
@@ -25,7 +19,7 @@ asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len);
2519
asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len,
2620
u8 out[16]);
2721

28-
u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
22+
static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
2923
{
3024
if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
3125
if (static_branch_likely(&have_pmull)) {
@@ -49,24 +43,13 @@ u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
4943
}
5044
return crc_t10dif_generic(crc, data, length);
5145
}
52-
EXPORT_SYMBOL(crc_t10dif_arch);
5346

54-
static int __init crc_t10dif_arm_init(void)
47+
#define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
48+
static inline void crc_t10dif_mod_init_arch(void)
5549
{
5650
if (elf_hwcap & HWCAP_NEON) {
5751
static_branch_enable(&have_neon);
5852
if (elf_hwcap2 & HWCAP2_PMULL)
5953
static_branch_enable(&have_pmull);
6054
}
61-
return 0;
6255
}
63-
subsys_initcall(crc_t10dif_arm_init);
64-
65-
static void __exit crc_t10dif_arm_exit(void)
66-
{
67-
}
68-
module_exit(crc_t10dif_arm_exit);
69-
70-
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
71-
MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions");
72-
MODULE_LICENSE("GPL v2");

arch/arm/lib/crc32-core.S renamed to lib/crc/arm/crc32-core.S

File renamed without changes.

arch/arm/lib/crc32.c renamed to lib/crc/arm/crc32.h

Lines changed: 8 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,6 @@
66
*/
77

88
#include <linux/cpufeature.h>
9-
#include <linux/crc32.h>
10-
#include <linux/init.h>
11-
#include <linux/kernel.h>
12-
#include <linux/module.h>
13-
#include <linux/string.h>
149

1510
#include <crypto/internal/simd.h>
1611

@@ -29,14 +24,14 @@ asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
2924
asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
3025
asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
3126

32-
static u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len)
27+
static inline u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len)
3328
{
3429
if (static_branch_likely(&have_crc32))
3530
return crc32_armv8_le(crc, p, len);
3631
return crc32_le_base(crc, p, len);
3732
}
3833

39-
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
34+
static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
4035
{
4136
if (len >= PMULL_MIN_LEN + 15 &&
4237
static_branch_likely(&have_pmull) && crypto_simd_usable()) {
@@ -57,16 +52,15 @@ u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
5752
}
5853
return crc32_le_scalar(crc, p, len);
5954
}
60-
EXPORT_SYMBOL(crc32_le_arch);
6155

62-
static u32 crc32c_scalar(u32 crc, const u8 *p, size_t len)
56+
static inline u32 crc32c_scalar(u32 crc, const u8 *p, size_t len)
6357
{
6458
if (static_branch_likely(&have_crc32))
6559
return crc32c_armv8_le(crc, p, len);
6660
return crc32c_base(crc, p, len);
6761
}
6862

69-
u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
63+
static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
7064
{
7165
if (len >= PMULL_MIN_LEN + 15 &&
7266
static_branch_likely(&have_pmull) && crypto_simd_usable()) {
@@ -87,37 +81,21 @@ u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
8781
}
8882
return crc32c_scalar(crc, p, len);
8983
}
90-
EXPORT_SYMBOL(crc32c_arch);
9184

92-
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
93-
{
94-
return crc32_be_base(crc, p, len);
95-
}
96-
EXPORT_SYMBOL(crc32_be_arch);
85+
#define crc32_be_arch crc32_be_base /* not implemented on this arch */
9786

98-
static int __init crc32_arm_init(void)
87+
#define crc32_mod_init_arch crc32_mod_init_arch
88+
static inline void crc32_mod_init_arch(void)
9989
{
10090
if (elf_hwcap2 & HWCAP2_CRC32)
10191
static_branch_enable(&have_crc32);
10292
if (elf_hwcap2 & HWCAP2_PMULL)
10393
static_branch_enable(&have_pmull);
104-
return 0;
10594
}
106-
subsys_initcall(crc32_arm_init);
10795

108-
static void __exit crc32_arm_exit(void)
109-
{
110-
}
111-
module_exit(crc32_arm_exit);
112-
113-
u32 crc32_optimizations(void)
96+
static inline u32 crc32_optimizations_arch(void)
11497
{
11598
if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL))
11699
return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
117100
return 0;
118101
}
119-
EXPORT_SYMBOL(crc32_optimizations);
120-
121-
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
122-
MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions");
123-
MODULE_LICENSE("GPL v2");

0 commit comments

Comments
 (0)