Skip to content

Commit 0bcfca5

Browse files
author
Eric Biggers
committed
lib/crc: Prepare for arch-optimized code in subdirs of lib/crc/
Rework how lib/crc/ supports arch-optimized code. First, instead of the arch-optimized CRC code being in arch/$(SRCARCH)/lib/, it will now be in lib/crc/$(SRCARCH)/. Second, the API functions (e.g. crc32c()), arch-optimized functions (e.g. crc32c_arch()), and generic functions (e.g. crc32c_base()) will now be part of a single module for each CRC type, allowing better inlining and dead code elimination. The second change is made possible by the first. As an example, consider CONFIG_CRC32=m on x86. We'll now have just crc32.ko instead of both crc32-x86.ko and crc32.ko. The two modules were already coupled together and always both got loaded together via direct symbol dependency, so the separation provided no benefit. Note: later I'd like to apply the same design to lib/crypto/ too, where often the API functions are out-of-line so this will work even better. In those cases, for each algorithm we currently have 3 modules all coupled together, e.g. libsha256.ko, libsha256-generic.ko, and sha256-x86.ko. We should have just one, inline things properly, and rely on the compiler's dead code elimination to decide the inclusion of the generic code instead of manually setting it via kconfig. Having arch-specific code outside arch/ was somewhat controversial when Zinc proposed it back in 2018. But I don't think the concerns are warranted. It's better from a technical perspective, as it enables the improvements mentioned above. This model is already successfully used in other places in the kernel such as lib/raid6/. The community of each architecture still remains free to work on the code, even if it's not in arch/. At the time there was also a desire to put the library code in the same files as the old-school crypto API, but that was a mistake; now that the library is separate, that's no longer a constraint either. Reviewed-by: "Martin K. Petersen" <martin.petersen@oracle.com> Acked-by: Ingo Molnar <mingo@kernel.org> Acked-by: "Jason A. Donenfeld" <Jason@zx2c4.com> Link: https://lore.kernel.org/r/20250607200454.73587-3-ebiggers@kernel.org Link: https://lore.kernel.org/r/20250612054514.142728-1-ebiggers@kernel.org Link: https://lore.kernel.org/r/20250621012221.4351-1-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
1 parent 89a5159 commit 0bcfca5

10 files changed

Lines changed: 159 additions & 89 deletions

File tree

‎Documentation/core-api/kernel-api.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ CRC Functions
148148
.. kernel-doc:: lib/crc/crc16.c
149149
:export:
150150

151-
.. kernel-doc:: lib/crc/crc32.c
151+
.. kernel-doc:: lib/crc/crc32-main.c
152152

153153
.. kernel-doc:: lib/crc/crc-ccitt.c
154154
:export:

‎MAINTAINERS‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6359,7 +6359,6 @@ L: linux-crypto@vger.kernel.org
63596359
S: Maintained
63606360
T: git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc-next
63616361
F: Documentation/staging/crc*
6362-
F: arch/*/lib/crc*
63636362
F: include/linux/crc*
63646363
F: lib/crc/
63656364
F: scripts/gen-crc-consts.py

‎include/linux/crc-t10dif.h‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,7 @@
44

55
#include <linux/types.h>
66

7-
u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len);
8-
u16 crc_t10dif_generic(u16 crc, const u8 *p, size_t len);
9-
10-
static inline u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len)
11-
{
12-
if (IS_ENABLED(CONFIG_CRC_T10DIF_ARCH))
13-
return crc_t10dif_arch(crc, p, len);
14-
return crc_t10dif_generic(crc, p, len);
15-
}
7+
u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len);
168

179
static inline u16 crc_t10dif(const u8 *p, size_t len)
1810
{

‎include/linux/crc32.h‎

Lines changed: 3 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,33 +5,9 @@
55
#include <linux/types.h>
66
#include <linux/bitrev.h>
77

8-
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len);
9-
u32 crc32_le_base(u32 crc, const u8 *p, size_t len);
10-
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len);
11-
u32 crc32_be_base(u32 crc, const u8 *p, size_t len);
12-
u32 crc32c_arch(u32 crc, const u8 *p, size_t len);
13-
u32 crc32c_base(u32 crc, const u8 *p, size_t len);
14-
15-
static inline u32 crc32_le(u32 crc, const void *p, size_t len)
16-
{
17-
if (IS_ENABLED(CONFIG_CRC32_ARCH))
18-
return crc32_le_arch(crc, p, len);
19-
return crc32_le_base(crc, p, len);
20-
}
21-
22-
static inline u32 crc32_be(u32 crc, const void *p, size_t len)
23-
{
24-
if (IS_ENABLED(CONFIG_CRC32_ARCH))
25-
return crc32_be_arch(crc, p, len);
26-
return crc32_be_base(crc, p, len);
27-
}
28-
29-
static inline u32 crc32c(u32 crc, const void *p, size_t len)
30-
{
31-
if (IS_ENABLED(CONFIG_CRC32_ARCH))
32-
return crc32c_arch(crc, p, len);
33-
return crc32c_base(crc, p, len);
34-
}
8+
u32 crc32_le(u32 crc, const void *p, size_t len);
9+
u32 crc32_be(u32 crc, const void *p, size_t len);
10+
u32 crc32c(u32 crc, const void *p, size_t len);
3511

3612
/*
3713
* crc32_optimizations() returns flags that indicate which CRC32 library

‎include/linux/crc64.h‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,14 @@
44

55
#include <linux/types.h>
66

7-
u64 crc64_be_arch(u64 crc, const u8 *p, size_t len);
8-
u64 crc64_be_generic(u64 crc, const u8 *p, size_t len);
9-
u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len);
10-
u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len);
11-
127
/**
138
* crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
149
* @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
1510
* or the previous crc64 value if computing incrementally.
1611
* @p: pointer to buffer over which CRC64 is run
1712
* @len: length of buffer @p
1813
*/
19-
static inline u64 crc64_be(u64 crc, const void *p, size_t len)
20-
{
21-
if (IS_ENABLED(CONFIG_CRC64_ARCH))
22-
return crc64_be_arch(crc, p, len);
23-
return crc64_be_generic(crc, p, len);
24-
}
14+
u64 crc64_be(u64 crc, const void *p, size_t len);
2515

2616
/**
2717
* crc64_nvme - Calculate CRC64-NVME
@@ -33,11 +23,6 @@ static inline u64 crc64_be(u64 crc, const void *p, size_t len)
3323
* This computes the CRC64 defined in the NVME NVM Command Set Specification,
3424
* *including the bitwise inversion at the beginning and end*.
3525
*/
36-
static inline u64 crc64_nvme(u64 crc, const void *p, size_t len)
37-
{
38-
if (IS_ENABLED(CONFIG_CRC64_ARCH))
39-
return ~crc64_nvme_arch(~crc, p, len);
40-
return ~crc64_nvme_generic(~crc, p, len);
41-
}
26+
u64 crc64_nvme(u64 crc, const void *p, size_t len);
4227

4328
#endif /* _LINUX_CRC64_H */

‎lib/crc/Kconfig‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ config ARCH_HAS_CRC_T10DIF
4848
bool
4949

5050
config CRC_T10DIF_ARCH
51-
tristate
52-
default CRC_T10DIF if ARCH_HAS_CRC_T10DIF && CRC_OPTIMIZATIONS
51+
bool
52+
depends on CRC_T10DIF && CRC_OPTIMIZATIONS
5353

5454
config CRC32
5555
tristate
@@ -62,8 +62,8 @@ config ARCH_HAS_CRC32
6262
bool
6363

6464
config CRC32_ARCH
65-
tristate
66-
default CRC32 if ARCH_HAS_CRC32 && CRC_OPTIMIZATIONS
65+
bool
66+
depends on CRC32 && CRC_OPTIMIZATIONS
6767

6868
config CRC64
6969
tristate
@@ -75,11 +75,12 @@ config ARCH_HAS_CRC64
7575
bool
7676

7777
config CRC64_ARCH
78-
tristate
79-
default CRC64 if ARCH_HAS_CRC64 && CRC_OPTIMIZATIONS
78+
bool
79+
depends on CRC64 && CRC_OPTIMIZATIONS
8080

8181
config CRC_OPTIMIZATIONS
8282
bool "Enable optimized CRC implementations" if EXPERT
83+
depends on !UML
8384
default y
8485
help
8586
Disabling this option reduces code size slightly by disabling the

‎lib/crc/Makefile‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,32 @@ obj-$(CONFIG_CRC8) += crc8.o
88
obj-$(CONFIG_CRC16) += crc16.o
99
obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o
1010
obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
11+
1112
obj-$(CONFIG_CRC_T10DIF) += crc-t10dif.o
13+
crc-t10dif-y := crc-t10dif-main.o
14+
ifeq ($(CONFIG_CRC_T10DIF_ARCH),y)
15+
CFLAGS_crc-t10dif-main.o += -I$(src)/$(SRCARCH)
16+
endif
17+
1218
obj-$(CONFIG_CRC32) += crc32.o
19+
crc32-y := crc32-main.o
20+
ifeq ($(CONFIG_CRC32_ARCH),y)
21+
CFLAGS_crc32-main.o += -I$(src)/$(SRCARCH)
22+
endif
23+
1324
obj-$(CONFIG_CRC64) += crc64.o
25+
crc64-y := crc64-main.o
26+
ifeq ($(CONFIG_CRC64_ARCH),y)
27+
CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH)
28+
endif
29+
1430
obj-y += tests/
1531

1632
hostprogs := gen_crc32table gen_crc64table
1733
clean-files := crc32table.h crc64table.h
1834

19-
$(obj)/crc32.o: $(obj)/crc32table.h
20-
$(obj)/crc64.o: $(obj)/crc64table.h
35+
$(obj)/crc32-main.o: $(obj)/crc32table.h
36+
$(obj)/crc64-main.o: $(obj)/crc64table.h
2137

2238
quiet_cmd_crc32 = GEN $@
2339
cmd_crc32 = $< > $@

lib/crc/crc-t10dif.c renamed to lib/crc/crc-t10dif-main.c

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,16 +50,39 @@ static const u16 t10_dif_crc_table[256] = {
5050
0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
5151
};
5252

53-
u16 crc_t10dif_generic(u16 crc, const u8 *p, size_t len)
53+
static inline u16 __maybe_unused
54+
crc_t10dif_generic(u16 crc, const u8 *p, size_t len)
5455
{
55-
size_t i;
56+
while (len--)
57+
crc = (crc << 8) ^ t10_dif_crc_table[(crc >> 8) ^ *p++];
58+
return crc;
59+
}
5660

57-
for (i = 0; i < len; i++)
58-
crc = (crc << 8) ^ t10_dif_crc_table[(crc >> 8) ^ p[i]];
61+
#ifdef CONFIG_CRC_T10DIF_ARCH
62+
#include "crc-t10dif.h" /* $(SRCARCH)/crc-t10dif.h */
63+
#else
64+
#define crc_t10dif_arch crc_t10dif_generic
65+
#endif
5966

60-
return crc;
67+
u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len)
68+
{
69+
return crc_t10dif_arch(crc, p, len);
70+
}
71+
EXPORT_SYMBOL(crc_t10dif_update);
72+
73+
#ifdef crc_t10dif_mod_init_arch
74+
static int __init crc_t10dif_mod_init(void)
75+
{
76+
crc_t10dif_mod_init_arch();
77+
return 0;
78+
}
79+
subsys_initcall(crc_t10dif_mod_init);
80+
81+
static void __exit crc_t10dif_mod_exit(void)
82+
{
6183
}
62-
EXPORT_SYMBOL(crc_t10dif_generic);
84+
module_exit(crc_t10dif_mod_exit);
85+
#endif
6386

64-
MODULE_DESCRIPTION("T10 DIF CRC calculation");
87+
MODULE_DESCRIPTION("CRC-T10DIF library functions");
6588
MODULE_LICENSE("GPL");

lib/crc/crc32.c renamed to lib/crc/crc32-main.c

Lines changed: 57 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,30 +30,75 @@
3030

3131
#include "crc32table.h"
3232

33-
MODULE_AUTHOR("Matt Domsch <Matt_Domsch@dell.com>");
34-
MODULE_DESCRIPTION("Various CRC32 calculations");
35-
MODULE_LICENSE("GPL");
36-
37-
u32 crc32_le_base(u32 crc, const u8 *p, size_t len)
33+
static inline u32 __maybe_unused
34+
crc32_le_base(u32 crc, const u8 *p, size_t len)
3835
{
3936
while (len--)
4037
crc = (crc >> 8) ^ crc32table_le[(crc & 255) ^ *p++];
4138
return crc;
4239
}
43-
EXPORT_SYMBOL(crc32_le_base);
4440

45-
u32 crc32c_base(u32 crc, const u8 *p, size_t len)
41+
static inline u32 __maybe_unused
42+
crc32_be_base(u32 crc, const u8 *p, size_t len)
4643
{
4744
while (len--)
48-
crc = (crc >> 8) ^ crc32ctable_le[(crc & 255) ^ *p++];
45+
crc = (crc << 8) ^ crc32table_be[(crc >> 24) ^ *p++];
4946
return crc;
5047
}
51-
EXPORT_SYMBOL(crc32c_base);
5248

53-
u32 crc32_be_base(u32 crc, const u8 *p, size_t len)
49+
static inline u32 __maybe_unused
50+
crc32c_base(u32 crc, const u8 *p, size_t len)
5451
{
5552
while (len--)
56-
crc = (crc << 8) ^ crc32table_be[(crc >> 24) ^ *p++];
53+
crc = (crc >> 8) ^ crc32ctable_le[(crc & 255) ^ *p++];
5754
return crc;
5855
}
59-
EXPORT_SYMBOL(crc32_be_base);
56+
57+
#ifdef CONFIG_CRC32_ARCH
58+
#include "crc32.h" /* $(SRCARCH)/crc32.h */
59+
60+
u32 crc32_optimizations(void)
61+
{
62+
return crc32_optimizations_arch();
63+
}
64+
EXPORT_SYMBOL(crc32_optimizations);
65+
#else
66+
#define crc32_le_arch crc32_le_base
67+
#define crc32_be_arch crc32_be_base
68+
#define crc32c_arch crc32c_base
69+
#endif
70+
71+
u32 crc32_le(u32 crc, const void *p, size_t len)
72+
{
73+
return crc32_le_arch(crc, p, len);
74+
}
75+
EXPORT_SYMBOL(crc32_le);
76+
77+
u32 crc32_be(u32 crc, const void *p, size_t len)
78+
{
79+
return crc32_be_arch(crc, p, len);
80+
}
81+
EXPORT_SYMBOL(crc32_be);
82+
83+
u32 crc32c(u32 crc, const void *p, size_t len)
84+
{
85+
return crc32c_arch(crc, p, len);
86+
}
87+
EXPORT_SYMBOL(crc32c);
88+
89+
#ifdef crc32_mod_init_arch
90+
static int __init crc32_mod_init(void)
91+
{
92+
crc32_mod_init_arch();
93+
return 0;
94+
}
95+
subsys_initcall(crc32_mod_init);
96+
97+
static void __exit crc32_mod_exit(void)
98+
{
99+
}
100+
module_exit(crc32_mod_exit);
101+
#endif
102+
103+
MODULE_DESCRIPTION("CRC32 library functions");
104+
MODULE_LICENSE("GPL");

lib/crc/crc64.c renamed to lib/crc/crc64-main.c

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,21 +38,54 @@
3838
#include <linux/crc64.h>
3939
#include "crc64table.h"
4040

41-
MODULE_DESCRIPTION("CRC64 calculations");
42-
MODULE_LICENSE("GPL v2");
43-
44-
u64 crc64_be_generic(u64 crc, const u8 *p, size_t len)
41+
static inline u64 __maybe_unused
42+
crc64_be_generic(u64 crc, const u8 *p, size_t len)
4543
{
4644
while (len--)
4745
crc = (crc << 8) ^ crc64table[(crc >> 56) ^ *p++];
4846
return crc;
4947
}
50-
EXPORT_SYMBOL_GPL(crc64_be_generic);
5148

52-
u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len)
49+
static inline u64 __maybe_unused
50+
crc64_nvme_generic(u64 crc, const u8 *p, size_t len)
5351
{
5452
while (len--)
5553
crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *p++];
5654
return crc;
5755
}
58-
EXPORT_SYMBOL_GPL(crc64_nvme_generic);
56+
57+
#ifdef CONFIG_CRC64_ARCH
58+
#include "crc64.h" /* $(SRCARCH)/crc64.h */
59+
#else
60+
#define crc64_be_arch crc64_be_generic
61+
#define crc64_nvme_arch crc64_nvme_generic
62+
#endif
63+
64+
u64 crc64_be(u64 crc, const void *p, size_t len)
65+
{
66+
return crc64_be_arch(crc, p, len);
67+
}
68+
EXPORT_SYMBOL_GPL(crc64_be);
69+
70+
u64 crc64_nvme(u64 crc, const void *p, size_t len)
71+
{
72+
return ~crc64_nvme_arch(~crc, p, len);
73+
}
74+
EXPORT_SYMBOL_GPL(crc64_nvme);
75+
76+
#ifdef crc64_mod_init_arch
77+
static int __init crc64_mod_init(void)
78+
{
79+
crc64_mod_init_arch();
80+
return 0;
81+
}
82+
subsys_initcall(crc64_mod_init);
83+
84+
static void __exit crc64_mod_exit(void)
85+
{
86+
}
87+
module_exit(crc64_mod_exit);
88+
#endif
89+
90+
MODULE_DESCRIPTION("CRC64 library functions");
91+
MODULE_LICENSE("GPL");

0 commit comments

Comments
 (0)