Skip to content

Commit acb13ea

Browse files
suryasaimadhuPeter Zijlstra
suryasaimadhu
authored and
Peter Zijlstra
committed
asm-generic/bitops: Always inline all bit manipulation helpers
Make it consistent with the atomic/atomic-instrumented.h helpers. And defconfig size is actually going down! text data bss dec hex filename 22352096 8213152 1917164 32482412 1efa46c vmlinux.x86-64.defconfig.before 22350551 8213184 1917164 32480899 1ef9e83 vmlinux.x86-64.defconfig.after Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Borislav Petkov <bp@suse.de> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Marco Elver <elver@google.com> Link: https://lore.kernel.org/r/20220113155357.4706-2-bp@alien8.de
1 parent 61cc453 commit acb13ea

2 files changed

Lines changed: 14 additions & 14 deletions

File tree

‎include/asm-generic/bitops/instrumented-atomic.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
* Note that @nr may be almost arbitrarily large; this function is not
2424
* restricted to acting on a single-word quantity.
2525
*/
26-
static inline void set_bit(long nr, volatile unsigned long *addr)
26+
static __always_inline void set_bit(long nr, volatile unsigned long *addr)
2727
{
2828
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
2929
arch_set_bit(nr, addr);
@@ -36,7 +36,7 @@ static inline void set_bit(long nr, volatile unsigned long *addr)
3636
*
3737
* This is a relaxed atomic operation (no implied memory barriers).
3838
*/
39-
static inline void clear_bit(long nr, volatile unsigned long *addr)
39+
static __always_inline void clear_bit(long nr, volatile unsigned long *addr)
4040
{
4141
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
4242
arch_clear_bit(nr, addr);
@@ -52,7 +52,7 @@ static inline void clear_bit(long nr, volatile unsigned long *addr)
5252
* Note that @nr may be almost arbitrarily large; this function is not
5353
* restricted to acting on a single-word quantity.
5454
*/
55-
static inline void change_bit(long nr, volatile unsigned long *addr)
55+
static __always_inline void change_bit(long nr, volatile unsigned long *addr)
5656
{
5757
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
5858
arch_change_bit(nr, addr);
@@ -65,7 +65,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr)
6565
*
6666
* This is an atomic fully-ordered operation (implied full memory barrier).
6767
*/
68-
static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
68+
static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
6969
{
7070
kcsan_mb();
7171
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
@@ -79,7 +79,7 @@ static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
7979
*
8080
* This is an atomic fully-ordered operation (implied full memory barrier).
8181
*/
82-
static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
82+
static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
8383
{
8484
kcsan_mb();
8585
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
@@ -93,7 +93,7 @@ static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
9393
*
9494
* This is an atomic fully-ordered operation (implied full memory barrier).
9595
*/
96-
static inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
96+
static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
9797
{
9898
kcsan_mb();
9999
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));

‎include/asm-generic/bitops/instrumented-non-atomic.h‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
* region of memory concurrently, the effect may be that only one operation
2323
* succeeds.
2424
*/
25-
static inline void __set_bit(long nr, volatile unsigned long *addr)
25+
static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
2626
{
2727
instrument_write(addr + BIT_WORD(nr), sizeof(long));
2828
arch___set_bit(nr, addr);
@@ -37,7 +37,7 @@ static inline void __set_bit(long nr, volatile unsigned long *addr)
3737
* region of memory concurrently, the effect may be that only one operation
3838
* succeeds.
3939
*/
40-
static inline void __clear_bit(long nr, volatile unsigned long *addr)
40+
static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
4141
{
4242
instrument_write(addr + BIT_WORD(nr), sizeof(long));
4343
arch___clear_bit(nr, addr);
@@ -52,13 +52,13 @@ static inline void __clear_bit(long nr, volatile unsigned long *addr)
5252
* region of memory concurrently, the effect may be that only one operation
5353
* succeeds.
5454
*/
55-
static inline void __change_bit(long nr, volatile unsigned long *addr)
55+
static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
5656
{
5757
instrument_write(addr + BIT_WORD(nr), sizeof(long));
5858
arch___change_bit(nr, addr);
5959
}
6060

61-
static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
61+
static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
6262
{
6363
if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
6464
/*
@@ -90,7 +90,7 @@ static inline void __instrument_read_write_bitop(long nr, volatile unsigned long
9090
* This operation is non-atomic. If two instances of this operation race, one
9191
* can appear to succeed but actually fail.
9292
*/
93-
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
93+
static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
9494
{
9595
__instrument_read_write_bitop(nr, addr);
9696
return arch___test_and_set_bit(nr, addr);
@@ -104,7 +104,7 @@ static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
104104
* This operation is non-atomic. If two instances of this operation race, one
105105
* can appear to succeed but actually fail.
106106
*/
107-
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
107+
static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
108108
{
109109
__instrument_read_write_bitop(nr, addr);
110110
return arch___test_and_clear_bit(nr, addr);
@@ -118,7 +118,7 @@ static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
118118
* This operation is non-atomic. If two instances of this operation race, one
119119
* can appear to succeed but actually fail.
120120
*/
121-
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
121+
static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
122122
{
123123
__instrument_read_write_bitop(nr, addr);
124124
return arch___test_and_change_bit(nr, addr);
@@ -129,7 +129,7 @@ static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
129129
* @nr: bit number to test
130130
* @addr: Address to start counting from
131131
*/
132-
static inline bool test_bit(long nr, const volatile unsigned long *addr)
132+
static __always_inline bool test_bit(long nr, const volatile unsigned long *addr)
133133
{
134134
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
135135
return arch_test_bit(nr, addr);

0 commit comments

Comments
 (0)