Skip to content

Commit ec57032

Browse files
mrutland-armingomolnar
mrutland-arm
authored and
ingomolnar
committed
locking/atomic: Correct (cmp)xchg() instrumentation
All xchg() and cmpxchg() ops are atomic RMWs, but currently we instrument these with instrument_atomic_write() rather than instrument_atomic_read_write(), missing the read aspect. Similarly, all try_cmpxchg() ops are non-atomic RMWs on *oldp, but we instrument these accesses with instrument_atomic_write() rather than instrument_read_write(), missing the read aspect and erroneously marking these as atomic. Fix the instrumentation for both points. Signed-off-by: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Ingo Molnar <mingo@kernel.org> Link: https://lkml.kernel.org/r/20230413160644.490976-1-mark.rutland@arm.com Cc: Linus Torvalds <torvalds@linux-foundation.org>
1 parent 5cd4c26 commit ec57032

2 files changed

Lines changed: 41 additions & 41 deletions

File tree

‎include/linux/atomic/atomic-instrumented.h‎

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1948,89 +1948,89 @@ atomic_long_dec_if_positive(atomic_long_t *v)
19481948
({ \
19491949
typeof(ptr) __ai_ptr = (ptr); \
19501950
kcsan_mb(); \
1951-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1951+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
19521952
arch_xchg(__ai_ptr, __VA_ARGS__); \
19531953
})
19541954

19551955
#define xchg_acquire(ptr, ...) \
19561956
({ \
19571957
typeof(ptr) __ai_ptr = (ptr); \
1958-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1958+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
19591959
arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
19601960
})
19611961

19621962
#define xchg_release(ptr, ...) \
19631963
({ \
19641964
typeof(ptr) __ai_ptr = (ptr); \
19651965
kcsan_release(); \
1966-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1966+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
19671967
arch_xchg_release(__ai_ptr, __VA_ARGS__); \
19681968
})
19691969

19701970
#define xchg_relaxed(ptr, ...) \
19711971
({ \
19721972
typeof(ptr) __ai_ptr = (ptr); \
1973-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1973+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
19741974
arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
19751975
})
19761976

19771977
#define cmpxchg(ptr, ...) \
19781978
({ \
19791979
typeof(ptr) __ai_ptr = (ptr); \
19801980
kcsan_mb(); \
1981-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1981+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
19821982
arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
19831983
})
19841984

19851985
#define cmpxchg_acquire(ptr, ...) \
19861986
({ \
19871987
typeof(ptr) __ai_ptr = (ptr); \
1988-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1988+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
19891989
arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
19901990
})
19911991

19921992
#define cmpxchg_release(ptr, ...) \
19931993
({ \
19941994
typeof(ptr) __ai_ptr = (ptr); \
19951995
kcsan_release(); \
1996-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
19971997
arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
19981998
})
19991999

20002000
#define cmpxchg_relaxed(ptr, ...) \
20012001
({ \
20022002
typeof(ptr) __ai_ptr = (ptr); \
2003-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2003+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
20042004
arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
20052005
})
20062006

20072007
#define cmpxchg64(ptr, ...) \
20082008
({ \
20092009
typeof(ptr) __ai_ptr = (ptr); \
20102010
kcsan_mb(); \
2011-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2011+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
20122012
arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
20132013
})
20142014

20152015
#define cmpxchg64_acquire(ptr, ...) \
20162016
({ \
20172017
typeof(ptr) __ai_ptr = (ptr); \
2018-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2018+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
20192019
arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
20202020
})
20212021

20222022
#define cmpxchg64_release(ptr, ...) \
20232023
({ \
20242024
typeof(ptr) __ai_ptr = (ptr); \
20252025
kcsan_release(); \
2026-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2026+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
20272027
arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
20282028
})
20292029

20302030
#define cmpxchg64_relaxed(ptr, ...) \
20312031
({ \
20322032
typeof(ptr) __ai_ptr = (ptr); \
2033-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2033+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
20342034
arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
20352035
})
20362036

@@ -2039,17 +2039,17 @@ atomic_long_dec_if_positive(atomic_long_t *v)
20392039
typeof(ptr) __ai_ptr = (ptr); \
20402040
typeof(oldp) __ai_oldp = (oldp); \
20412041
kcsan_mb(); \
2042-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2042+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
20442044
arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
20452045
})
20462046

20472047
#define try_cmpxchg_acquire(ptr, oldp, ...) \
20482048
({ \
20492049
typeof(ptr) __ai_ptr = (ptr); \
20502050
typeof(oldp) __ai_oldp = (oldp); \
2051-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2052-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2051+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2052+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
20532053
arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
20542054
})
20552055

@@ -2058,17 +2058,17 @@ atomic_long_dec_if_positive(atomic_long_t *v)
20582058
typeof(ptr) __ai_ptr = (ptr); \
20592059
typeof(oldp) __ai_oldp = (oldp); \
20602060
kcsan_release(); \
2061-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2062-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2061+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2062+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
20632063
arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
20642064
})
20652065

20662066
#define try_cmpxchg_relaxed(ptr, oldp, ...) \
20672067
({ \
20682068
typeof(ptr) __ai_ptr = (ptr); \
20692069
typeof(oldp) __ai_oldp = (oldp); \
2070-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2071-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2070+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2071+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
20722072
arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
20732073
})
20742074

@@ -2077,17 +2077,17 @@ atomic_long_dec_if_positive(atomic_long_t *v)
20772077
typeof(ptr) __ai_ptr = (ptr); \
20782078
typeof(oldp) __ai_oldp = (oldp); \
20792079
kcsan_mb(); \
2080-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2081-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2080+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2081+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
20822082
arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
20832083
})
20842084

20852085
#define try_cmpxchg64_acquire(ptr, oldp, ...) \
20862086
({ \
20872087
typeof(ptr) __ai_ptr = (ptr); \
20882088
typeof(oldp) __ai_oldp = (oldp); \
2089-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2090-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2089+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2090+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
20912091
arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
20922092
})
20932093

@@ -2096,75 +2096,75 @@ atomic_long_dec_if_positive(atomic_long_t *v)
20962096
typeof(ptr) __ai_ptr = (ptr); \
20972097
typeof(oldp) __ai_oldp = (oldp); \
20982098
kcsan_release(); \
2099-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2100-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2099+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2100+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
21012101
arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
21022102
})
21032103

21042104
#define try_cmpxchg64_relaxed(ptr, oldp, ...) \
21052105
({ \
21062106
typeof(ptr) __ai_ptr = (ptr); \
21072107
typeof(oldp) __ai_oldp = (oldp); \
2108-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2109-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2108+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2109+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
21102110
arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
21112111
})
21122112

21132113
#define cmpxchg_local(ptr, ...) \
21142114
({ \
21152115
typeof(ptr) __ai_ptr = (ptr); \
2116-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2116+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
21172117
arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
21182118
})
21192119

21202120
#define cmpxchg64_local(ptr, ...) \
21212121
({ \
21222122
typeof(ptr) __ai_ptr = (ptr); \
2123-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2123+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
21242124
arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
21252125
})
21262126

21272127
#define sync_cmpxchg(ptr, ...) \
21282128
({ \
21292129
typeof(ptr) __ai_ptr = (ptr); \
21302130
kcsan_mb(); \
2131-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2131+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
21322132
arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
21332133
})
21342134

21352135
#define try_cmpxchg_local(ptr, oldp, ...) \
21362136
({ \
21372137
typeof(ptr) __ai_ptr = (ptr); \
21382138
typeof(oldp) __ai_oldp = (oldp); \
2139-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2140-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2139+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2140+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
21412141
arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
21422142
})
21432143

21442144
#define try_cmpxchg64_local(ptr, oldp, ...) \
21452145
({ \
21462146
typeof(ptr) __ai_ptr = (ptr); \
21472147
typeof(oldp) __ai_oldp = (oldp); \
2148-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2149-
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2148+
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2149+
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
21502150
arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
21512151
})
21522152

21532153
#define cmpxchg_double(ptr, ...) \
21542154
({ \
21552155
typeof(ptr) __ai_ptr = (ptr); \
21562156
kcsan_mb(); \
2157-
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2157+
instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
21582158
arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
21592159
})
21602160

21612161

21622162
#define cmpxchg_double_local(ptr, ...) \
21632163
({ \
21642164
typeof(ptr) __ai_ptr = (ptr); \
2165-
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2165+
instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
21662166
arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
21672167
})
21682168

21692169
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2170-
// 97fe4d79aa058d2164df824632cbc4f716d2a407
2170+
// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e

‎scripts/atomic/gen-atomic-instrumented.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,8 @@ cat <<EOF
104104
EOF
105105
[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
106106
cat <<EOF
107-
instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
108-
instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
107+
instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
108+
instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
109109
arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
110110
})
111111
EOF
@@ -119,7 +119,7 @@ cat <<EOF
119119
EOF
120120
[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
121121
cat <<EOF
122-
instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
122+
instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
123123
arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\
124124
})
125125
EOF

0 commit comments

Comments
 (0)