Skip to content

Commit 08cac60

Browse files
melverpaulmckrcu
authored andcommitted
kcsan: Reduce get_ctx() uses in kcsan_found_watchpoint()
There are a number get_ctx() calls that are close to each other, which results in poor codegen (repeated preempt_count loads). Specifically in kcsan_found_watchpoint() (even though it's a slow-path) it is beneficial to keep the race-window small until the watchpoint has actually been consumed to avoid missed opportunities to report a race. Let's clean it up a bit before we add more code in kcsan_found_watchpoint(). Signed-off-by: Marco Elver <elver@google.com> Acked-by: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
1 parent e675d25 commit 08cac60

1 file changed

Lines changed: 16 additions & 10 deletions

File tree

kernel/kcsan/core.c

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -301,9 +301,9 @@ static inline void reset_kcsan_skip(void)
301301
this_cpu_write(kcsan_skip, skip_count);
302302
}
303303

304-
static __always_inline bool kcsan_is_enabled(void)
304+
static __always_inline bool kcsan_is_enabled(struct kcsan_ctx *ctx)
305305
{
306-
return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0;
306+
return READ_ONCE(kcsan_enabled) && !ctx->disable_count;
307307
}
308308

309309
/* Introduce delay depending on context and configuration. */
@@ -353,25 +353,30 @@ static noinline void kcsan_found_watchpoint(const volatile void *ptr,
353353
atomic_long_t *watchpoint,
354354
long encoded_watchpoint)
355355
{
356+
struct kcsan_ctx *ctx = get_ctx();
356357
unsigned long flags;
357358
bool consumed;
358359

359-
if (!kcsan_is_enabled())
360+
/*
361+
* We know a watchpoint exists. Let's try to keep the race-window
362+
* between here and finally consuming the watchpoint below as small as
363+
* possible -- avoid unneccessarily complex code until consumed.
364+
*/
365+
366+
if (!kcsan_is_enabled(ctx))
360367
return;
361368

362369
/*
363370
* The access_mask check relies on value-change comparison. To avoid
364371
* reporting a race where e.g. the writer set up the watchpoint, but the
365372
* reader has access_mask!=0, we have to ignore the found watchpoint.
366373
*/
367-
if (get_ctx()->access_mask != 0)
374+
if (ctx->access_mask)
368375
return;
369376

370377
/*
371-
* Consume the watchpoint as soon as possible, to minimize the chances
372-
* of !consumed. Consuming the watchpoint must always be guarded by
373-
* kcsan_is_enabled() check, as otherwise we might erroneously
374-
* triggering reports when disabled.
378+
* Consuming the watchpoint must be guarded by kcsan_is_enabled() to
379+
* avoid erroneously triggering reports if the context is disabled.
375380
*/
376381
consumed = try_consume_watchpoint(watchpoint, encoded_watchpoint);
377382

@@ -409,6 +414,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type)
409414
unsigned long access_mask;
410415
enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE;
411416
unsigned long ua_flags = user_access_save();
417+
struct kcsan_ctx *ctx = get_ctx();
412418
unsigned long irq_flags = 0;
413419

414420
/*
@@ -417,7 +423,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type)
417423
*/
418424
reset_kcsan_skip();
419425

420-
if (!kcsan_is_enabled())
426+
if (!kcsan_is_enabled(ctx))
421427
goto out;
422428

423429
/*
@@ -489,7 +495,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type)
489495
* Re-read value, and check if it is as expected; if not, we infer a
490496
* racy access.
491497
*/
492-
access_mask = get_ctx()->access_mask;
498+
access_mask = ctx->access_mask;
493499
new = 0;
494500
switch (size) {
495501
case 1:

0 commit comments

Comments
 (0)