Skip to content

Commit ba1f2b2

Browse files
author
Peter Zijlstra
committed
x86/entry: Fix NMI vs IRQ state tracking
While the nmi_enter() users did trace_hardirqs_{off_prepare,on_finish}() there was no matching lockdep_hardirqs_*() calls to complete the picture. Introduce idtentry_{enter,exit}_nmi() to enable proper IRQ state tracking across the NMIs. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Reviewed-by: Ingo Molnar <mingo@kernel.org> Link: https://lkml.kernel.org/r/20200623083721.216740948@infradead.org
1 parent 859d069 commit ba1f2b2

5 files changed

Lines changed: 70 additions & 29 deletions

File tree

‎arch/x86/entry/common.c‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -592,7 +592,7 @@ SYSCALL_DEFINE0(ni_syscall)
592592
* The return value must be fed into the state argument of
593593
* idtentry_exit().
594594
*/
595-
idtentry_state_t noinstr idtentry_enter(struct pt_regs *regs)
595+
noinstr idtentry_state_t idtentry_enter(struct pt_regs *regs)
596596
{
597597
idtentry_state_t ret = {
598598
.exit_rcu = false,
@@ -687,7 +687,7 @@ static void idtentry_exit_cond_resched(struct pt_regs *regs, bool may_sched)
687687
* Counterpart to idtentry_enter(). The return value of the entry
688688
* function must be fed into the @state argument.
689689
*/
690-
void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
690+
noinstr void idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
691691
{
692692
lockdep_assert_irqs_disabled();
693693

@@ -731,7 +731,7 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
731731
* Invokes enter_from_user_mode() to establish the proper context for
732732
* NOHZ_FULL. Otherwise scheduling on exit would not be possible.
733733
*/
734-
void noinstr idtentry_enter_user(struct pt_regs *regs)
734+
noinstr void idtentry_enter_user(struct pt_regs *regs)
735735
{
736736
check_user_regs(regs);
737737
enter_from_user_mode();
@@ -749,13 +749,47 @@ void noinstr idtentry_enter_user(struct pt_regs *regs)
749749
*
750750
* Counterpart to idtentry_enter_user().
751751
*/
752-
void noinstr idtentry_exit_user(struct pt_regs *regs)
752+
noinstr void idtentry_exit_user(struct pt_regs *regs)
753753
{
754754
lockdep_assert_irqs_disabled();
755755

756756
prepare_exit_to_usermode(regs);
757757
}
758758

759+
noinstr bool idtentry_enter_nmi(struct pt_regs *regs)
760+
{
761+
bool irq_state = lockdep_hardirqs_enabled(current);
762+
763+
__nmi_enter();
764+
lockdep_hardirqs_off(CALLER_ADDR0);
765+
lockdep_hardirq_enter();
766+
rcu_nmi_enter();
767+
768+
instrumentation_begin();
769+
trace_hardirqs_off_finish();
770+
ftrace_nmi_enter();
771+
instrumentation_end();
772+
773+
return irq_state;
774+
}
775+
776+
noinstr void idtentry_exit_nmi(struct pt_regs *regs, bool restore)
777+
{
778+
instrumentation_begin();
779+
ftrace_nmi_exit();
780+
if (restore) {
781+
trace_hardirqs_on_prepare();
782+
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
783+
}
784+
instrumentation_end();
785+
786+
rcu_nmi_exit();
787+
lockdep_hardirq_exit();
788+
if (restore)
789+
lockdep_hardirqs_on(CALLER_ADDR0);
790+
__nmi_exit();
791+
}
792+
759793
#ifdef CONFIG_XEN_PV
760794
#ifndef CONFIG_PREEMPTION
761795
/*

‎arch/x86/include/asm/idtentry.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ typedef struct idtentry_state {
2020
idtentry_state_t idtentry_enter(struct pt_regs *regs);
2121
void idtentry_exit(struct pt_regs *regs, idtentry_state_t state);
2222

23+
bool idtentry_enter_nmi(struct pt_regs *regs);
24+
void idtentry_exit_nmi(struct pt_regs *regs, bool irq_state);
25+
2326
/**
2427
* DECLARE_IDTENTRY - Declare functions for simple IDT entry points
2528
* No error code pushed by hardware

‎arch/x86/kernel/nmi.c‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,6 @@ static noinstr void default_do_nmi(struct pt_regs *regs)
330330
__this_cpu_write(last_nmi_rip, regs->ip);
331331

332332
instrumentation_begin();
333-
trace_hardirqs_off_finish();
334333

335334
handled = nmi_handle(NMI_LOCAL, regs);
336335
__this_cpu_add(nmi_stats.normal, handled);
@@ -417,8 +416,6 @@ static noinstr void default_do_nmi(struct pt_regs *regs)
417416
unknown_nmi_error(reason, regs);
418417

419418
out:
420-
if (regs->flags & X86_EFLAGS_IF)
421-
trace_hardirqs_on_prepare();
422419
instrumentation_end();
423420
}
424421

@@ -478,6 +475,8 @@ static DEFINE_PER_CPU(unsigned long, nmi_dr7);
478475

479476
DEFINE_IDTENTRY_RAW(exc_nmi)
480477
{
478+
bool irq_state;
479+
481480
if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id()))
482481
return;
483482

@@ -491,14 +490,14 @@ DEFINE_IDTENTRY_RAW(exc_nmi)
491490

492491
this_cpu_write(nmi_dr7, local_db_save());
493492

494-
nmi_enter();
493+
irq_state = idtentry_enter_nmi(regs);
495494

496495
inc_irq_stat(__nmi_count);
497496

498497
if (!ignore_nmis)
499498
default_do_nmi(regs);
500499

501-
nmi_exit();
500+
idtentry_exit_nmi(regs, irq_state);
502501

503502
local_db_restore(this_cpu_read(nmi_dr7));
504503

‎arch/x86/kernel/traps.c‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ DEFINE_IDTENTRY_DF(exc_double_fault)
403403
}
404404
#endif
405405

406-
nmi_enter();
406+
idtentry_enter_nmi(regs);
407407
instrumentation_begin();
408408
notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
409409

@@ -649,15 +649,12 @@ DEFINE_IDTENTRY_RAW(exc_int3)
649649
instrumentation_end();
650650
idtentry_exit_user(regs);
651651
} else {
652-
nmi_enter();
652+
bool irq_state = idtentry_enter_nmi(regs);
653653
instrumentation_begin();
654-
trace_hardirqs_off_finish();
655654
if (!do_int3(regs))
656655
die("int3", regs, 0);
657-
if (regs->flags & X86_EFLAGS_IF)
658-
trace_hardirqs_on_prepare();
659656
instrumentation_end();
660-
nmi_exit();
657+
idtentry_exit_nmi(regs, irq_state);
661658
}
662659
}
663660

@@ -865,9 +862,8 @@ static void handle_debug(struct pt_regs *regs, unsigned long dr6, bool user)
865862
static __always_inline void exc_debug_kernel(struct pt_regs *regs,
866863
unsigned long dr6)
867864
{
868-
nmi_enter();
865+
bool irq_state = idtentry_enter_nmi(regs);
869866
instrumentation_begin();
870-
trace_hardirqs_off_finish();
871867

872868
/*
873869
* If something gets miswired and we end up here for a user mode
@@ -884,10 +880,8 @@ static __always_inline void exc_debug_kernel(struct pt_regs *regs,
884880

885881
handle_debug(regs, dr6, false);
886882

887-
if (regs->flags & X86_EFLAGS_IF)
888-
trace_hardirqs_on_prepare();
889883
instrumentation_end();
890-
nmi_exit();
884+
idtentry_exit_nmi(regs, irq_state);
891885
}
892886

893887
static __always_inline void exc_debug_user(struct pt_regs *regs,
@@ -903,6 +897,7 @@ static __always_inline void exc_debug_user(struct pt_regs *regs,
903897
instrumentation_begin();
904898

905899
handle_debug(regs, dr6, true);
900+
906901
instrumentation_end();
907902
idtentry_exit_user(regs);
908903
}

‎include/linux/hardirq.h‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -111,32 +111,42 @@ extern void rcu_nmi_exit(void);
111111
/*
112112
* nmi_enter() can nest up to 15 times; see NMI_BITS.
113113
*/
114-
#define nmi_enter() \
114+
#define __nmi_enter() \
115115
do { \
116+
lockdep_off(); \
116117
arch_nmi_enter(); \
117118
printk_nmi_enter(); \
118-
lockdep_off(); \
119119
BUG_ON(in_nmi() == NMI_MASK); \
120120
__preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
121-
rcu_nmi_enter(); \
121+
} while (0)
122+
123+
#define nmi_enter() \
124+
do { \
125+
__nmi_enter(); \
122126
lockdep_hardirq_enter(); \
127+
rcu_nmi_enter(); \
123128
instrumentation_begin(); \
124129
ftrace_nmi_enter(); \
125130
instrumentation_end(); \
126131
} while (0)
127132

133+
#define __nmi_exit() \
134+
do { \
135+
BUG_ON(!in_nmi()); \
136+
__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
137+
printk_nmi_exit(); \
138+
arch_nmi_exit(); \
139+
lockdep_on(); \
140+
} while (0)
141+
128142
#define nmi_exit() \
129143
do { \
130144
instrumentation_begin(); \
131145
ftrace_nmi_exit(); \
132146
instrumentation_end(); \
133-
lockdep_hardirq_exit(); \
134147
rcu_nmi_exit(); \
135-
BUG_ON(!in_nmi()); \
136-
__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
137-
lockdep_on(); \
138-
printk_nmi_exit(); \
139-
arch_nmi_exit(); \
148+
lockdep_hardirq_exit(); \
149+
__nmi_exit(); \
140150
} while (0)
141151

142152
#endif /* LINUX_HARDIRQ_H */

0 commit comments

Comments
 (0)