Skip to content

Commit ec841aa

Browse files
mrutland-armwilldeacon
mrutland-arm
authored and
willdeacon
committed
arm64: entry: handle all vectors with C
We have 16 architectural exception vectors, and depending on kernel configuration we handle 8 or 12 of these with C code, with the remaining 8 or 4 of these handled as special cases in the entry assembly. It would be nicer if the entry assembly were uniform for all exceptions, and we deferred any specific handling of the exceptions to C code. This way the entry assembly can be more easily templated without ifdeffery or special cases, and it's easier to modify the handling of these cases in future (e.g. to dump additional registers other context). This patch reworks the entry code so that we always have a C handler for every architectural exception vector, with the entry assembly being completely uniform. We now have to handle exceptions from EL1t and EL1h, and also have to handle exceptions from AArch32 even when the kernel is built without CONFIG_COMPAT. To make this clear and to simplify templating, we rename the top-level exception handlers with a consistent naming scheme: asm: <el+sp>_<regsize>_<type> c: <el+sp>_<regsize>_<type>_handler .. where: <el+sp> is `el1t`, `el1h`, or `el0t` <regsize> is `64` or `32` <type> is `sync`, `irq`, `fiq`, or `error` ... e.g. asm: el1h_64_sync c: el1h_64_sync_handler ... with lower-level handlers simply using "el1" and "compat" as today. For unexpected exceptions, this information is passed to __panic_unhandled(), so it can report the specific vector an unexpected exception was taken from, e.g. | Unhandled 64-bit el1t sync exception For vectors we never expect to enter legitimately, the C code is generated using a macro to avoid code duplication. The exceptions are handled via __panic_unhandled(), replacing bad_mode() (which is removed). The `kernel_ventry` and `entry_handler` assembly macros are updated to handle the new naming scheme. In theory it should be possible to generate the entry functions at the same time as the vectors using a single table, but this will require reworking the linker script to split the two into separate sections, so for now we have separate tables. Signed-off-by: Mark Rutland <mark.rutland@arm.com> Acked-by: Catalin Marinas <catalin.marinas@arm.com> Acked-by: Marc Zyngier <maz@kernel.org> Reviewed-by: Joey Gouly <joey.gouly@arm.com> Cc: James Morse <james.morse@arm.com> Cc: Will Deacon <will@kernel.org> Link: https://lore.kernel.org/r/20210607094624.34689-15-mark.rutland@arm.com Signed-off-by: Will Deacon <will@kernel.org>
1 parent a5b43a8 commit ec841aa

4 files changed

Lines changed: 93 additions & 138 deletions

File tree

‎arch/arm64/include/asm/exception.h‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,25 @@ static inline u32 disr_to_esr(u64 disr)
3131
return esr;
3232
}
3333

34-
asmlinkage void el1_sync_handler(struct pt_regs *regs);
35-
asmlinkage void el1_irq_handler(struct pt_regs *regs);
36-
asmlinkage void el1_fiq_handler(struct pt_regs *regs);
37-
asmlinkage void el1_error_handler(struct pt_regs *regs);
38-
asmlinkage void el0_sync_handler(struct pt_regs *regs);
39-
asmlinkage void el0_irq_handler(struct pt_regs *regs);
40-
asmlinkage void el0_fiq_handler(struct pt_regs *regs);
41-
asmlinkage void el0_error_handler(struct pt_regs *regs);
42-
asmlinkage void el0_sync_compat_handler(struct pt_regs *regs);
43-
asmlinkage void el0_irq_compat_handler(struct pt_regs *regs);
44-
asmlinkage void el0_fiq_compat_handler(struct pt_regs *regs);
45-
asmlinkage void el0_error_compat_handler(struct pt_regs *regs);
34+
asmlinkage void el1t_64_sync_handler(struct pt_regs *regs);
35+
asmlinkage void el1t_64_irq_handler(struct pt_regs *regs);
36+
asmlinkage void el1t_64_fiq_handler(struct pt_regs *regs);
37+
asmlinkage void el1t_64_error_handler(struct pt_regs *regs);
38+
39+
asmlinkage void el1h_64_sync_handler(struct pt_regs *regs);
40+
asmlinkage void el1h_64_irq_handler(struct pt_regs *regs);
41+
asmlinkage void el1h_64_fiq_handler(struct pt_regs *regs);
42+
asmlinkage void el1h_64_error_handler(struct pt_regs *regs);
43+
44+
asmlinkage void el0t_64_sync_handler(struct pt_regs *regs);
45+
asmlinkage void el0t_64_irq_handler(struct pt_regs *regs);
46+
asmlinkage void el0t_64_fiq_handler(struct pt_regs *regs);
47+
asmlinkage void el0t_64_error_handler(struct pt_regs *regs);
48+
49+
asmlinkage void el0t_32_sync_handler(struct pt_regs *regs);
50+
asmlinkage void el0t_32_irq_handler(struct pt_regs *regs);
51+
asmlinkage void el0t_32_fiq_handler(struct pt_regs *regs);
52+
asmlinkage void el0t_32_error_handler(struct pt_regs *regs);
4653

4754
asmlinkage void call_on_irq_stack(struct pt_regs *regs,
4855
void (*func)(struct pt_regs *));
@@ -53,7 +60,6 @@ void arm64_exit_nmi(struct pt_regs *regs);
5360
void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs);
5461
void do_undefinstr(struct pt_regs *regs);
5562
void do_bti(struct pt_regs *regs);
56-
asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr);
5763
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
5864
struct pt_regs *regs);
5965
void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs);

‎arch/arm64/kernel/entry-common.c‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -172,16 +172,11 @@ static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
172172
panic("Unhandled exception");
173173
}
174174

175-
asmlinkage void noinstr bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
176-
{
177-
const char *handler[] = {
178-
"Synchronous Abort",
179-
"IRQ",
180-
"FIQ",
181-
"Error"
182-
};
183-
184-
__panic_unhandled(regs, handler[reason], esr);
175+
#define UNHANDLED(el, regsize, vector) \
176+
asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
177+
{ \
178+
const char *desc = #regsize "-bit " #el " " #vector; \
179+
__panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
185180
}
186181

187182
#ifdef CONFIG_ARM64_ERRATUM_1463225
@@ -233,6 +228,11 @@ static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
233228
}
234229
#endif /* CONFIG_ARM64_ERRATUM_1463225 */
235230

231+
UNHANDLED(el1t, 64, sync)
232+
UNHANDLED(el1t, 64, irq)
233+
UNHANDLED(el1t, 64, fiq)
234+
UNHANDLED(el1t, 64, error)
235+
236236
static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
237237
{
238238
unsigned long far = read_sysreg(far_el1);
@@ -268,7 +268,7 @@ static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
268268
{
269269
enter_from_kernel_mode(regs);
270270
local_daif_inherit(regs);
271-
bad_mode(regs, 0, esr);
271+
__panic_unhandled(regs, "64-bit el1h sync", esr);
272272
local_daif_mask();
273273
exit_to_kernel_mode(regs);
274274
}
@@ -316,7 +316,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
316316
exit_to_kernel_mode(regs);
317317
}
318318

319-
asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
319+
asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
320320
{
321321
unsigned long esr = read_sysreg(esr_el1);
322322

@@ -370,17 +370,17 @@ static void noinstr el1_interrupt(struct pt_regs *regs,
370370
exit_el1_irq_or_nmi(regs);
371371
}
372372

373-
asmlinkage void noinstr el1_irq_handler(struct pt_regs *regs)
373+
asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
374374
{
375375
el1_interrupt(regs, handle_arch_irq);
376376
}
377377

378-
asmlinkage void noinstr el1_fiq_handler(struct pt_regs *regs)
378+
asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
379379
{
380380
el1_interrupt(regs, handle_arch_fiq);
381381
}
382382

383-
asmlinkage void noinstr el1_error_handler(struct pt_regs *regs)
383+
asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
384384
{
385385
unsigned long esr = read_sysreg(esr_el1);
386386

@@ -526,7 +526,7 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
526526
do_ptrauth_fault(regs, esr);
527527
}
528528

529-
asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)
529+
asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
530530
{
531531
unsigned long esr = read_sysreg(esr_el1);
532532

@@ -597,7 +597,7 @@ static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
597597
el0_interrupt(regs, handle_arch_irq);
598598
}
599599

600-
asmlinkage void noinstr el0_irq_handler(struct pt_regs *regs)
600+
asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
601601
{
602602
__el0_irq_handler_common(regs);
603603
}
@@ -607,7 +607,7 @@ static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
607607
el0_interrupt(regs, handle_arch_fiq);
608608
}
609609

610-
asmlinkage void noinstr el0_fiq_handler(struct pt_regs *regs)
610+
asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
611611
{
612612
__el0_fiq_handler_common(regs);
613613
}
@@ -624,7 +624,7 @@ static void __el0_error_handler_common(struct pt_regs *regs)
624624
local_daif_restore(DAIF_PROCCTX);
625625
}
626626

627-
asmlinkage void noinstr el0_error_handler(struct pt_regs *regs)
627+
asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
628628
{
629629
__el0_error_handler_common(regs);
630630
}
@@ -644,7 +644,7 @@ static void noinstr el0_svc_compat(struct pt_regs *regs)
644644
do_el0_svc_compat(regs);
645645
}
646646

647-
asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
647+
asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
648648
{
649649
unsigned long esr = read_sysreg(esr_el1);
650650

@@ -688,18 +688,23 @@ asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
688688
}
689689
}
690690

691-
asmlinkage void noinstr el0_irq_compat_handler(struct pt_regs *regs)
691+
asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
692692
{
693693
__el0_irq_handler_common(regs);
694694
}
695695

696-
asmlinkage void noinstr el0_fiq_compat_handler(struct pt_regs *regs)
696+
asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)
697697
{
698698
__el0_fiq_handler_common(regs);
699699
}
700700

701-
asmlinkage void noinstr el0_error_compat_handler(struct pt_regs *regs)
701+
asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)
702702
{
703703
__el0_error_handler_common(regs);
704704
}
705+
#else /* CONFIG_COMPAT */
706+
UNHANDLED(el0t, 32, sync)
707+
UNHANDLED(el0t, 32, irq)
708+
UNHANDLED(el0t, 32, fiq)
709+
UNHANDLED(el0t, 32, error)
705710
#endif /* CONFIG_COMPAT */

‎arch/arm64/kernel/entry.S‎

Lines changed: 45 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -45,16 +45,7 @@
4545
.endr
4646
.endm
4747

48-
/*
49-
* Bad Abort numbers
50-
*-----------------
51-
*/
52-
#define BAD_SYNC 0
53-
#define BAD_IRQ 1
54-
#define BAD_FIQ 2
55-
#define BAD_ERROR 3
56-
57-
.macro kernel_ventry, el:req, regsize:req, label:req
48+
.macro kernel_ventry, el:req, ht:req, regsize:req, label:req
5849
.align 7
5950
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
6051
.if \el == 0
@@ -81,7 +72,7 @@ alternative_else_nop_endif
8172
tbnz x0, #THREAD_SHIFT, 0f
8273
sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
8374
sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
84-
b el\()\el\()_\label
75+
b el\el\ht\()_\regsize\()_\label
8576

8677
0:
8778
/*
@@ -113,7 +104,7 @@ alternative_else_nop_endif
113104
sub sp, sp, x0
114105
mrs x0, tpidrro_el0
115106
#endif
116-
b el\()\el\()_\label
107+
b el\el\ht\()_\regsize\()_\label
117108
.endm
118109

119110
.macro tramp_alias, dst, sym
@@ -504,32 +495,25 @@ tsk .req x28 // current thread_info
504495

505496
.align 11
506497
SYM_CODE_START(vectors)
507-
kernel_ventry 1, 64, sync_invalid // Synchronous EL1t
508-
kernel_ventry 1, 64, irq_invalid // IRQ EL1t
509-
kernel_ventry 1, 64, fiq_invalid // FIQ EL1t
510-
kernel_ventry 1, 64, error_invalid // Error EL1t
511-
512-
kernel_ventry 1, 64, sync // Synchronous EL1h
513-
kernel_ventry 1, 64, irq // IRQ EL1h
514-
kernel_ventry 1, 64, fiq // FIQ EL1h
515-
kernel_ventry 1, 64, error // Error EL1h
516-
517-
kernel_ventry 0, 64, sync // Synchronous 64-bit EL0
518-
kernel_ventry 0, 64, irq // IRQ 64-bit EL0
519-
kernel_ventry 0, 64, fiq // FIQ 64-bit EL0
520-
kernel_ventry 0, 64, error // Error 64-bit EL0
521-
522-
#ifdef CONFIG_COMPAT
523-
kernel_ventry 0, 32, sync_compat // Synchronous 32-bit EL0
524-
kernel_ventry 0, 32, irq_compat // IRQ 32-bit EL0
525-
kernel_ventry 0, 32, fiq_compat // FIQ 32-bit EL0
526-
kernel_ventry 0, 32, error_compat // Error 32-bit EL0
527-
#else
528-
kernel_ventry 0, 32, sync_invalid // Synchronous 32-bit EL0
529-
kernel_ventry 0, 32, irq_invalid // IRQ 32-bit EL0
530-
kernel_ventry 0, 32, fiq_invalid // FIQ 32-bit EL0
531-
kernel_ventry 0, 32, error_invalid // Error 32-bit EL0
532-
#endif
498+
kernel_ventry 1, t, 64, sync // Synchronous EL1t
499+
kernel_ventry 1, t, 64, irq // IRQ EL1t
500+
kernel_ventry 1, t, 64, fiq // FIQ EL1h
501+
kernel_ventry 1, t, 64, error // Error EL1t
502+
503+
kernel_ventry 1, h, 64, sync // Synchronous EL1h
504+
kernel_ventry 1, h, 64, irq // IRQ EL1h
505+
kernel_ventry 1, h, 64, fiq // FIQ EL1h
506+
kernel_ventry 1, h, 64, error // Error EL1h
507+
508+
kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0
509+
kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0
510+
kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0
511+
kernel_ventry 0, t, 64, error // Error 64-bit EL0
512+
513+
kernel_ventry 0, t, 32, sync // Synchronous 32-bit EL0
514+
kernel_ventry 0, t, 32, irq // IRQ 32-bit EL0
515+
kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0
516+
kernel_ventry 0, t, 32, error // Error 32-bit EL0
533517
SYM_CODE_END(vectors)
534518

535519
#ifdef CONFIG_VMAP_STACK
@@ -560,82 +544,42 @@ __bad_stack:
560544
ASM_BUG()
561545
#endif /* CONFIG_VMAP_STACK */
562546

563-
/*
564-
* Invalid mode handlers
565-
*/
566-
.macro inv_entry, el, reason, regsize = 64
567-
kernel_entry \el, \regsize
568-
mov x0, sp
569-
mov x1, #\reason
570-
mrs x2, esr_el1
571-
bl bad_mode
572-
ASM_BUG()
573-
.endm
574-
575-
SYM_CODE_START_LOCAL(el0_sync_invalid)
576-
inv_entry 0, BAD_SYNC
577-
SYM_CODE_END(el0_sync_invalid)
578-
579-
SYM_CODE_START_LOCAL(el0_irq_invalid)
580-
inv_entry 0, BAD_IRQ
581-
SYM_CODE_END(el0_irq_invalid)
582-
583-
SYM_CODE_START_LOCAL(el0_fiq_invalid)
584-
inv_entry 0, BAD_FIQ
585-
SYM_CODE_END(el0_fiq_invalid)
586-
587-
SYM_CODE_START_LOCAL(el0_error_invalid)
588-
inv_entry 0, BAD_ERROR
589-
SYM_CODE_END(el0_error_invalid)
590547

591-
SYM_CODE_START_LOCAL(el1_sync_invalid)
592-
inv_entry 1, BAD_SYNC
593-
SYM_CODE_END(el1_sync_invalid)
594-
595-
SYM_CODE_START_LOCAL(el1_irq_invalid)
596-
inv_entry 1, BAD_IRQ
597-
SYM_CODE_END(el1_irq_invalid)
598-
599-
SYM_CODE_START_LOCAL(el1_fiq_invalid)
600-
inv_entry 1, BAD_FIQ
601-
SYM_CODE_END(el1_fiq_invalid)
602-
603-
SYM_CODE_START_LOCAL(el1_error_invalid)
604-
inv_entry 1, BAD_ERROR
605-
SYM_CODE_END(el1_error_invalid)
606-
607-
.macro entry_handler el:req, regsize:req, label:req
608-
SYM_CODE_START_LOCAL(el\el\()_\label)
548+
.macro entry_handler el:req, ht:req, regsize:req, label:req
549+
SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label)
609550
kernel_entry \el, \regsize
610551
mov x0, sp
611-
bl el\el\()_\label\()_handler
552+
bl el\el\ht\()_\regsize\()_\label\()_handler
612553
.if \el == 0
613554
b ret_to_user
614555
.else
615556
b ret_to_kernel
616557
.endif
617-
SYM_CODE_END(el\el\()_\label)
558+
SYM_CODE_END(el\el\ht\()_\regsize\()_\label)
618559
.endm
619560

620561
/*
621562
* Early exception handlers
622563
*/
623-
entry_handler 1, 64, sync
624-
entry_handler 1, 64, irq
625-
entry_handler 1, 64, fiq
626-
entry_handler 1, 64, error
627-
628-
entry_handler 0, 64, sync
629-
entry_handler 0, 64, irq
630-
entry_handler 0, 64, fiq
631-
entry_handler 0, 64, error
632-
633-
#ifdef CONFIG_COMPAT
634-
entry_handler 0, 32, sync_compat
635-
entry_handler 0, 32, irq_compat
636-
entry_handler 0, 32, fiq_compat
637-
entry_handler 0, 32, error_compat
638-
#endif
564+
entry_handler 1, t, 64, sync
565+
entry_handler 1, t, 64, irq
566+
entry_handler 1, t, 64, fiq
567+
entry_handler 1, t, 64, error
568+
569+
entry_handler 1, h, 64, sync
570+
entry_handler 1, h, 64, irq
571+
entry_handler 1, h, 64, fiq
572+
entry_handler 1, h, 64, error
573+
574+
entry_handler 0, t, 64, sync
575+
entry_handler 0, t, 64, irq
576+
entry_handler 0, t, 64, fiq
577+
entry_handler 0, t, 64, error
578+
579+
entry_handler 0, t, 32, sync
580+
entry_handler 0, t, 32, irq
581+
entry_handler 0, t, 32, fiq
582+
entry_handler 0, t, 32, error
639583

640584
SYM_CODE_START_LOCAL(ret_to_kernel)
641585
kernel_exit 1

‎arch/arm64/kernel/traps.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -745,7 +745,7 @@ const char *esr_get_class_string(u32 esr)
745745

746746
/*
747747
* bad_el0_sync handles unexpected, but potentially recoverable synchronous
748-
* exceptions taken from EL0. Unlike bad_mode, this returns.
748+
* exceptions taken from EL0.
749749
*/
750750
void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
751751
{

0 commit comments

Comments
 (0)