Skip to content

Commit b5e13f3

Browse files
palmer-dabbeltpalmer-dabbelt
committed
Merge patch series "riscv: Add independent irq/softirq stacks support"
guoren@kernel.org <guoren@kernel.org> says: From: Guo Ren <guoren@linux.alibaba.com> This patch series adds independent irq/softirq stacks to decrease the press of the thread stack. Also, add a thread STACK_SIZE config for users to adjust the proper size during compile time. * b4-shazam-merge: riscv: stack: Add config of thread stack size riscv: stack: Support HAVE_SOFTIRQ_ON_OWN_STACK riscv: stack: Support HAVE_IRQ_EXIT_ON_IRQ_STACK Link: https://lore.kernel.org/r/20230614013018.2168426-1-guoren@kernel.org Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
2 parents 42b8944 + a7555f6 commit b5e13f3

5 files changed

Lines changed: 153 additions & 13 deletions

File tree

‎arch/riscv/Kconfig‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -591,6 +591,25 @@ config FPU
591591

592592
If you don't know what to do here, say Y.
593593

594+
config IRQ_STACKS
595+
bool "Independent irq & softirq stacks" if EXPERT
596+
default y
597+
select HAVE_IRQ_EXIT_ON_IRQ_STACK
598+
select HAVE_SOFTIRQ_ON_OWN_STACK
599+
help
600+
Add independent irq & softirq stacks for percpu to prevent kernel stack
601+
overflows. We may save some memory footprint by disabling IRQ_STACKS.
602+
603+
config THREAD_SIZE_ORDER
604+
int "Kernel stack size (in power-of-two numbers of page size)" if VMAP_STACK && EXPERT
605+
range 0 4
606+
default 1 if 32BIT && !KASAN
607+
default 3 if 64BIT && KASAN
608+
default 2
609+
help
610+
Specify the Pages of thread stack size (from 4KB to 64KB), which also
611+
affects irq stack size, which is equal to thread stack size.
612+
594613
endmenu # "Platform type"
595614

596615
menu "Kernel features"

‎arch/riscv/include/asm/irq_stack.h‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
3+
#ifndef _ASM_RISCV_IRQ_STACK_H
4+
#define _ASM_RISCV_IRQ_STACK_H
5+
6+
#include <linux/bug.h>
7+
#include <linux/gfp.h>
8+
#include <linux/kconfig.h>
9+
#include <linux/vmalloc.h>
10+
#include <linux/pgtable.h>
11+
#include <asm/thread_info.h>
12+
13+
DECLARE_PER_CPU(ulong *, irq_stack_ptr);
14+
15+
#ifdef CONFIG_VMAP_STACK
16+
/*
17+
* To ensure that VMAP'd stack overflow detection works correctly, all VMAP'd
18+
* stacks need to have the same alignment.
19+
*/
20+
static inline unsigned long *arch_alloc_vmap_stack(size_t stack_size, int node)
21+
{
22+
void *p;
23+
24+
p = __vmalloc_node(stack_size, THREAD_ALIGN, THREADINFO_GFP, node,
25+
__builtin_return_address(0));
26+
return kasan_reset_tag(p);
27+
}
28+
#endif /* CONFIG_VMAP_STACK */
29+
30+
#endif /* _ASM_RISCV_IRQ_STACK_H */

‎arch/riscv/include/asm/thread_info.h‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,8 @@
1111
#include <asm/page.h>
1212
#include <linux/const.h>
1313

14-
#ifdef CONFIG_KASAN
15-
#define KASAN_STACK_ORDER 1
16-
#else
17-
#define KASAN_STACK_ORDER 0
18-
#endif
19-
2014
/* thread information allocation */
21-
#ifdef CONFIG_64BIT
22-
#define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER)
23-
#else
24-
#define THREAD_SIZE_ORDER (1 + KASAN_STACK_ORDER)
25-
#endif
15+
#define THREAD_SIZE_ORDER CONFIG_THREAD_SIZE_ORDER
2616
#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
2717

2818
/*
@@ -40,6 +30,8 @@
4030
#define OVERFLOW_STACK_SIZE SZ_4K
4131
#define SHADOW_OVERFLOW_STACK_SIZE (1024)
4232

33+
#define IRQ_STACK_SIZE THREAD_SIZE
34+
4335
#ifndef __ASSEMBLY__
4436

4537
extern long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE / sizeof(long)];

‎arch/riscv/kernel/irq.c‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
#include <linux/module.h>
1212
#include <linux/seq_file.h>
1313
#include <asm/sbi.h>
14+
#include <asm/smp.h>
15+
#include <asm/softirq_stack.h>
16+
#include <asm/stacktrace.h>
1417

1518
static struct fwnode_handle *(*__get_intc_node)(void);
1619

@@ -28,6 +31,70 @@ struct fwnode_handle *riscv_get_intc_hwnode(void)
2831
}
2932
EXPORT_SYMBOL_GPL(riscv_get_intc_hwnode);
3033

34+
#ifdef CONFIG_IRQ_STACKS
35+
#include <asm/irq_stack.h>
36+
37+
DEFINE_PER_CPU(ulong *, irq_stack_ptr);
38+
39+
#ifdef CONFIG_VMAP_STACK
40+
static void init_irq_stacks(void)
41+
{
42+
int cpu;
43+
ulong *p;
44+
45+
for_each_possible_cpu(cpu) {
46+
p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, cpu_to_node(cpu));
47+
per_cpu(irq_stack_ptr, cpu) = p;
48+
}
49+
}
50+
#else
51+
/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
52+
DEFINE_PER_CPU_ALIGNED(ulong [IRQ_STACK_SIZE/sizeof(ulong)], irq_stack);
53+
54+
static void init_irq_stacks(void)
55+
{
56+
int cpu;
57+
58+
for_each_possible_cpu(cpu)
59+
per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
60+
}
61+
#endif /* CONFIG_VMAP_STACK */
62+
63+
#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK
64+
void do_softirq_own_stack(void)
65+
{
66+
#ifdef CONFIG_IRQ_STACKS
67+
if (on_thread_stack()) {
68+
ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
69+
+ IRQ_STACK_SIZE/sizeof(ulong);
70+
__asm__ __volatile(
71+
"addi sp, sp, -"RISCV_SZPTR "\n"
72+
REG_S" ra, (sp) \n"
73+
"addi sp, sp, -"RISCV_SZPTR "\n"
74+
REG_S" s0, (sp) \n"
75+
"addi s0, sp, 2*"RISCV_SZPTR "\n"
76+
"move sp, %[sp] \n"
77+
"call __do_softirq \n"
78+
"addi sp, s0, -2*"RISCV_SZPTR"\n"
79+
REG_L" s0, (sp) \n"
80+
"addi sp, sp, "RISCV_SZPTR "\n"
81+
REG_L" ra, (sp) \n"
82+
"addi sp, sp, "RISCV_SZPTR "\n"
83+
:
84+
: [sp] "r" (sp)
85+
: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
86+
"t0", "t1", "t2", "t3", "t4", "t5", "t6",
87+
"memory");
88+
} else
89+
#endif
90+
__do_softirq();
91+
}
92+
#endif /* CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK */
93+
94+
#else
95+
static void init_irq_stacks(void) {}
96+
#endif /* CONFIG_IRQ_STACKS */
97+
3198
int arch_show_interrupts(struct seq_file *p, int prec)
3299
{
33100
show_ipi_stats(p, prec);
@@ -36,6 +103,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
36103

37104
void __init init_IRQ(void)
38105
{
106+
init_irq_stacks();
39107
irqchip_init();
40108
if (!handle_arch_irq)
41109
panic("No interrupt controller found.");

‎arch/riscv/kernel/traps.c‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <asm/syscall.h>
2828
#include <asm/thread_info.h>
2929
#include <asm/vector.h>
30+
#include <asm/irq_stack.h>
3031

3132
int show_unhandled_signals = 1;
3233

@@ -327,16 +328,46 @@ asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
327328
}
328329
#endif
329330

330-
asmlinkage __visible noinstr void do_irq(struct pt_regs *regs)
331+
static void noinstr handle_riscv_irq(struct pt_regs *regs)
331332
{
332333
struct pt_regs *old_regs;
333-
irqentry_state_t state = irqentry_enter(regs);
334334

335335
irq_enter_rcu();
336336
old_regs = set_irq_regs(regs);
337337
handle_arch_irq(regs);
338338
set_irq_regs(old_regs);
339339
irq_exit_rcu();
340+
}
341+
342+
asmlinkage void noinstr do_irq(struct pt_regs *regs)
343+
{
344+
irqentry_state_t state = irqentry_enter(regs);
345+
#ifdef CONFIG_IRQ_STACKS
346+
if (on_thread_stack()) {
347+
ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
348+
+ IRQ_STACK_SIZE/sizeof(ulong);
349+
__asm__ __volatile(
350+
"addi sp, sp, -"RISCV_SZPTR "\n"
351+
REG_S" ra, (sp) \n"
352+
"addi sp, sp, -"RISCV_SZPTR "\n"
353+
REG_S" s0, (sp) \n"
354+
"addi s0, sp, 2*"RISCV_SZPTR "\n"
355+
"move sp, %[sp] \n"
356+
"move a0, %[regs] \n"
357+
"call handle_riscv_irq \n"
358+
"addi sp, s0, -2*"RISCV_SZPTR"\n"
359+
REG_L" s0, (sp) \n"
360+
"addi sp, sp, "RISCV_SZPTR "\n"
361+
REG_L" ra, (sp) \n"
362+
"addi sp, sp, "RISCV_SZPTR "\n"
363+
:
364+
: [sp] "r" (sp), [regs] "r" (regs)
365+
: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
366+
"t0", "t1", "t2", "t3", "t4", "t5", "t6",
367+
"memory");
368+
} else
369+
#endif
370+
handle_riscv_irq(regs);
340371

341372
irqentry_exit(regs, state);
342373
}

0 commit comments

Comments
 (0)