Skip to content

Commit 005b2a9

Browse files
torvaldstorvalds
committed
Merge tag 'tif-task_work.arch-2020-12-14' of git://git.kernel.dk/linux-block
Pull TIF_NOTIFY_SIGNAL updates from Jens Axboe: "This sits on top of of the core entry/exit and x86 entry branch from the tip tree, which contains the generic and x86 parts of this work. Here we convert the rest of the archs to support TIF_NOTIFY_SIGNAL. With that done, we can get rid of JOBCTL_TASK_WORK from task_work and signal.c, and also remove a deadlock work-around in io_uring around knowing that signal based task_work waking is invoked with the sighand wait queue head lock. The motivation for this work is to decouple signal notify based task_work, of which io_uring is a heavy user of, from sighand. The sighand lock becomes a huge contention point, particularly for threaded workloads where it's shared between threads. Even outside of threaded applications it's slower than it needs to be. Roman Gershman <romger@amazon.com> reported that his networked workload dropped from 1.6M QPS at 80% CPU to 1.0M QPS at 100% CPU after io_uring was changed to use TIF_NOTIFY_SIGNAL. The time was all spent hammering on the sighand lock, showing 57% of the CPU time there [1]. There are further cleanups possible on top of this. One example is TIF_PATCH_PENDING, where a patch already exists to use TIF_NOTIFY_SIGNAL instead. Hopefully this will also lead to more consolidation, but the work stands on its own as well" [1] axboe/liburing#215 * tag 'tif-task_work.arch-2020-12-14' of git://git.kernel.dk/linux-block: (28 commits) io_uring: remove 'twa_signal_ok' deadlock work-around kernel: remove checking for TIF_NOTIFY_SIGNAL signal: kill JOBCTL_TASK_WORK io_uring: JOBCTL_TASK_WORK is no longer used by task_work task_work: remove legacy TWA_SIGNAL path sparc: add support for TIF_NOTIFY_SIGNAL riscv: add support for TIF_NOTIFY_SIGNAL nds32: add support for TIF_NOTIFY_SIGNAL ia64: add support for TIF_NOTIFY_SIGNAL h8300: add support for TIF_NOTIFY_SIGNAL c6x: add support for TIF_NOTIFY_SIGNAL alpha: add support for TIF_NOTIFY_SIGNAL xtensa: add support for TIF_NOTIFY_SIGNAL arm: add support for TIF_NOTIFY_SIGNAL microblaze: add support for TIF_NOTIFY_SIGNAL hexagon: add support for TIF_NOTIFY_SIGNAL csky: add support for TIF_NOTIFY_SIGNAL openrisc: add support for TIF_NOTIFY_SIGNAL sh: add support for TIF_NOTIFY_SIGNAL um: add support for TIF_NOTIFY_SIGNAL ...
2 parents 5ee863b + 355fb9e commit 005b2a9

61 files changed

Lines changed: 124 additions & 138 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎arch/alpha/include/asm/thread_info.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ register struct thread_info *__current_thread_info __asm__("$8");
6262
#define TIF_SIGPENDING 2 /* signal pending */
6363
#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
6464
#define TIF_SYSCALL_AUDIT 4 /* syscall audit active */
65+
#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */
6566
#define TIF_DIE_IF_KERNEL 9 /* dik recursion lock */
6667
#define TIF_MEMDIE 13 /* is terminating due to OOM killer */
6768
#define TIF_POLLING_NRFLAG 14 /* idle is polling for TIF_NEED_RESCHED */
@@ -71,6 +72,7 @@ register struct thread_info *__current_thread_info __asm__("$8");
7172
#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED)
7273
#define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME)
7374
#define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT)
75+
#define _TIF_NOTIFY_SIGNAL (1<<TIF_NOTIFY_SIGNAL)
7476
#define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG)
7577

7678
/* Work to do on interrupt/exception return. */

‎arch/alpha/kernel/entry.S‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ $ret_success:
544544
.align 4
545545
.type work_pending, @function
546546
work_pending:
547-
and $17, _TIF_NOTIFY_RESUME | _TIF_SIGPENDING, $2
547+
and $17, _TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL, $2
548548
bne $2, $work_notifysig
549549

550550
$work_resched:

‎arch/alpha/kernel/signal.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -527,7 +527,7 @@ do_work_pending(struct pt_regs *regs, unsigned long thread_flags,
527527
schedule();
528528
} else {
529529
local_irq_enable();
530-
if (thread_flags & _TIF_SIGPENDING) {
530+
if (thread_flags & (_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL)) {
531531
do_signal(regs, r0, r19);
532532
r0 = 0;
533533
} else {

‎arch/arc/include/asm/thread_info.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ static inline __attribute_const__ struct thread_info *current_thread_info(void)
7979
#define TIF_SIGPENDING 2 /* signal pending */
8080
#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
8181
#define TIF_SYSCALL_AUDIT 4 /* syscall auditing active */
82+
#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */
8283
#define TIF_SYSCALL_TRACE 15 /* syscall trace active */
8384

8485
/* true if poll_idle() is polling TIF_NEED_RESCHED */
@@ -89,11 +90,12 @@ static inline __attribute_const__ struct thread_info *current_thread_info(void)
8990
#define _TIF_SIGPENDING (1<<TIF_SIGPENDING)
9091
#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED)
9192
#define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT)
93+
#define _TIF_NOTIFY_SIGNAL (1<<TIF_NOTIFY_SIGNAL)
9294
#define _TIF_MEMDIE (1<<TIF_MEMDIE)
9395

9496
/* work to do on interrupt/exception return */
9597
#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
96-
_TIF_NOTIFY_RESUME)
98+
_TIF_NOTIFY_RESUME | _TIF_NOTIFY_SIGNAL)
9799

98100
/*
99101
* _TIF_ALLWORK_MASK includes SYSCALL_TRACE, but we don't need it.

‎arch/arc/kernel/entry.S‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,8 @@ resume_user_mode_begin:
307307
mov r0, sp ; pt_regs for arg to do_signal()/do_notify_resume()
308308

309309
GET_CURR_THR_INFO_FLAGS r9
310-
bbit0 r9, TIF_SIGPENDING, .Lchk_notify_resume
310+
and.f 0, r9, TIF_SIGPENDING|TIF_NOTIFY_SIGNAL
311+
bz .Lchk_notify_resume
311312

312313
; Normal Trap/IRQ entry only saves Scratch (caller-saved) regs
313314
; in pt_reg since the "C" ABI (kernel code) will automatically

‎arch/arc/kernel/signal.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,7 @@ void do_signal(struct pt_regs *regs)
362362

363363
restart_scall = in_syscall(regs) && syscall_restartable(regs);
364364

365-
if (get_signal(&ksig)) {
365+
if (test_thread_flag(TIF_SIGPENDING) && get_signal(&ksig)) {
366366
if (restart_scall) {
367367
arc_restart_syscall(&ksig.ka, regs);
368368
syscall_wont_restart(regs); /* No more restarts */

‎arch/arm/include/asm/thread_info.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,8 @@ extern int vfp_restore_user_hwstate(struct user_vfp *,
126126
* thread information flags:
127127
* TIF_USEDFPU - FPU was used by this task this quantum (SMP)
128128
* TIF_POLLING_NRFLAG - true if poll_idle() is polling TIF_NEED_RESCHED
129+
*
130+
* Any bit in the range of 0..15 will cause do_work_pending() to be invoked.
129131
*/
130132
#define TIF_SIGPENDING 0 /* signal pending */
131133
#define TIF_NEED_RESCHED 1 /* rescheduling necessary */
@@ -135,6 +137,7 @@ extern int vfp_restore_user_hwstate(struct user_vfp *,
135137
#define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */
136138
#define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */
137139
#define TIF_SECCOMP 7 /* seccomp syscall filtering active */
140+
#define TIF_NOTIFY_SIGNAL 8 /* signal notifications exist */
138141

139142
#define TIF_USING_IWMMXT 17
140143
#define TIF_MEMDIE 18 /* is terminating due to OOM killer */
@@ -148,6 +151,7 @@ extern int vfp_restore_user_hwstate(struct user_vfp *,
148151
#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
149152
#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
150153
#define _TIF_SECCOMP (1 << TIF_SECCOMP)
154+
#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
151155
#define _TIF_USING_IWMMXT (1 << TIF_USING_IWMMXT)
152156

153157
/* Checks for any syscall work in entry-common.S */
@@ -158,7 +162,8 @@ extern int vfp_restore_user_hwstate(struct user_vfp *,
158162
* Change these and you break ASM code in entry-common.S
159163
*/
160164
#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
161-
_TIF_NOTIFY_RESUME | _TIF_UPROBE)
165+
_TIF_NOTIFY_RESUME | _TIF_UPROBE | \
166+
_TIF_NOTIFY_SIGNAL)
162167

163168
#endif /* __KERNEL__ */
164169
#endif /* __ASM_ARM_THREAD_INFO_H */

‎arch/arm/kernel/entry-common.S‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ __ret_fast_syscall:
5353
cmp r2, #TASK_SIZE
5454
blne addr_limit_check_failed
5555
ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing
56-
tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
56+
movs r1, r1, lsl #16
5757
bne fast_work_pending
5858

5959

@@ -90,7 +90,7 @@ __ret_fast_syscall:
9090
cmp r2, #TASK_SIZE
9191
blne addr_limit_check_failed
9292
ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing
93-
tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
93+
movs r1, r1, lsl #16
9494
beq no_work_pending
9595
UNWIND(.fnend )
9696
ENDPROC(ret_fast_syscall)
@@ -131,7 +131,7 @@ ENTRY(ret_to_user_from_irq)
131131
cmp r2, #TASK_SIZE
132132
blne addr_limit_check_failed
133133
ldr r1, [tsk, #TI_FLAGS]
134-
tst r1, #_TIF_WORK_MASK
134+
movs r1, r1, lsl #16
135135
bne slow_work_pending
136136
no_work_pending:
137137
asm_trace_hardirqs_on save = 0

‎arch/arm/kernel/entry-v7m.S‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ __irq_entry:
5959

6060
get_thread_info tsk
6161
ldr r2, [tsk, #TI_FLAGS]
62-
tst r2, #_TIF_WORK_MASK
62+
movs r2, r2, lsl #16
6363
beq 2f @ no work pending
6464
mov r0, #V7M_SCB_ICSR_PENDSVSET
6565
str r0, [r1, V7M_SCB_ICSR] @ raise PendSV

‎arch/arm/kernel/signal.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -655,7 +655,7 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall)
655655
if (unlikely(!user_mode(regs)))
656656
return 0;
657657
local_irq_enable();
658-
if (thread_flags & _TIF_SIGPENDING) {
658+
if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) {
659659
int restart = do_signal(regs, syscall);
660660
if (unlikely(restart)) {
661661
/*

0 commit comments

Comments
 (0)