Skip to content

Commit 4e4e36d

Browse files
willdeaconwilldeacon
committed
Merge branch 'for-next/uprobes' into for-next/core
* for-next/uprobes: arm64: probes: Fix incorrect bl/blr address and register usage uprobes: uprobe_warn should use passed task arm64: Kconfig: Remove GCS restrictions on UPROBES arm64: uprobes: Add GCS support to uretprobes arm64: probes: Add GCS support to bl/blr/ret arm64: uaccess: Add additional userspace GCS accessors arm64: uaccess: Move existing GCS accessors definitions to gcs.h arm64: probes: Break ret out from bl/blr
2 parents c7c7eb4 + ea87c55 commit 4e4e36d

8 files changed

Lines changed: 172 additions & 55 deletions

File tree

‎arch/arm64/Kconfig‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2213,7 +2213,6 @@ config ARM64_GCS
22132213
default y
22142214
select ARCH_HAS_USER_SHADOW_STACK
22152215
select ARCH_USES_HIGH_VMA_FLAGS
2216-
depends on !UPROBES
22172216
help
22182217
Guarded Control Stack (GCS) provides support for a separate
22192218
stack with restricted access which contains only return

‎arch/arm64/include/asm/gcs.h‎

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ static inline void gcsstr(u64 *addr, u64 val)
2121
register u64 *_addr __asm__ ("x0") = addr;
2222
register long _val __asm__ ("x1") = val;
2323

24-
/* GCSSTTR x1, x0 */
24+
/* GCSSTTR x1, [x0] */
2525
asm volatile(
2626
".inst 0xd91f1c01\n"
2727
:
@@ -81,6 +81,82 @@ static inline int gcs_check_locked(struct task_struct *task,
8181
return 0;
8282
}
8383

84+
static inline int gcssttr(unsigned long __user *addr, unsigned long val)
85+
{
86+
register unsigned long __user *_addr __asm__ ("x0") = addr;
87+
register unsigned long _val __asm__ ("x1") = val;
88+
int err = 0;
89+
90+
/* GCSSTTR x1, [x0] */
91+
asm volatile(
92+
"1: .inst 0xd91f1c01\n"
93+
"2: \n"
94+
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0)
95+
: "+r" (err)
96+
: "rZ" (_val), "r" (_addr)
97+
: "memory");
98+
99+
return err;
100+
}
101+
102+
static inline void put_user_gcs(unsigned long val, unsigned long __user *addr,
103+
int *err)
104+
{
105+
int ret;
106+
107+
if (!access_ok((char __user *)addr, sizeof(u64))) {
108+
*err = -EFAULT;
109+
return;
110+
}
111+
112+
uaccess_ttbr0_enable();
113+
ret = gcssttr(addr, val);
114+
if (ret != 0)
115+
*err = ret;
116+
uaccess_ttbr0_disable();
117+
}
118+
119+
static inline void push_user_gcs(unsigned long val, int *err)
120+
{
121+
u64 gcspr = read_sysreg_s(SYS_GCSPR_EL0);
122+
123+
gcspr -= sizeof(u64);
124+
put_user_gcs(val, (unsigned long __user *)gcspr, err);
125+
if (!*err)
126+
write_sysreg_s(gcspr, SYS_GCSPR_EL0);
127+
}
128+
129+
/*
130+
* Unlike put/push_user_gcs() above, get/pop_user_gsc() doesn't
131+
* validate the GCS permission is set on the page being read. This
132+
* differs from how the hardware works when it consumes data stored at
133+
* GCSPR. Callers should ensure this is acceptable.
134+
*/
135+
static inline u64 get_user_gcs(unsigned long __user *addr, int *err)
136+
{
137+
unsigned long ret;
138+
u64 load = 0;
139+
140+
/* Ensure previous GCS operation are visible before we read the page */
141+
gcsb_dsync();
142+
ret = copy_from_user(&load, addr, sizeof(load));
143+
if (ret != 0)
144+
*err = ret;
145+
return load;
146+
}
147+
148+
static inline u64 pop_user_gcs(int *err)
149+
{
150+
u64 gcspr = read_sysreg_s(SYS_GCSPR_EL0);
151+
u64 read_val;
152+
153+
read_val = get_user_gcs((__force unsigned long __user *)gcspr, err);
154+
if (!*err)
155+
write_sysreg_s(gcspr + sizeof(u64), SYS_GCSPR_EL0);
156+
157+
return read_val;
158+
}
159+
84160
#else
85161

86162
static inline bool task_gcs_el0_enabled(struct task_struct *task)
@@ -91,6 +167,10 @@ static inline bool task_gcs_el0_enabled(struct task_struct *task)
91167
static inline void gcs_set_el0_mode(struct task_struct *task) { }
92168
static inline void gcs_free(struct task_struct *task) { }
93169
static inline void gcs_preserve_current_state(void) { }
170+
static inline void put_user_gcs(unsigned long val, unsigned long __user *addr,
171+
int *err) { }
172+
static inline void push_user_gcs(unsigned long val, int *err) { }
173+
94174
static inline unsigned long gcs_alloc_thread_stack(struct task_struct *tsk,
95175
const struct kernel_clone_args *args)
96176
{
@@ -101,6 +181,15 @@ static inline int gcs_check_locked(struct task_struct *task,
101181
{
102182
return 0;
103183
}
184+
static inline u64 get_user_gcs(unsigned long __user *addr, int *err)
185+
{
186+
*err = -EFAULT;
187+
return 0;
188+
}
189+
static inline u64 pop_user_gcs(int *err)
190+
{
191+
return 0;
192+
}
104193

105194
#endif
106195

‎arch/arm64/include/asm/uaccess.h‎

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -502,44 +502,4 @@ static inline size_t probe_subpage_writeable(const char __user *uaddr,
502502

503503
#endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */
504504

505-
#ifdef CONFIG_ARM64_GCS
506-
507-
static inline int gcssttr(unsigned long __user *addr, unsigned long val)
508-
{
509-
register unsigned long __user *_addr __asm__ ("x0") = addr;
510-
register unsigned long _val __asm__ ("x1") = val;
511-
int err = 0;
512-
513-
/* GCSSTTR x1, x0 */
514-
asm volatile(
515-
"1: .inst 0xd91f1c01\n"
516-
"2: \n"
517-
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0)
518-
: "+r" (err)
519-
: "rZ" (_val), "r" (_addr)
520-
: "memory");
521-
522-
return err;
523-
}
524-
525-
static inline void put_user_gcs(unsigned long val, unsigned long __user *addr,
526-
int *err)
527-
{
528-
int ret;
529-
530-
if (!access_ok((char __user *)addr, sizeof(u64))) {
531-
*err = -EFAULT;
532-
return;
533-
}
534-
535-
uaccess_ttbr0_enable();
536-
ret = gcssttr(addr, val);
537-
if (ret != 0)
538-
*err = ret;
539-
uaccess_ttbr0_disable();
540-
}
541-
542-
543-
#endif /* CONFIG_ARM64_GCS */
544-
545505
#endif /* __ASM_UACCESS_H */

‎arch/arm64/kernel/probes/decode-insn.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,10 @@ arm_probe_decode_insn(u32 insn, struct arch_probe_insn *api)
108108
aarch64_insn_is_bl(insn)) {
109109
api->handler = simulate_b_bl;
110110
} else if (aarch64_insn_is_br(insn) ||
111-
aarch64_insn_is_blr(insn) ||
112-
aarch64_insn_is_ret(insn)) {
113-
api->handler = simulate_br_blr_ret;
111+
aarch64_insn_is_blr(insn)) {
112+
api->handler = simulate_br_blr;
113+
} else if (aarch64_insn_is_ret(insn)) {
114+
api->handler = simulate_ret;
114115
} else {
115116
/*
116117
* Instruction cannot be stepped out-of-line and we don't

‎arch/arm64/kernel/probes/simulate-insn.c‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <asm/traps.h>
1414

1515
#include "simulate-insn.h"
16+
#include "asm/gcs.h"
1617

1718
#define bbl_displacement(insn) \
1819
sign_extend32(((insn) & 0x3ffffff) << 2, 27)
@@ -49,6 +50,21 @@ static inline u32 get_w_reg(struct pt_regs *regs, int reg)
4950
return lower_32_bits(pt_regs_read_reg(regs, reg));
5051
}
5152

53+
static inline int update_lr(struct pt_regs *regs, long addr)
54+
{
55+
int err = 0;
56+
57+
if (user_mode(regs) && task_gcs_el0_enabled(current)) {
58+
push_user_gcs(addr, &err);
59+
if (err) {
60+
force_sig(SIGSEGV);
61+
return err;
62+
}
63+
}
64+
procedure_link_pointer_set(regs, addr);
65+
return err;
66+
}
67+
5268
static bool __kprobes check_cbz(u32 opcode, struct pt_regs *regs)
5369
{
5470
int xn = opcode & 0x1f;
@@ -107,9 +123,9 @@ simulate_b_bl(u32 opcode, long addr, struct pt_regs *regs)
107123
{
108124
int disp = bbl_displacement(opcode);
109125

110-
/* Link register is x30 */
111126
if (opcode & (1 << 31))
112-
set_x_reg(regs, 30, addr + 4);
127+
if (update_lr(regs, addr + 4))
128+
return;
113129

114130
instruction_pointer_set(regs, addr + disp);
115131
}
@@ -126,16 +142,34 @@ simulate_b_cond(u32 opcode, long addr, struct pt_regs *regs)
126142
}
127143

128144
void __kprobes
129-
simulate_br_blr_ret(u32 opcode, long addr, struct pt_regs *regs)
145+
simulate_br_blr(u32 opcode, long addr, struct pt_regs *regs)
130146
{
131147
int xn = (opcode >> 5) & 0x1f;
148+
u64 b_target = get_x_reg(regs, xn);
132149

133-
/* update pc first in case we're doing a "blr lr" */
134-
instruction_pointer_set(regs, get_x_reg(regs, xn));
135-
136-
/* Link register is x30 */
137150
if (((opcode >> 21) & 0x3) == 1)
138-
set_x_reg(regs, 30, addr + 4);
151+
if (update_lr(regs, addr + 4))
152+
return;
153+
154+
instruction_pointer_set(regs, b_target);
155+
}
156+
157+
void __kprobes
158+
simulate_ret(u32 opcode, long addr, struct pt_regs *regs)
159+
{
160+
u64 ret_addr;
161+
int err = 0;
162+
int xn = (opcode >> 5) & 0x1f;
163+
u64 r_target = get_x_reg(regs, xn);
164+
165+
if (user_mode(regs) && task_gcs_el0_enabled(current)) {
166+
ret_addr = pop_user_gcs(&err);
167+
if (err || ret_addr != r_target) {
168+
force_sig(SIGSEGV);
169+
return;
170+
}
171+
}
172+
instruction_pointer_set(regs, r_target);
139173
}
140174

141175
void __kprobes

‎arch/arm64/kernel/probes/simulate-insn.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
void simulate_adr_adrp(u32 opcode, long addr, struct pt_regs *regs);
1212
void simulate_b_bl(u32 opcode, long addr, struct pt_regs *regs);
1313
void simulate_b_cond(u32 opcode, long addr, struct pt_regs *regs);
14-
void simulate_br_blr_ret(u32 opcode, long addr, struct pt_regs *regs);
14+
void simulate_br_blr(u32 opcode, long addr, struct pt_regs *regs);
15+
void simulate_ret(u32 opcode, long addr, struct pt_regs *regs);
1516
void simulate_cbz_cbnz(u32 opcode, long addr, struct pt_regs *regs);
1617
void simulate_tbz_tbnz(u32 opcode, long addr, struct pt_regs *regs);
1718
void simulate_ldr_literal(u32 opcode, long addr, struct pt_regs *regs);

‎arch/arm64/kernel/probes/uprobes.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <linux/ptrace.h>
77
#include <linux/uprobes.h>
88
#include <asm/cacheflush.h>
9+
#include <asm/gcs.h>
910

1011
#include "decode-insn.h"
1112

@@ -159,11 +160,43 @@ arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr,
159160
struct pt_regs *regs)
160161
{
161162
unsigned long orig_ret_vaddr;
163+
unsigned long gcs_ret_vaddr;
164+
int err = 0;
165+
u64 gcspr;
162166

163167
orig_ret_vaddr = procedure_link_pointer(regs);
168+
169+
if (task_gcs_el0_enabled(current)) {
170+
gcspr = read_sysreg_s(SYS_GCSPR_EL0);
171+
gcs_ret_vaddr = get_user_gcs((__force unsigned long __user *)gcspr, &err);
172+
if (err) {
173+
force_sig(SIGSEGV);
174+
goto out;
175+
}
176+
177+
/*
178+
* If the LR and GCS return addr don't match, then some kind of PAC
179+
* signing or control flow occurred since entering the probed function.
180+
* Likely because the user is attempting to retprobe on an instruction
181+
* that isn't a function boundary or inside a leaf function. Explicitly
182+
* abort this retprobe because it will generate a GCS exception.
183+
*/
184+
if (gcs_ret_vaddr != orig_ret_vaddr) {
185+
orig_ret_vaddr = -1;
186+
goto out;
187+
}
188+
189+
put_user_gcs(trampoline_vaddr, (__force unsigned long __user *)gcspr, &err);
190+
if (err) {
191+
force_sig(SIGSEGV);
192+
goto out;
193+
}
194+
}
195+
164196
/* Replace the return addr with trampoline addr */
165197
procedure_link_pointer_set(regs, trampoline_vaddr);
166198

199+
out:
167200
return orig_ret_vaddr;
168201
}
169202

‎kernel/events/uprobes.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ struct xol_area {
121121

122122
static void uprobe_warn(struct task_struct *t, const char *msg)
123123
{
124-
pr_warn("uprobe: %s:%d failed to %s\n", current->comm, current->pid, msg);
124+
pr_warn("uprobe: %s:%d failed to %s\n", t->comm, t->pid, msg);
125125
}
126126

127127
/*

0 commit comments

Comments
 (0)