Skip to content

Commit 0e73ef1

Browse files
anakryikoanakryiko
committed
Merge branch 'bpf: Add missed stats for kprobes'
Jiri Olsa says: ==================== hi, at the moment we can't retrieve the number of missed kprobe executions and subsequent execution of BPF programs. This patchset adds: - counting of missed execution on attach layer for: . kprobes attached through perf link (kprobe/ftrace) . kprobes attached through kprobe.multi link (fprobe) - counting of recursion_misses for BPF kprobe programs It's still technically possible to create kprobe without perf link (using SET_BPF perf ioctl) in which case we don't have a way to retrieve the kprobe's 'missed' count. However both libbpf and cilium/ebpf libraries use perf link if it's available, and for old kernels without perf link support we can use BPF program to retrieve the kprobe missed count. v3 changes: - added acks [Song] - make test_missed not serial [Andrii] Also available at: https://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git bpf/missed_stats thanks, jirka ==================== Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
2 parents e0fa652 + 85981e0 commit 0e73ef1

15 files changed

Lines changed: 322 additions & 13 deletions

File tree

‎include/linux/bpf.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2922,6 +2922,22 @@ static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
29222922
#endif /* CONFIG_BPF_SYSCALL */
29232923
#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
29242924

2925+
static __always_inline void
2926+
bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
2927+
{
2928+
const struct bpf_prog_array_item *item;
2929+
struct bpf_prog *prog;
2930+
2931+
if (unlikely(!array))
2932+
return;
2933+
2934+
item = &array->items[0];
2935+
while ((prog = READ_ONCE(item->prog))) {
2936+
bpf_prog_inc_misses_counter(prog);
2937+
item++;
2938+
}
2939+
}
2940+
29252941
#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
29262942
void bpf_sk_reuseport_detach(struct sock *sk);
29272943
int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,

‎include/linux/trace_events.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,8 @@ struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name);
761761
void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);
762762
int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
763763
u32 *fd_type, const char **buf,
764-
u64 *probe_offset, u64 *probe_addr);
764+
u64 *probe_offset, u64 *probe_addr,
765+
unsigned long *missed);
765766
int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
766767
int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
767768
#else
@@ -801,7 +802,7 @@ static inline void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
801802
static inline int bpf_get_perf_event_info(const struct perf_event *event,
802803
u32 *prog_id, u32 *fd_type,
803804
const char **buf, u64 *probe_offset,
804-
u64 *probe_addr)
805+
u64 *probe_addr, unsigned long *missed)
805806
{
806807
return -EOPNOTSUPP;
807808
}
@@ -877,6 +878,7 @@ extern void perf_kprobe_destroy(struct perf_event *event);
877878
extern int bpf_get_kprobe_info(const struct perf_event *event,
878879
u32 *fd_type, const char **symbol,
879880
u64 *probe_offset, u64 *probe_addr,
881+
unsigned long *missed,
880882
bool perf_type_tracepoint);
881883
#endif
882884
#ifdef CONFIG_UPROBE_EVENTS

‎include/uapi/linux/bpf.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6532,6 +6532,7 @@ struct bpf_link_info {
65326532
__aligned_u64 addrs;
65336533
__u32 count; /* in/out: kprobe_multi function count */
65346534
__u32 flags;
6535+
__u64 missed;
65356536
} kprobe_multi;
65366537
struct {
65376538
__u32 type; /* enum bpf_perf_event_type */
@@ -6547,6 +6548,7 @@ struct bpf_link_info {
65476548
__u32 name_len;
65486549
__u32 offset; /* offset from func_name */
65496550
__u64 addr;
6551+
__u64 missed;
65506552
} kprobe; /* BPF_PERF_EVENT_KPROBE, BPF_PERF_EVENT_KRETPROBE */
65516553
struct {
65526554
__aligned_u64 tp_name; /* in/out */

‎kernel/bpf/syscall.c‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3374,7 +3374,7 @@ static void bpf_perf_link_dealloc(struct bpf_link *link)
33743374
static int bpf_perf_link_fill_common(const struct perf_event *event,
33753375
char __user *uname, u32 ulen,
33763376
u64 *probe_offset, u64 *probe_addr,
3377-
u32 *fd_type)
3377+
u32 *fd_type, unsigned long *missed)
33783378
{
33793379
const char *buf;
33803380
u32 prog_id;
@@ -3385,7 +3385,7 @@ static int bpf_perf_link_fill_common(const struct perf_event *event,
33853385
return -EINVAL;
33863386

33873387
err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf,
3388-
probe_offset, probe_addr);
3388+
probe_offset, probe_addr, missed);
33893389
if (err)
33903390
return err;
33913391
if (!uname)
@@ -3408,6 +3408,7 @@ static int bpf_perf_link_fill_common(const struct perf_event *event,
34083408
static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
34093409
struct bpf_link_info *info)
34103410
{
3411+
unsigned long missed;
34113412
char __user *uname;
34123413
u64 addr, offset;
34133414
u32 ulen, type;
@@ -3416,7 +3417,7 @@ static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
34163417
uname = u64_to_user_ptr(info->perf_event.kprobe.func_name);
34173418
ulen = info->perf_event.kprobe.name_len;
34183419
err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr,
3419-
&type);
3420+
&type, &missed);
34203421
if (err)
34213422
return err;
34223423
if (type == BPF_FD_TYPE_KRETPROBE)
@@ -3425,6 +3426,7 @@ static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
34253426
info->perf_event.type = BPF_PERF_EVENT_KPROBE;
34263427

34273428
info->perf_event.kprobe.offset = offset;
3429+
info->perf_event.kprobe.missed = missed;
34283430
if (!kallsyms_show_value(current_cred()))
34293431
addr = 0;
34303432
info->perf_event.kprobe.addr = addr;
@@ -3444,7 +3446,7 @@ static int bpf_perf_link_fill_uprobe(const struct perf_event *event,
34443446
uname = u64_to_user_ptr(info->perf_event.uprobe.file_name);
34453447
ulen = info->perf_event.uprobe.name_len;
34463448
err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr,
3447-
&type);
3449+
&type, NULL);
34483450
if (err)
34493451
return err;
34503452

@@ -3480,7 +3482,7 @@ static int bpf_perf_link_fill_tracepoint(const struct perf_event *event,
34803482
uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name);
34813483
ulen = info->perf_event.tracepoint.name_len;
34823484
info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT;
3483-
return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL);
3485+
return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL, NULL);
34843486
}
34853487

34863488
static int bpf_perf_link_fill_perf_event(const struct perf_event *event,
@@ -4813,7 +4815,7 @@ static int bpf_task_fd_query(const union bpf_attr *attr,
48134815

48144816
err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
48154817
&buf, &probe_offset,
4816-
&probe_addr);
4818+
&probe_addr, NULL);
48174819
if (!err)
48184820
err = bpf_task_fd_query_copy(attr, uattr, prog_id,
48194821
fd_type, buf,

‎kernel/trace/bpf_trace.c‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,9 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
117117
* and don't send kprobe event into ring-buffer,
118118
* so return zero here
119119
*/
120+
rcu_read_lock();
121+
bpf_prog_inc_misses_counters(rcu_dereference(call->prog_array));
122+
rcu_read_unlock();
120123
ret = 0;
121124
goto out;
122125
}
@@ -2384,7 +2387,8 @@ int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
23842387

23852388
int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
23862389
u32 *fd_type, const char **buf,
2387-
u64 *probe_offset, u64 *probe_addr)
2390+
u64 *probe_offset, u64 *probe_addr,
2391+
unsigned long *missed)
23882392
{
23892393
bool is_tracepoint, is_syscall_tp;
23902394
struct bpf_prog *prog;
@@ -2419,7 +2423,7 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
24192423
#ifdef CONFIG_KPROBE_EVENTS
24202424
if (flags & TRACE_EVENT_FL_KPROBE)
24212425
err = bpf_get_kprobe_info(event, fd_type, buf,
2422-
probe_offset, probe_addr,
2426+
probe_offset, probe_addr, missed,
24232427
event->attr.type == PERF_TYPE_TRACEPOINT);
24242428
#endif
24252429
#ifdef CONFIG_UPROBE_EVENTS
@@ -2614,6 +2618,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
26142618
kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
26152619
info->kprobe_multi.count = kmulti_link->cnt;
26162620
info->kprobe_multi.flags = kmulti_link->flags;
2621+
info->kprobe_multi.missed = kmulti_link->fp.nmissed;
26172622

26182623
if (!uaddrs)
26192624
return 0;
@@ -2710,6 +2715,7 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,
27102715
int err;
27112716

27122717
if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
2718+
bpf_prog_inc_misses_counter(link->link.prog);
27132719
err = 0;
27142720
goto out;
27152721
}

‎kernel/trace/trace_kprobe.c‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1189,6 +1189,12 @@ static const struct file_operations kprobe_events_ops = {
11891189
.write = probes_write,
11901190
};
11911191

1192+
static unsigned long trace_kprobe_missed(struct trace_kprobe *tk)
1193+
{
1194+
return trace_kprobe_is_return(tk) ?
1195+
tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;
1196+
}
1197+
11921198
/* Probes profiling interfaces */
11931199
static int probes_profile_seq_show(struct seq_file *m, void *v)
11941200
{
@@ -1200,8 +1206,7 @@ static int probes_profile_seq_show(struct seq_file *m, void *v)
12001206
return 0;
12011207

12021208
tk = to_trace_kprobe(ev);
1203-
nmissed = trace_kprobe_is_return(tk) ?
1204-
tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;
1209+
nmissed = trace_kprobe_missed(tk);
12051210
seq_printf(m, " %-44s %15lu %15lu\n",
12061211
trace_probe_name(&tk->tp),
12071212
trace_kprobe_nhit(tk),
@@ -1547,7 +1552,8 @@ NOKPROBE_SYMBOL(kretprobe_perf_func);
15471552

15481553
int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
15491554
const char **symbol, u64 *probe_offset,
1550-
u64 *probe_addr, bool perf_type_tracepoint)
1555+
u64 *probe_addr, unsigned long *missed,
1556+
bool perf_type_tracepoint)
15511557
{
15521558
const char *pevent = trace_event_name(event->tp_event);
15531559
const char *group = event->tp_event->class->system;
@@ -1566,6 +1572,8 @@ int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
15661572
*probe_addr = kallsyms_show_value(current_cred()) ?
15671573
(unsigned long)tk->rp.kp.addr : 0;
15681574
*symbol = tk->symbol;
1575+
if (missed)
1576+
*missed = trace_kprobe_missed(tk);
15691577
return 0;
15701578
}
15711579
#endif /* CONFIG_PERF_EVENTS */

‎tools/bpf/bpftool/link.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
265265
jsonw_bool_field(json_wtr, "retprobe",
266266
info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN);
267267
jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count);
268+
jsonw_uint_field(json_wtr, "missed", info->kprobe_multi.missed);
268269
jsonw_name(json_wtr, "funcs");
269270
jsonw_start_array(json_wtr);
270271
addrs = u64_to_ptr(info->kprobe_multi.addrs);
@@ -301,6 +302,7 @@ show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr)
301302
jsonw_string_field(wtr, "func",
302303
u64_to_ptr(info->perf_event.kprobe.func_name));
303304
jsonw_uint_field(wtr, "offset", info->perf_event.kprobe.offset);
305+
jsonw_uint_field(wtr, "missed", info->perf_event.kprobe.missed);
304306
}
305307

306308
static void
@@ -641,6 +643,8 @@ static void show_kprobe_multi_plain(struct bpf_link_info *info)
641643
else
642644
printf("\n\tkprobe.multi ");
643645
printf("func_cnt %u ", info->kprobe_multi.count);
646+
if (info->kprobe_multi.missed)
647+
printf("missed %llu ", info->kprobe_multi.missed);
644648
addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
645649
qsort(addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
646650

@@ -683,6 +687,8 @@ static void show_perf_event_kprobe_plain(struct bpf_link_info *info)
683687
printf("%s", buf);
684688
if (info->perf_event.kprobe.offset)
685689
printf("+%#x", info->perf_event.kprobe.offset);
690+
if (info->perf_event.kprobe.missed)
691+
printf(" missed %llu", info->perf_event.kprobe.missed);
686692
printf(" ");
687693
}
688694

‎tools/include/uapi/linux/bpf.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6532,6 +6532,7 @@ struct bpf_link_info {
65326532
__aligned_u64 addrs;
65336533
__u32 count; /* in/out: kprobe_multi function count */
65346534
__u32 flags;
6535+
__u64 missed;
65356536
} kprobe_multi;
65366537
struct {
65376538
__u32 type; /* enum bpf_perf_event_type */
@@ -6547,6 +6548,7 @@ struct bpf_link_info {
65476548
__u32 name_len;
65486549
__u32 offset; /* offset from func_name */
65496550
__u64 addr;
6551+
__u64 missed;
65506552
} kprobe; /* BPF_PERF_EVENT_KPROBE, BPF_PERF_EVENT_KRETPROBE */
65516553
struct {
65526554
__aligned_u64 tp_name; /* in/out */

‎tools/testing/selftests/bpf/DENYLIST.aarch64‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ fexit_test/fexit_many_args # fexit_many_args:FAIL:fexit_ma
1010
fill_link_info/kprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95
1111
fill_link_info/kretprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95
1212
fill_link_info/kprobe_multi_invalid_ubuff # bpf_program__attach_kprobe_multi_opts unexpected error: -95
13+
missed/kprobe_recursion # missed_kprobe_recursion__attach unexpected error: -95 (errno 95)

‎tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,10 @@ __bpf_kfunc void bpf_iter_testmod_seq_destroy(struct bpf_iter_testmod_seq *it)
138138
it->cnt = 0;
139139
}
140140

141+
__bpf_kfunc void bpf_kfunc_common_test(void)
142+
{
143+
}
144+
141145
struct bpf_testmod_btf_type_tag_1 {
142146
int a;
143147
};
@@ -343,6 +347,7 @@ BTF_SET8_START(bpf_testmod_common_kfunc_ids)
343347
BTF_ID_FLAGS(func, bpf_iter_testmod_seq_new, KF_ITER_NEW)
344348
BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL)
345349
BTF_ID_FLAGS(func, bpf_iter_testmod_seq_destroy, KF_ITER_DESTROY)
350+
BTF_ID_FLAGS(func, bpf_kfunc_common_test)
346351
BTF_SET8_END(bpf_testmod_common_kfunc_ids)
347352

348353
static const struct btf_kfunc_id_set bpf_testmod_common_kfunc_set = {

0 commit comments

Comments
 (0)