Skip to content

Commit 5902da6

Browse files
olsajiriAlexei Starovoitov
olsajiri
authored and
Alexei Starovoitov
committed
libbpf: Add uprobe multi link support to bpf_program__attach_usdt
Adding support for usdt_manager_attach_usdt to use uprobe_multi link to attach to usdt probes. The uprobe_multi support is detected before the usdt program is loaded and its expected_attach_type is set accordingly. If uprobe_multi support is detected the usdt_manager_attach_usdt gathers uprobes info and calls bpf_program__attach_uprobe to create all needed uprobes. If uprobe_multi support is not detected the old behaviour stays. Also adding usdt.s program section for sleepable usdt probes. Signed-off-by: Jiri Olsa <jolsa@kernel.org> Link: https://lore.kernel.org/r/20230809083440.3209381-18-jolsa@kernel.org Signed-off-by: Alexei Starovoitov <ast@kernel.org>
1 parent 7e1b468 commit 5902da6

2 files changed

Lines changed: 82 additions & 17 deletions

File tree

‎tools/lib/bpf/libbpf.c‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,8 @@ enum sec_def_flags {
367367
SEC_SLEEPABLE = 8,
368368
/* BPF program support non-linear XDP buffer */
369369
SEC_XDP_FRAGS = 16,
370+
/* Setup proper attach type for usdt probes. */
371+
SEC_USDT = 32,
370372
};
371373

372374
struct bpf_sec_def {
@@ -6828,6 +6830,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog,
68286830
if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
68296831
opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
68306832

6833+
/* special check for usdt to use uprobe_multi link */
6834+
if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
6835+
prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
6836+
68316837
if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
68326838
int btf_obj_fd = 0, btf_type_id = 0, err;
68336839
const char *attach_name;
@@ -6896,7 +6902,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
68966902
if (!insns || !insns_cnt)
68976903
return -EINVAL;
68986904

6899-
load_attr.expected_attach_type = prog->expected_attach_type;
69006905
if (kernel_supports(obj, FEAT_PROG_NAME))
69016906
prog_name = prog->name;
69026907
load_attr.attach_prog_fd = prog->attach_prog_fd;
@@ -6932,6 +6937,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
69326937
insns_cnt = prog->insns_cnt;
69336938
}
69346939

6940+
/* allow prog_prepare_load_fn to change expected_attach_type */
6941+
load_attr.expected_attach_type = prog->expected_attach_type;
6942+
69356943
if (obj->gen_loader) {
69366944
bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
69376945
license, insns, insns_cnt, &load_attr,
@@ -8759,7 +8767,8 @@ static const struct bpf_sec_def section_defs[] = {
87598767
SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
87608768
SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
87618769
SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
8762-
SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),
8770+
SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
8771+
SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT | SEC_SLEEPABLE, attach_usdt),
87638772
SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */
87648773
SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */
87658774
SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE),

‎tools/lib/bpf/usdt.c‎

Lines changed: 71 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,7 @@ struct usdt_manager {
250250

251251
bool has_bpf_cookie;
252252
bool has_sema_refcnt;
253+
bool has_uprobe_multi;
253254
};
254255

255256
struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
@@ -284,6 +285,11 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
284285
*/
285286
man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0;
286287

288+
/*
289+
* Detect kernel support for uprobe multi link to be used for attaching
290+
* usdt probes.
291+
*/
292+
man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK);
287293
return man;
288294
}
289295

@@ -808,6 +814,8 @@ struct bpf_link_usdt {
808814
long abs_ip;
809815
struct bpf_link *link;
810816
} *uprobes;
817+
818+
struct bpf_link *multi_link;
811819
};
812820

813821
static int bpf_link_usdt_detach(struct bpf_link *link)
@@ -816,6 +824,9 @@ static int bpf_link_usdt_detach(struct bpf_link *link)
816824
struct usdt_manager *man = usdt_link->usdt_man;
817825
int i;
818826

827+
bpf_link__destroy(usdt_link->multi_link);
828+
829+
/* When having multi_link, uprobe_cnt is 0 */
819830
for (i = 0; i < usdt_link->uprobe_cnt; i++) {
820831
/* detach underlying uprobe link */
821832
bpf_link__destroy(usdt_link->uprobes[i].link);
@@ -946,11 +957,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
946957
const char *usdt_provider, const char *usdt_name,
947958
__u64 usdt_cookie)
948959
{
960+
unsigned long *offsets = NULL, *ref_ctr_offsets = NULL;
949961
int i, err, spec_map_fd, ip_map_fd;
950962
LIBBPF_OPTS(bpf_uprobe_opts, opts);
951963
struct hashmap *specs_hash = NULL;
952964
struct bpf_link_usdt *link = NULL;
953965
struct usdt_target *targets = NULL;
966+
__u64 *cookies = NULL;
954967
struct elf_fd elf_fd;
955968
size_t target_cnt;
956969

@@ -997,10 +1010,21 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
9971010
link->link.detach = &bpf_link_usdt_detach;
9981011
link->link.dealloc = &bpf_link_usdt_dealloc;
9991012

1000-
link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
1001-
if (!link->uprobes) {
1002-
err = -ENOMEM;
1003-
goto err_out;
1013+
if (man->has_uprobe_multi) {
1014+
offsets = calloc(target_cnt, sizeof(*offsets));
1015+
cookies = calloc(target_cnt, sizeof(*cookies));
1016+
ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets));
1017+
1018+
if (!offsets || !ref_ctr_offsets || !cookies) {
1019+
err = -ENOMEM;
1020+
goto err_out;
1021+
}
1022+
} else {
1023+
link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
1024+
if (!link->uprobes) {
1025+
err = -ENOMEM;
1026+
goto err_out;
1027+
}
10041028
}
10051029

10061030
for (i = 0; i < target_cnt; i++) {
@@ -1041,20 +1065,48 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
10411065
goto err_out;
10421066
}
10431067

1044-
opts.ref_ctr_offset = target->sema_off;
1045-
opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
1046-
uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
1047-
target->rel_ip, &opts);
1048-
err = libbpf_get_error(uprobe_link);
1049-
if (err) {
1050-
pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
1051-
i, usdt_provider, usdt_name, path, err);
1068+
if (man->has_uprobe_multi) {
1069+
offsets[i] = target->rel_ip;
1070+
ref_ctr_offsets[i] = target->sema_off;
1071+
cookies[i] = spec_id;
1072+
} else {
1073+
opts.ref_ctr_offset = target->sema_off;
1074+
opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
1075+
uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
1076+
target->rel_ip, &opts);
1077+
err = libbpf_get_error(uprobe_link);
1078+
if (err) {
1079+
pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
1080+
i, usdt_provider, usdt_name, path, err);
1081+
goto err_out;
1082+
}
1083+
1084+
link->uprobes[i].link = uprobe_link;
1085+
link->uprobes[i].abs_ip = target->abs_ip;
1086+
link->uprobe_cnt++;
1087+
}
1088+
}
1089+
1090+
if (man->has_uprobe_multi) {
1091+
LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi,
1092+
.ref_ctr_offsets = ref_ctr_offsets,
1093+
.offsets = offsets,
1094+
.cookies = cookies,
1095+
.cnt = target_cnt,
1096+
);
1097+
1098+
link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path,
1099+
NULL, &opts_multi);
1100+
if (!link->multi_link) {
1101+
err = -errno;
1102+
pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n",
1103+
usdt_provider, usdt_name, path, err);
10521104
goto err_out;
10531105
}
10541106

1055-
link->uprobes[i].link = uprobe_link;
1056-
link->uprobes[i].abs_ip = target->abs_ip;
1057-
link->uprobe_cnt++;
1107+
free(offsets);
1108+
free(ref_ctr_offsets);
1109+
free(cookies);
10581110
}
10591111

10601112
free(targets);
@@ -1063,6 +1115,10 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
10631115
return &link->link;
10641116

10651117
err_out:
1118+
free(offsets);
1119+
free(ref_ctr_offsets);
1120+
free(cookies);
1121+
10661122
if (link)
10671123
bpf_link__destroy(&link->link);
10681124
free(targets);

0 commit comments

Comments
 (0)