Skip to content

Commit 2ed2d8f

Browse files
Abhishek Dubeymaddy-kerneldev
Abhishek Dubey
authored and
maddy-kerneldev
committed
powerpc64/bpf: Support tailcalls with subprogs
Enable tailcalls support in subprogs by passing tail call count as reference instead of value. The actual tailcall count is always maintained in the tailcall field present in the frame of main function (also called entry function). The tailcall field in the stack frame of subprogs contains reference to the tailcall field in the stack frame of main BPF program. Accordingly, rename tail_call_cnt field in the stack layout to tail_call_info. Signed-off-by: Abhishek Dubey <adubey@linux.ibm.com> Signed-off-by: Madhavan Srinivasan <maddy@linux.ibm.com> Link: https://patch.msgid.link/20260124075223.6033-3-adubey@linux.ibm.com
1 parent 15513be commit 2ed2d8f

3 files changed

Lines changed: 122 additions & 22 deletions

File tree

‎arch/powerpc/net/bpf_jit.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,13 @@
5252
EMIT(PPC_INST_BRANCH_COND | (((cond) & 0x3ff) << 16) | (offset & 0xfffc)); \
5353
} while (0)
5454

55+
/* When constant jump offset is known prior */
56+
#define PPC_BCC_CONST_SHORT(cond, offset) \
57+
do { \
58+
BUILD_BUG_ON(offset < -0x8000 || offset > 0x7fff || (offset & 0x3)); \
59+
EMIT(PPC_INST_BRANCH_COND | (((cond) & 0x3ff) << 16) | (offset & 0xfffc)); \
60+
} while (0)
61+
5562
/*
5663
* Sign-extended 32-bit immediate load
5764
*
@@ -73,6 +80,10 @@
7380
} } while (0)
7481

7582
#ifdef CONFIG_PPC64
83+
84+
/* for gpr non volatile registers BPG_REG_6 to 10 */
85+
#define BPF_PPC_STACK_SAVE (6 * 8)
86+
7687
/* If dummy pass (!image), account for maximum possible instructions */
7788
#define PPC_LI64(d, i) do { \
7889
if (!image) \
@@ -167,6 +178,7 @@ struct codegen_context {
167178
unsigned int alt_exit_addr;
168179
u64 arena_vm_start;
169180
u64 user_vm_start;
181+
bool is_subprog;
170182
};
171183

172184
#define bpf_to_ppc(r) (ctx->b2p[r])
@@ -206,6 +218,7 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass
206218
struct codegen_context *ctx, int insn_idx,
207219
int jmp_off, int dst_reg, u32 code);
208220

221+
int bpf_jit_stack_tailcallinfo_offset(struct codegen_context *ctx);
209222
#endif
210223

211224
#endif

‎arch/powerpc/net/bpf_jit_comp.c‎

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
206206
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
207207
cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);
208208
cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena);
209+
cgctx.is_subprog = bpf_is_subprog(fp);
209210

210211
/* Scouting faux-generate pass 0 */
211212
if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
@@ -435,6 +436,11 @@ void bpf_jit_free(struct bpf_prog *fp)
435436
bpf_prog_unlock_free(fp);
436437
}
437438

439+
bool bpf_jit_supports_subprog_tailcalls(void)
440+
{
441+
return IS_ENABLED(CONFIG_PPC64);
442+
}
443+
438444
bool bpf_jit_supports_kfunc_call(void)
439445
{
440446
return true;
@@ -617,15 +623,50 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context
617623
return 0;
618624
}
619625

620-
static void bpf_trampoline_setup_tail_call_cnt(u32 *image, struct codegen_context *ctx,
621-
int func_frame_offset, int r4_off)
626+
/*
627+
* Refer __arch_prepare_bpf_trampoline() for stack component details.
628+
*
629+
* The tailcall count/reference is present in caller's stack frame. The
630+
* tail_call_info is saved at the same offset on the trampoline frame
631+
* for the traced function (BPF subprog/callee) to fetch it.
632+
*/
633+
static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx,
634+
int func_frame_offset,
635+
int bpf_dummy_frame_size, int r4_off)
622636
{
623637
if (IS_ENABLED(CONFIG_PPC64)) {
624638
/* See Generated stack layout */
625-
int tailcallcnt_offset = BPF_PPC_TAILCALL;
639+
int tailcallinfo_offset = BPF_PPC_TAILCALL;
640+
641+
/*
642+
* func_frame_offset = ...(1)
643+
* bpf_dummy_frame_size + trampoline_frame_size
644+
*/
645+
EMIT(PPC_RAW_LD(_R4, _R1, func_frame_offset));
646+
EMIT(PPC_RAW_LD(_R3, _R4, -tailcallinfo_offset));
626647

627-
EMIT(PPC_RAW_LL(_R3, _R1, func_frame_offset - tailcallcnt_offset));
628-
EMIT(PPC_RAW_STL(_R3, _R1, -tailcallcnt_offset));
648+
/*
649+
* Setting the tail_call_info in trampoline's frame
650+
* depending on if previous frame had value or reference.
651+
*/
652+
EMIT(PPC_RAW_CMPLWI(_R3, MAX_TAIL_CALL_CNT));
653+
PPC_BCC_CONST_SHORT(COND_GT, 8);
654+
EMIT(PPC_RAW_ADDI(_R3, _R4, bpf_jit_stack_tailcallinfo_offset(ctx)));
655+
/*
656+
* From ...(1) above:
657+
* trampoline_frame_bottom = ...(2)
658+
* func_frame_offset - bpf_dummy_frame_size
659+
*
660+
* Using ...(2) derived above:
661+
* trampoline_tail_call_info_offset = ...(3)
662+
* trampoline_frame_bottom - tailcallinfo_offset
663+
*
664+
* From ...(3):
665+
* Use trampoline_tail_call_info_offset to write reference of main's
666+
* tail_call_info in trampoline frame.
667+
*/
668+
EMIT(PPC_RAW_STL(_R3, _R1, (func_frame_offset - bpf_dummy_frame_size)
669+
- tailcallinfo_offset));
629670
} else {
630671
/* See bpf_jit_stack_offsetof() and BPF_PPC_TC */
631672
EMIT(PPC_RAW_LL(_R4, _R1, r4_off));
@@ -731,6 +772,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
731772
* LR save area [ r0 save (64-bit) ] | header
732773
* [ r0 save (32-bit) ] |
733774
* dummy frame for unwind [ back chain 1 ] --
775+
* [ tail_call_info ] optional - 64-bit powerpc
734776
* [ padding ] align stack frame
735777
* r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc
736778
* alt_lr_off [ real lr (ool stub)] optional - actual lr
@@ -812,6 +854,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
812854
}
813855
}
814856

857+
/*
858+
* Save tailcall count pointer at the same offset on the
859+
* stack where subprogs expect it
860+
*/
861+
if ((flags & BPF_TRAMP_F_CALL_ORIG) &&
862+
(flags & BPF_TRAMP_F_TAIL_CALL_CTX))
863+
bpf_frame_size += BPF_PPC_TAILCALL;
864+
815865
/* Padding to align stack frame, if any */
816866
bpf_frame_size = round_up(bpf_frame_size, SZL * 2);
817867

@@ -913,7 +963,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
913963

914964
/* Replicate tail_call_cnt before calling the original BPF prog */
915965
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
916-
bpf_trampoline_setup_tail_call_cnt(image, ctx, func_frame_offset, r4_off);
966+
bpf_trampoline_setup_tail_call_info(image, ctx, func_frame_offset,
967+
bpf_dummy_frame_size, r4_off);
917968

918969
/* Restore args */
919970
bpf_trampoline_restore_args_stack(image, ctx, func_frame_offset, nr_regs, regs_off);

‎arch/powerpc/net/bpf_jit_comp64.c‎

Lines changed: 52 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,19 @@
2626
* Ensure the top half (upto local_tmp_var) stays consistent
2727
* with our redzone usage.
2828
*
29+
* tail_call_info - stores tailcall count value in main program's
30+
* frame, stores reference to tail_call_info of
31+
* main's frame in sub-prog's frame.
32+
*
2933
* [ prev sp ] <-------------
30-
* [ tail_call_cnt ] 8 |
34+
* [ tail_call_info ] 8 |
3135
* [ nv gpr save area ] 6*8 |
3236
* [ local_tmp_var ] 24 |
3337
* fp (r31) --> [ ebpf stack space ] upto 512 |
3438
* [ frame header ] 32/112 |
3539
* sp (r1) ---> [ stack pointer ] --------------
3640
*/
3741

38-
/* for gpr non volatile registers BPG_REG_6 to 10 */
39-
#define BPF_PPC_STACK_SAVE (6*8)
4042
/* for bpf JIT code internal usage */
4143
#define BPF_PPC_STACK_LOCALS 24
4244
/* stack frame excluding BPF stack, ensure this is quadword aligned */
@@ -98,7 +100,7 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
98100
* [ prev sp ] <-------------
99101
* [ ... ] |
100102
* sp (r1) ---> [ stack pointer ] --------------
101-
* [ tail_call_cnt ] 8
103+
* [ tail_call_info ] 8
102104
* [ nv gpr save area ] 6*8
103105
* [ local_tmp_var ] 24
104106
* [ unused red zone ] 224
@@ -114,7 +116,7 @@ static int bpf_jit_stack_local(struct codegen_context *ctx)
114116
}
115117
}
116118

117-
static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)
119+
int bpf_jit_stack_tailcallinfo_offset(struct codegen_context *ctx)
118120
{
119121
return bpf_jit_stack_local(ctx) + BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE;
120122
}
@@ -147,17 +149,32 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
147149
#endif
148150

149151
/*
150-
* Initialize tail_call_cnt if we do tail calls.
151-
* Otherwise, put in NOPs so that it can be skipped when we are
152-
* invoked through a tail call.
152+
* Tail call count(tcc) is saved & updated only in main
153+
* program's frame and the address of tcc in main program's
154+
* frame (tcc_ptr) is saved in subprogs frame.
155+
*
156+
* Offset of tail_call_info on any frame will be interpreted
157+
* as either tcc_ptr or tcc value depending on whether it is
158+
* greater than MAX_TAIL_CALL_CNT or not.
153159
*/
154-
if (ctx->seen & SEEN_TAILCALL) {
160+
if (!ctx->is_subprog) {
155161
EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0));
156162
/* this goes in the redzone */
157163
EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL)));
158164
} else {
159-
EMIT(PPC_RAW_NOP());
160-
EMIT(PPC_RAW_NOP());
165+
/*
166+
* if tail_call_info < MAX_TAIL_CALL_CNT
167+
* main prog calling first subprog -> copy reference
168+
* else
169+
* subsequent subprog calling another subprog -> directly copy content
170+
*/
171+
EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), _R1, 0));
172+
EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), -(BPF_PPC_TAILCALL)));
173+
EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT));
174+
PPC_BCC_CONST_SHORT(COND_GT, 8);
175+
EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2),
176+
-(BPF_PPC_TAILCALL)));
177+
EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL)));
161178
}
162179

163180
if (bpf_has_stack_frame(ctx)) {
@@ -352,19 +369,38 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o
352369
EMIT(PPC_RAW_CMPLW(b2p_index, bpf_to_ppc(TMP_REG_1)));
353370
PPC_BCC_SHORT(COND_GE, out);
354371

372+
EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallinfo_offset(ctx)));
373+
EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT));
374+
PPC_BCC_CONST_SHORT(COND_LE, 8);
375+
376+
/* dereference TMP_REG_1 */
377+
EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 0));
378+
355379
/*
356-
* if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
380+
* if (tail_call_info == MAX_TAIL_CALL_CNT)
357381
* goto out;
358382
*/
359-
EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx)));
360383
EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT));
361-
PPC_BCC_SHORT(COND_GE, out);
384+
PPC_BCC_SHORT(COND_EQ, out);
362385

363386
/*
364-
* tail_call_cnt++;
387+
* tail_call_info++; <- Actual value of tcc here
365388
*/
366389
EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 1));
367-
EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx)));
390+
391+
/*
392+
* Before writing updated tail_call_info, distinguish if current frame
393+
* is storing a reference to tail_call_info or actual tcc value in
394+
* tail_call_info.
395+
*/
396+
EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), _R1, bpf_jit_stack_tailcallinfo_offset(ctx)));
397+
EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_2), MAX_TAIL_CALL_CNT));
398+
PPC_BCC_CONST_SHORT(COND_GT, 8);
399+
400+
/* First get address of tail_call_info */
401+
EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2), _R1, bpf_jit_stack_tailcallinfo_offset(ctx)));
402+
/* Writeback updated value to tail_call_info */
403+
EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), 0));
368404

369405
/* prog = array->ptrs[index]; */
370406
EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_1), b2p_index, 8));

0 commit comments

Comments
 (0)