Skip to content

Commit cf67d28

Browse files
author
Alexei Starovoitov
committed
Merge branch 'implement-cpuv4-support-for-s390x'
Ilya Leoshkevich says: ==================== Implement cpuv4 support for s390x v1: https://lore.kernel.org/bpf/20230830011128.1415752-1-iii@linux.ibm.com/ v1 -> v2: - Redo Disable zero-extension for BPF_MEMSX as Puranjay and Alexei suggested. - Drop the bpf_ct_insert_entry() patch, it went in via the bpf tree. - Rebase, don't apply A-bs because there were fixed conflicts. Hi, This series adds the cpuv4 support to the s390x eBPF JIT. Patches 1-3 are preliminary bugfixes. Patches 4-8 implement the new instructions. Patches 9-10 enable the tests. Best regards, Ilya ==================== Link: https://lore.kernel.org/r/20230919101336.2223655-1-iii@linux.ibm.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2 parents e9cbc89 + c29913b commit cf67d28

10 files changed

Lines changed: 331 additions & 164 deletions

File tree

‎arch/s390/net/bpf_jit_comp.c‎

Lines changed: 202 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -670,15 +670,18 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
670670
static int get_probe_mem_regno(const u8 *insn)
671671
{
672672
/*
673-
* insn must point to llgc, llgh, llgf or lg, which have destination
674-
* register at the same position.
673+
* insn must point to llgc, llgh, llgf, lg, lgb, lgh or lgf, which have
674+
* destination register at the same position.
675675
*/
676-
if (insn[0] != 0xe3) /* common llgc, llgh, llgf and lg prefix */
676+
if (insn[0] != 0xe3) /* common prefix */
677677
return -1;
678678
if (insn[5] != 0x90 && /* llgc */
679679
insn[5] != 0x91 && /* llgh */
680680
insn[5] != 0x16 && /* llgf */
681-
insn[5] != 0x04) /* lg */
681+
insn[5] != 0x04 && /* lg */
682+
insn[5] != 0x77 && /* lgb */
683+
insn[5] != 0x15 && /* lgh */
684+
insn[5] != 0x14) /* lgf */
682685
return -1;
683686
return insn[1] >> 4;
684687
}
@@ -776,6 +779,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
776779
int i, bool extra_pass, u32 stack_depth)
777780
{
778781
struct bpf_insn *insn = &fp->insnsi[i];
782+
s16 branch_oc_off = insn->off;
779783
u32 dst_reg = insn->dst_reg;
780784
u32 src_reg = insn->src_reg;
781785
int last, insn_count = 1;
@@ -788,22 +792,55 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
788792
int err;
789793

790794
if (BPF_CLASS(insn->code) == BPF_LDX &&
791-
BPF_MODE(insn->code) == BPF_PROBE_MEM)
795+
(BPF_MODE(insn->code) == BPF_PROBE_MEM ||
796+
BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
792797
probe_prg = jit->prg;
793798

794799
switch (insn->code) {
795800
/*
796801
* BPF_MOV
797802
*/
798-
case BPF_ALU | BPF_MOV | BPF_X: /* dst = (u32) src */
799-
/* llgfr %dst,%src */
800-
EMIT4(0xb9160000, dst_reg, src_reg);
801-
if (insn_is_zext(&insn[1]))
802-
insn_count = 2;
803+
case BPF_ALU | BPF_MOV | BPF_X:
804+
switch (insn->off) {
805+
case 0: /* DST = (u32) SRC */
806+
/* llgfr %dst,%src */
807+
EMIT4(0xb9160000, dst_reg, src_reg);
808+
if (insn_is_zext(&insn[1]))
809+
insn_count = 2;
810+
break;
811+
case 8: /* DST = (u32)(s8) SRC */
812+
/* lbr %dst,%src */
813+
EMIT4(0xb9260000, dst_reg, src_reg);
814+
/* llgfr %dst,%dst */
815+
EMIT4(0xb9160000, dst_reg, dst_reg);
816+
break;
817+
case 16: /* DST = (u32)(s16) SRC */
818+
/* lhr %dst,%src */
819+
EMIT4(0xb9270000, dst_reg, src_reg);
820+
/* llgfr %dst,%dst */
821+
EMIT4(0xb9160000, dst_reg, dst_reg);
822+
break;
823+
}
803824
break;
804-
case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
805-
/* lgr %dst,%src */
806-
EMIT4(0xb9040000, dst_reg, src_reg);
825+
case BPF_ALU64 | BPF_MOV | BPF_X:
826+
switch (insn->off) {
827+
case 0: /* DST = SRC */
828+
/* lgr %dst,%src */
829+
EMIT4(0xb9040000, dst_reg, src_reg);
830+
break;
831+
case 8: /* DST = (s8) SRC */
832+
/* lgbr %dst,%src */
833+
EMIT4(0xb9060000, dst_reg, src_reg);
834+
break;
835+
case 16: /* DST = (s16) SRC */
836+
/* lghr %dst,%src */
837+
EMIT4(0xb9070000, dst_reg, src_reg);
838+
break;
839+
case 32: /* DST = (s32) SRC */
840+
/* lgfr %dst,%src */
841+
EMIT4(0xb9140000, dst_reg, src_reg);
842+
break;
843+
}
807844
break;
808845
case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */
809846
/* llilf %dst,imm */
@@ -912,75 +949,124 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
912949
/*
913950
* BPF_DIV / BPF_MOD
914951
*/
915-
case BPF_ALU | BPF_DIV | BPF_X: /* dst = (u32) dst / (u32) src */
916-
case BPF_ALU | BPF_MOD | BPF_X: /* dst = (u32) dst % (u32) src */
952+
case BPF_ALU | BPF_DIV | BPF_X:
953+
case BPF_ALU | BPF_MOD | BPF_X:
917954
{
918955
int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
919956

920-
/* lhi %w0,0 */
921-
EMIT4_IMM(0xa7080000, REG_W0, 0);
922-
/* lr %w1,%dst */
923-
EMIT2(0x1800, REG_W1, dst_reg);
924-
/* dlr %w0,%src */
925-
EMIT4(0xb9970000, REG_W0, src_reg);
957+
switch (off) {
958+
case 0: /* dst = (u32) dst {/,%} (u32) src */
959+
/* xr %w0,%w0 */
960+
EMIT2(0x1700, REG_W0, REG_W0);
961+
/* lr %w1,%dst */
962+
EMIT2(0x1800, REG_W1, dst_reg);
963+
/* dlr %w0,%src */
964+
EMIT4(0xb9970000, REG_W0, src_reg);
965+
break;
966+
case 1: /* dst = (u32) ((s32) dst {/,%} (s32) src) */
967+
/* lgfr %r1,%dst */
968+
EMIT4(0xb9140000, REG_W1, dst_reg);
969+
/* dsgfr %r0,%src */
970+
EMIT4(0xb91d0000, REG_W0, src_reg);
971+
break;
972+
}
926973
/* llgfr %dst,%rc */
927974
EMIT4(0xb9160000, dst_reg, rc_reg);
928975
if (insn_is_zext(&insn[1]))
929976
insn_count = 2;
930977
break;
931978
}
932-
case BPF_ALU64 | BPF_DIV | BPF_X: /* dst = dst / src */
933-
case BPF_ALU64 | BPF_MOD | BPF_X: /* dst = dst % src */
979+
case BPF_ALU64 | BPF_DIV | BPF_X:
980+
case BPF_ALU64 | BPF_MOD | BPF_X:
934981
{
935982
int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
936983

937-
/* lghi %w0,0 */
938-
EMIT4_IMM(0xa7090000, REG_W0, 0);
939-
/* lgr %w1,%dst */
940-
EMIT4(0xb9040000, REG_W1, dst_reg);
941-
/* dlgr %w0,%dst */
942-
EMIT4(0xb9870000, REG_W0, src_reg);
984+
switch (off) {
985+
case 0: /* dst = dst {/,%} src */
986+
/* lghi %w0,0 */
987+
EMIT4_IMM(0xa7090000, REG_W0, 0);
988+
/* lgr %w1,%dst */
989+
EMIT4(0xb9040000, REG_W1, dst_reg);
990+
/* dlgr %w0,%src */
991+
EMIT4(0xb9870000, REG_W0, src_reg);
992+
break;
993+
case 1: /* dst = (s64) dst {/,%} (s64) src */
994+
/* lgr %w1,%dst */
995+
EMIT4(0xb9040000, REG_W1, dst_reg);
996+
/* dsgr %w0,%src */
997+
EMIT4(0xb90d0000, REG_W0, src_reg);
998+
break;
999+
}
9431000
/* lgr %dst,%rc */
9441001
EMIT4(0xb9040000, dst_reg, rc_reg);
9451002
break;
9461003
}
947-
case BPF_ALU | BPF_DIV | BPF_K: /* dst = (u32) dst / (u32) imm */
948-
case BPF_ALU | BPF_MOD | BPF_K: /* dst = (u32) dst % (u32) imm */
1004+
case BPF_ALU | BPF_DIV | BPF_K:
1005+
case BPF_ALU | BPF_MOD | BPF_K:
9491006
{
9501007
int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
9511008

9521009
if (imm == 1) {
9531010
if (BPF_OP(insn->code) == BPF_MOD)
954-
/* lhgi %dst,0 */
1011+
/* lghi %dst,0 */
9551012
EMIT4_IMM(0xa7090000, dst_reg, 0);
9561013
else
9571014
EMIT_ZERO(dst_reg);
9581015
break;
9591016
}
960-
/* lhi %w0,0 */
961-
EMIT4_IMM(0xa7080000, REG_W0, 0);
962-
/* lr %w1,%dst */
963-
EMIT2(0x1800, REG_W1, dst_reg);
9641017
if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {
965-
/* dl %w0,<d(imm)>(%l) */
966-
EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,
967-
EMIT_CONST_U32(imm));
1018+
switch (off) {
1019+
case 0: /* dst = (u32) dst {/,%} (u32) imm */
1020+
/* xr %w0,%w0 */
1021+
EMIT2(0x1700, REG_W0, REG_W0);
1022+
/* lr %w1,%dst */
1023+
EMIT2(0x1800, REG_W1, dst_reg);
1024+
/* dl %w0,<d(imm)>(%l) */
1025+
EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0,
1026+
REG_L, EMIT_CONST_U32(imm));
1027+
break;
1028+
case 1: /* dst = (s32) dst {/,%} (s32) imm */
1029+
/* lgfr %r1,%dst */
1030+
EMIT4(0xb9140000, REG_W1, dst_reg);
1031+
/* dsgf %r0,<d(imm)>(%l) */
1032+
EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0,
1033+
REG_L, EMIT_CONST_U32(imm));
1034+
break;
1035+
}
9681036
} else {
969-
/* lgfrl %dst,imm */
970-
EMIT6_PCREL_RILB(0xc40c0000, dst_reg,
971-
_EMIT_CONST_U32(imm));
972-
jit->seen |= SEEN_LITERAL;
973-
/* dlr %w0,%dst */
974-
EMIT4(0xb9970000, REG_W0, dst_reg);
1037+
switch (off) {
1038+
case 0: /* dst = (u32) dst {/,%} (u32) imm */
1039+
/* xr %w0,%w0 */
1040+
EMIT2(0x1700, REG_W0, REG_W0);
1041+
/* lr %w1,%dst */
1042+
EMIT2(0x1800, REG_W1, dst_reg);
1043+
/* lrl %dst,imm */
1044+
EMIT6_PCREL_RILB(0xc40d0000, dst_reg,
1045+
_EMIT_CONST_U32(imm));
1046+
jit->seen |= SEEN_LITERAL;
1047+
/* dlr %w0,%dst */
1048+
EMIT4(0xb9970000, REG_W0, dst_reg);
1049+
break;
1050+
case 1: /* dst = (s32) dst {/,%} (s32) imm */
1051+
/* lgfr %w1,%dst */
1052+
EMIT4(0xb9140000, REG_W1, dst_reg);
1053+
/* lgfrl %dst,imm */
1054+
EMIT6_PCREL_RILB(0xc40c0000, dst_reg,
1055+
_EMIT_CONST_U32(imm));
1056+
jit->seen |= SEEN_LITERAL;
1057+
/* dsgr %w0,%dst */
1058+
EMIT4(0xb90d0000, REG_W0, dst_reg);
1059+
break;
1060+
}
9751061
}
9761062
/* llgfr %dst,%rc */
9771063
EMIT4(0xb9160000, dst_reg, rc_reg);
9781064
if (insn_is_zext(&insn[1]))
9791065
insn_count = 2;
9801066
break;
9811067
}
982-
case BPF_ALU64 | BPF_DIV | BPF_K: /* dst = dst / imm */
983-
case BPF_ALU64 | BPF_MOD | BPF_K: /* dst = dst % imm */
1068+
case BPF_ALU64 | BPF_DIV | BPF_K:
1069+
case BPF_ALU64 | BPF_MOD | BPF_K:
9841070
{
9851071
int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
9861072

@@ -990,21 +1076,50 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
9901076
EMIT4_IMM(0xa7090000, dst_reg, 0);
9911077
break;
9921078
}
993-
/* lghi %w0,0 */
994-
EMIT4_IMM(0xa7090000, REG_W0, 0);
995-
/* lgr %w1,%dst */
996-
EMIT4(0xb9040000, REG_W1, dst_reg);
9971079
if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
998-
/* dlg %w0,<d(imm)>(%l) */
999-
EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,
1000-
EMIT_CONST_U64(imm));
1080+
switch (off) {
1081+
case 0: /* dst = dst {/,%} imm */
1082+
/* lghi %w0,0 */
1083+
EMIT4_IMM(0xa7090000, REG_W0, 0);
1084+
/* lgr %w1,%dst */
1085+
EMIT4(0xb9040000, REG_W1, dst_reg);
1086+
/* dlg %w0,<d(imm)>(%l) */
1087+
EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0,
1088+
REG_L, EMIT_CONST_U64(imm));
1089+
break;
1090+
case 1: /* dst = (s64) dst {/,%} (s64) imm */
1091+
/* lgr %w1,%dst */
1092+
EMIT4(0xb9040000, REG_W1, dst_reg);
1093+
/* dsg %w0,<d(imm)>(%l) */
1094+
EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0,
1095+
REG_L, EMIT_CONST_U64(imm));
1096+
break;
1097+
}
10011098
} else {
1002-
/* lgrl %dst,imm */
1003-
EMIT6_PCREL_RILB(0xc4080000, dst_reg,
1004-
_EMIT_CONST_U64(imm));
1005-
jit->seen |= SEEN_LITERAL;
1006-
/* dlgr %w0,%dst */
1007-
EMIT4(0xb9870000, REG_W0, dst_reg);
1099+
switch (off) {
1100+
case 0: /* dst = dst {/,%} imm */
1101+
/* lghi %w0,0 */
1102+
EMIT4_IMM(0xa7090000, REG_W0, 0);
1103+
/* lgr %w1,%dst */
1104+
EMIT4(0xb9040000, REG_W1, dst_reg);
1105+
/* lgrl %dst,imm */
1106+
EMIT6_PCREL_RILB(0xc4080000, dst_reg,
1107+
_EMIT_CONST_U64(imm));
1108+
jit->seen |= SEEN_LITERAL;
1109+
/* dlgr %w0,%dst */
1110+
EMIT4(0xb9870000, REG_W0, dst_reg);
1111+
break;
1112+
case 1: /* dst = (s64) dst {/,%} (s64) imm */
1113+
/* lgr %w1,%dst */
1114+
EMIT4(0xb9040000, REG_W1, dst_reg);
1115+
/* lgrl %dst,imm */
1116+
EMIT6_PCREL_RILB(0xc4080000, dst_reg,
1117+
_EMIT_CONST_U64(imm));
1118+
jit->seen |= SEEN_LITERAL;
1119+
/* dsgr %w0,%dst */
1120+
EMIT4(0xb90d0000, REG_W0, dst_reg);
1121+
break;
1122+
}
10081123
}
10091124
/* lgr %dst,%rc */
10101125
EMIT4(0xb9040000, dst_reg, rc_reg);
@@ -1217,6 +1332,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
12171332
}
12181333
break;
12191334
case BPF_ALU | BPF_END | BPF_FROM_LE:
1335+
case BPF_ALU64 | BPF_END | BPF_FROM_LE:
12201336
switch (imm) {
12211337
case 16: /* dst = (u16) cpu_to_le16(dst) */
12221338
/* lrvr %dst,%dst */
@@ -1374,6 +1490,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
13741490
if (insn_is_zext(&insn[1]))
13751491
insn_count = 2;
13761492
break;
1493+
case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */
1494+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1495+
/* lgb %dst,0(off,%src) */
1496+
EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off);
1497+
jit->seen |= SEEN_MEM;
1498+
break;
13771499
case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
13781500
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
13791501
/* llgh %dst,0(off,%src) */
@@ -1382,6 +1504,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
13821504
if (insn_is_zext(&insn[1]))
13831505
insn_count = 2;
13841506
break;
1507+
case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */
1508+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1509+
/* lgh %dst,0(off,%src) */
1510+
EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off);
1511+
jit->seen |= SEEN_MEM;
1512+
break;
13851513
case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
13861514
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
13871515
/* llgf %dst,off(%src) */
@@ -1390,6 +1518,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
13901518
if (insn_is_zext(&insn[1]))
13911519
insn_count = 2;
13921520
break;
1521+
case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */
1522+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1523+
/* lgf %dst,off(%src) */
1524+
jit->seen |= SEEN_MEM;
1525+
EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off);
1526+
break;
13931527
case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
13941528
case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
13951529
/* lg %dst,0(off,%src) */
@@ -1570,6 +1704,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
15701704
* instruction itself (loop) and for BPF with offset 0 we
15711705
* branch to the instruction behind the branch.
15721706
*/
1707+
case BPF_JMP32 | BPF_JA: /* if (true) */
1708+
branch_oc_off = imm;
1709+
fallthrough;
15731710
case BPF_JMP | BPF_JA: /* if (true) */
15741711
mask = 0xf000; /* j */
15751712
goto branch_oc;
@@ -1738,14 +1875,16 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
17381875
break;
17391876
branch_oc:
17401877
if (!is_first_pass(jit) &&
1741-
can_use_rel(jit, addrs[i + off + 1])) {
1878+
can_use_rel(jit, addrs[i + branch_oc_off + 1])) {
17421879
/* brc mask,off */
17431880
EMIT4_PCREL_RIC(0xa7040000,
1744-
mask >> 12, addrs[i + off + 1]);
1881+
mask >> 12,
1882+
addrs[i + branch_oc_off + 1]);
17451883
} else {
17461884
/* brcl mask,off */
17471885
EMIT6_PCREL_RILC(0xc0040000,
1748-
mask >> 12, addrs[i + off + 1]);
1886+
mask >> 12,
1887+
addrs[i + branch_oc_off + 1]);
17491888
}
17501889
break;
17511890
}

0 commit comments

Comments
 (0)