Skip to content

Commit 91d2ad7

Browse files
iii-iAlexei Starovoitov
iii-i
authored and
Alexei Starovoitov
committed
s390/bpf: Implement signed division
Implement the cpuv4 signed division. It is encoded as unsigned division, but with off field set to 1. s390x has the necessary instructions: dsgfr, dsgf and dsgr. Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com> Link: https://lore.kernel.org/r/20230919101336.2223655-9-iii@linux.ibm.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
1 parent c690191 commit 91d2ad7

1 file changed

Lines changed: 125 additions & 47 deletions

File tree

‎arch/s390/net/bpf_jit_comp.c‎

Lines changed: 125 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -949,75 +949,124 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
949949
/*
950950
* BPF_DIV / BPF_MOD
951951
*/
952-
case BPF_ALU | BPF_DIV | BPF_X: /* dst = (u32) dst / (u32) src */
953-
case BPF_ALU | BPF_MOD | BPF_X: /* dst = (u32) dst % (u32) src */
952+
case BPF_ALU | BPF_DIV | BPF_X:
953+
case BPF_ALU | BPF_MOD | BPF_X:
954954
{
955955
int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
956956

957-
/* lhi %w0,0 */
958-
EMIT4_IMM(0xa7080000, REG_W0, 0);
959-
/* lr %w1,%dst */
960-
EMIT2(0x1800, REG_W1, dst_reg);
961-
/* dlr %w0,%src */
962-
EMIT4(0xb9970000, REG_W0, src_reg);
957+
switch (off) {
958+
case 0: /* dst = (u32) dst {/,%} (u32) src */
959+
/* xr %w0,%w0 */
960+
EMIT2(0x1700, REG_W0, REG_W0);
961+
/* lr %w1,%dst */
962+
EMIT2(0x1800, REG_W1, dst_reg);
963+
/* dlr %w0,%src */
964+
EMIT4(0xb9970000, REG_W0, src_reg);
965+
break;
966+
case 1: /* dst = (u32) ((s32) dst {/,%} (s32) src) */
967+
/* lgfr %r1,%dst */
968+
EMIT4(0xb9140000, REG_W1, dst_reg);
969+
/* dsgfr %r0,%src */
970+
EMIT4(0xb91d0000, REG_W0, src_reg);
971+
break;
972+
}
963973
/* llgfr %dst,%rc */
964974
EMIT4(0xb9160000, dst_reg, rc_reg);
965975
if (insn_is_zext(&insn[1]))
966976
insn_count = 2;
967977
break;
968978
}
969-
case BPF_ALU64 | BPF_DIV | BPF_X: /* dst = dst / src */
970-
case BPF_ALU64 | BPF_MOD | BPF_X: /* dst = dst % src */
979+
case BPF_ALU64 | BPF_DIV | BPF_X:
980+
case BPF_ALU64 | BPF_MOD | BPF_X:
971981
{
972982
int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
973983

974-
/* lghi %w0,0 */
975-
EMIT4_IMM(0xa7090000, REG_W0, 0);
976-
/* lgr %w1,%dst */
977-
EMIT4(0xb9040000, REG_W1, dst_reg);
978-
/* dlgr %w0,%dst */
979-
EMIT4(0xb9870000, REG_W0, src_reg);
984+
switch (off) {
985+
case 0: /* dst = dst {/,%} src */
986+
/* lghi %w0,0 */
987+
EMIT4_IMM(0xa7090000, REG_W0, 0);
988+
/* lgr %w1,%dst */
989+
EMIT4(0xb9040000, REG_W1, dst_reg);
990+
/* dlgr %w0,%src */
991+
EMIT4(0xb9870000, REG_W0, src_reg);
992+
break;
993+
case 1: /* dst = (s64) dst {/,%} (s64) src */
994+
/* lgr %w1,%dst */
995+
EMIT4(0xb9040000, REG_W1, dst_reg);
996+
/* dsgr %w0,%src */
997+
EMIT4(0xb90d0000, REG_W0, src_reg);
998+
break;
999+
}
9801000
/* lgr %dst,%rc */
9811001
EMIT4(0xb9040000, dst_reg, rc_reg);
9821002
break;
9831003
}
984-
case BPF_ALU | BPF_DIV | BPF_K: /* dst = (u32) dst / (u32) imm */
985-
case BPF_ALU | BPF_MOD | BPF_K: /* dst = (u32) dst % (u32) imm */
1004+
case BPF_ALU | BPF_DIV | BPF_K:
1005+
case BPF_ALU | BPF_MOD | BPF_K:
9861006
{
9871007
int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
9881008

9891009
if (imm == 1) {
9901010
if (BPF_OP(insn->code) == BPF_MOD)
991-
/* lhgi %dst,0 */
1011+
/* lghi %dst,0 */
9921012
EMIT4_IMM(0xa7090000, dst_reg, 0);
9931013
else
9941014
EMIT_ZERO(dst_reg);
9951015
break;
9961016
}
997-
/* lhi %w0,0 */
998-
EMIT4_IMM(0xa7080000, REG_W0, 0);
999-
/* lr %w1,%dst */
1000-
EMIT2(0x1800, REG_W1, dst_reg);
10011017
if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {
1002-
/* dl %w0,<d(imm)>(%l) */
1003-
EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,
1004-
EMIT_CONST_U32(imm));
1018+
switch (off) {
1019+
case 0: /* dst = (u32) dst {/,%} (u32) imm */
1020+
/* xr %w0,%w0 */
1021+
EMIT2(0x1700, REG_W0, REG_W0);
1022+
/* lr %w1,%dst */
1023+
EMIT2(0x1800, REG_W1, dst_reg);
1024+
/* dl %w0,<d(imm)>(%l) */
1025+
EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0,
1026+
REG_L, EMIT_CONST_U32(imm));
1027+
break;
1028+
case 1: /* dst = (s32) dst {/,%} (s32) imm */
1029+
/* lgfr %r1,%dst */
1030+
EMIT4(0xb9140000, REG_W1, dst_reg);
1031+
/* dsgf %r0,<d(imm)>(%l) */
1032+
EMIT6_DISP_LH(0xe3000000, 0x001d, REG_W0, REG_0,
1033+
REG_L, EMIT_CONST_U32(imm));
1034+
break;
1035+
}
10051036
} else {
1006-
/* lgfrl %dst,imm */
1007-
EMIT6_PCREL_RILB(0xc40c0000, dst_reg,
1008-
_EMIT_CONST_U32(imm));
1009-
jit->seen |= SEEN_LITERAL;
1010-
/* dlr %w0,%dst */
1011-
EMIT4(0xb9970000, REG_W0, dst_reg);
1037+
switch (off) {
1038+
case 0: /* dst = (u32) dst {/,%} (u32) imm */
1039+
/* xr %w0,%w0 */
1040+
EMIT2(0x1700, REG_W0, REG_W0);
1041+
/* lr %w1,%dst */
1042+
EMIT2(0x1800, REG_W1, dst_reg);
1043+
/* lrl %dst,imm */
1044+
EMIT6_PCREL_RILB(0xc40d0000, dst_reg,
1045+
_EMIT_CONST_U32(imm));
1046+
jit->seen |= SEEN_LITERAL;
1047+
/* dlr %w0,%dst */
1048+
EMIT4(0xb9970000, REG_W0, dst_reg);
1049+
break;
1050+
case 1: /* dst = (s32) dst {/,%} (s32) imm */
1051+
/* lgfr %w1,%dst */
1052+
EMIT4(0xb9140000, REG_W1, dst_reg);
1053+
/* lgfrl %dst,imm */
1054+
EMIT6_PCREL_RILB(0xc40c0000, dst_reg,
1055+
_EMIT_CONST_U32(imm));
1056+
jit->seen |= SEEN_LITERAL;
1057+
/* dsgr %w0,%dst */
1058+
EMIT4(0xb90d0000, REG_W0, dst_reg);
1059+
break;
1060+
}
10121061
}
10131062
/* llgfr %dst,%rc */
10141063
EMIT4(0xb9160000, dst_reg, rc_reg);
10151064
if (insn_is_zext(&insn[1]))
10161065
insn_count = 2;
10171066
break;
10181067
}
1019-
case BPF_ALU64 | BPF_DIV | BPF_K: /* dst = dst / imm */
1020-
case BPF_ALU64 | BPF_MOD | BPF_K: /* dst = dst % imm */
1068+
case BPF_ALU64 | BPF_DIV | BPF_K:
1069+
case BPF_ALU64 | BPF_MOD | BPF_K:
10211070
{
10221071
int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
10231072

@@ -1027,21 +1076,50 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
10271076
EMIT4_IMM(0xa7090000, dst_reg, 0);
10281077
break;
10291078
}
1030-
/* lghi %w0,0 */
1031-
EMIT4_IMM(0xa7090000, REG_W0, 0);
1032-
/* lgr %w1,%dst */
1033-
EMIT4(0xb9040000, REG_W1, dst_reg);
10341079
if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1035-
/* dlg %w0,<d(imm)>(%l) */
1036-
EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,
1037-
EMIT_CONST_U64(imm));
1080+
switch (off) {
1081+
case 0: /* dst = dst {/,%} imm */
1082+
/* lghi %w0,0 */
1083+
EMIT4_IMM(0xa7090000, REG_W0, 0);
1084+
/* lgr %w1,%dst */
1085+
EMIT4(0xb9040000, REG_W1, dst_reg);
1086+
/* dlg %w0,<d(imm)>(%l) */
1087+
EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0,
1088+
REG_L, EMIT_CONST_U64(imm));
1089+
break;
1090+
case 1: /* dst = (s64) dst {/,%} (s64) imm */
1091+
/* lgr %w1,%dst */
1092+
EMIT4(0xb9040000, REG_W1, dst_reg);
1093+
/* dsg %w0,<d(imm)>(%l) */
1094+
EMIT6_DISP_LH(0xe3000000, 0x000d, REG_W0, REG_0,
1095+
REG_L, EMIT_CONST_U64(imm));
1096+
break;
1097+
}
10381098
} else {
1039-
/* lgrl %dst,imm */
1040-
EMIT6_PCREL_RILB(0xc4080000, dst_reg,
1041-
_EMIT_CONST_U64(imm));
1042-
jit->seen |= SEEN_LITERAL;
1043-
/* dlgr %w0,%dst */
1044-
EMIT4(0xb9870000, REG_W0, dst_reg);
1099+
switch (off) {
1100+
case 0: /* dst = dst {/,%} imm */
1101+
/* lghi %w0,0 */
1102+
EMIT4_IMM(0xa7090000, REG_W0, 0);
1103+
/* lgr %w1,%dst */
1104+
EMIT4(0xb9040000, REG_W1, dst_reg);
1105+
/* lgrl %dst,imm */
1106+
EMIT6_PCREL_RILB(0xc4080000, dst_reg,
1107+
_EMIT_CONST_U64(imm));
1108+
jit->seen |= SEEN_LITERAL;
1109+
/* dlgr %w0,%dst */
1110+
EMIT4(0xb9870000, REG_W0, dst_reg);
1111+
break;
1112+
case 1: /* dst = (s64) dst {/,%} (s64) imm */
1113+
/* lgr %w1,%dst */
1114+
EMIT4(0xb9040000, REG_W1, dst_reg);
1115+
/* lgrl %dst,imm */
1116+
EMIT6_PCREL_RILB(0xc4080000, dst_reg,
1117+
_EMIT_CONST_U64(imm));
1118+
jit->seen |= SEEN_LITERAL;
1119+
/* dsgr %w0,%dst */
1120+
EMIT4(0xb90d0000, REG_W0, dst_reg);
1121+
break;
1122+
}
10451123
}
10461124
/* lgr %dst,%rc */
10471125
EMIT4(0xb9040000, dst_reg, rc_reg);

0 commit comments

Comments
 (0)