kernel_5.10 no rt

hc

2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f

 kernel/arch/arm/net/bpf_jit_32.c
..
..
@@ -1,12 +1,9 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  * Just-In-Time compiler for eBPF filters on 32bit ARM
3
4
  *
4
5
  * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
5
6
  * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
6
- *
7
- * This program is free software; you can redistribute it and/or modify it
8
- * under the terms of the GNU General Public License as published by the
9
- * Free Software Foundation; version 2 of the License.
10
7
  */
11
8
 
12
9
 #include <linux/bpf.h>
..
..
@@ -755,7 +752,8 @@
755
752
 
756
753
 		/* ALU operation */
757
754
 		emit_alu_r(rd[1], rs, true, false, op, ctx);
758
-		emit_a32_mov_i(rd[0], 0, ctx);
755
+		if (!ctx->prog->aux->verifier_zext)
756
+			emit_a32_mov_i(rd[0], 0, ctx);
759
757
 	}
760
758
 
761
759
 	arm_bpf_put_reg64(dst, rd, ctx);
..
..
@@ -777,8 +775,9 @@
777
775
 				  struct jit_ctx *ctx) {
778
776
 	if (!is64) {
779
777
 		emit_a32_mov_r(dst_lo, src_lo, ctx);
780
-		/* Zero out high 4 bytes */
781
-		emit_a32_mov_i(dst_hi, 0, ctx);
778
+		if (!ctx->prog->aux->verifier_zext)
779
+			/* Zero out high 4 bytes */
780
+			emit_a32_mov_i(dst_hi, 0, ctx);
782
781
 	} else if (__LINUX_ARM_ARCH__ < 6 &&
783
782
 		   ctx->cpu_architecture < CPU_ARCH_ARMv5TE) {
784
783
 		/* complete 8 byte move */
..
..
@@ -814,6 +813,9 @@
814
813
 		break;
815
814
 	case BPF_RSH:
816
815
 		emit(ARM_LSR_I(rd, rd, val), ctx);
816
+		break;
817
+	case BPF_ARSH:
818
+		emit(ARM_ASR_I(rd, rd, val), ctx);
817
819
 		break;
818
820
 	case BPF_NEG:
819
821
 		emit(ARM_RSB_I(rd, rd, val), ctx);
..
..
@@ -880,8 +882,8 @@
880
882
 	emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
881
883
 	emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);
882
884
 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);
883
-	_emit(ARM_COND_MI, ARM_B(0), ctx);
884
-	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASR, tmp2[0]), ctx);
885
+	_emit(ARM_COND_PL,
886
+	      ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASR, tmp2[0]), ctx);
885
887
 	emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx);
886
888
 
887
889
 	arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
..
..
@@ -1095,17 +1097,20 @@
1095
1097
 	case BPF_B:
1096
1098
 		/* Load a Byte */
1097
1099
 		emit(ARM_LDRB_I(rd[1], rm, off), ctx);
1098
-		emit_a32_mov_i(rd[0], 0, ctx);
1100
+		if (!ctx->prog->aux->verifier_zext)
1101
+			emit_a32_mov_i(rd[0], 0, ctx);
1099
1102
 		break;
1100
1103
 	case BPF_H:
1101
1104
 		/* Load a HalfWord */
1102
1105
 		emit(ARM_LDRH_I(rd[1], rm, off), ctx);
1103
-		emit_a32_mov_i(rd[0], 0, ctx);
1106
+		if (!ctx->prog->aux->verifier_zext)
1107
+			emit_a32_mov_i(rd[0], 0, ctx);
1104
1108
 		break;
1105
1109
 	case BPF_W:
1106
1110
 		/* Load a Word */
1107
1111
 		emit(ARM_LDR_I(rd[1], rm, off), ctx);
1108
-		emit_a32_mov_i(rd[0], 0, ctx);
1112
+		if (!ctx->prog->aux->verifier_zext)
1113
+			emit_a32_mov_i(rd[0], 0, ctx);
1109
1114
 		break;
1110
1115
 	case BPF_DW:
1111
1116
 		/* Load a Double Word */
..
..
@@ -1118,12 +1123,17 @@
1118
1123
 
1119
1124
 /* Arithmatic Operation */
1120
1125
 static inline void emit_ar_r(const u8 rd, const u8 rt, const u8 rm,
1121
-			     const u8 rn, struct jit_ctx *ctx, u8 op) {
1126
+			     const u8 rn, struct jit_ctx *ctx, u8 op,
1127
+			     bool is_jmp64) {
1122
1128
 	switch (op) {
1123
1129
 	case BPF_JSET:
1124
-		emit(ARM_AND_R(ARM_IP, rt, rn), ctx);
1125
-		emit(ARM_AND_R(ARM_LR, rd, rm), ctx);
1126
-		emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx);
1130
+		if (is_jmp64) {
1131
+			emit(ARM_AND_R(ARM_IP, rt, rn), ctx);
1132
+			emit(ARM_AND_R(ARM_LR, rd, rm), ctx);
1133
+			emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx);
1134
+		} else {
1135
+			emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx);
1136
+		}
1127
1137
 		break;
1128
1138
 	case BPF_JEQ:
1129
1139
 	case BPF_JNE:
..
..
@@ -1131,18 +1141,25 @@
1131
1141
 	case BPF_JGE:
1132
1142
 	case BPF_JLE:
1133
1143
 	case BPF_JLT:
1134
-		emit(ARM_CMP_R(rd, rm), ctx);
1135
-		_emit(ARM_COND_EQ, ARM_CMP_R(rt, rn), ctx);
1144
+		if (is_jmp64) {
1145
+			emit(ARM_CMP_R(rd, rm), ctx);
1146
+			/* Only compare low halve if high halve are equal. */
1147
+			_emit(ARM_COND_EQ, ARM_CMP_R(rt, rn), ctx);
1148
+		} else {
1149
+			emit(ARM_CMP_R(rt, rn), ctx);
1150
+		}
1136
1151
 		break;
1137
1152
 	case BPF_JSLE:
1138
1153
 	case BPF_JSGT:
1139
1154
 		emit(ARM_CMP_R(rn, rt), ctx);
1140
-		emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx);
1155
+		if (is_jmp64)
1156
+			emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx);
1141
1157
 		break;
1142
1158
 	case BPF_JSLT:
1143
1159
 	case BPF_JSGE:
1144
1160
 		emit(ARM_CMP_R(rt, rn), ctx);
1145
-		emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx);
1161
+		if (is_jmp64)
1162
+			emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx);
1146
1163
 		break;
1147
1164
 	}
1148
1165
 }
..
..
@@ -1281,12 +1298,9 @@
1281
1298
 
1282
1299
 static void build_prologue(struct jit_ctx *ctx)
1283
1300
 {
1284
-	const s8 r0 = bpf2a32[BPF_REG_0][1];
1285
-	const s8 r2 = bpf2a32[BPF_REG_1][1];
1286
-	const s8 r3 = bpf2a32[BPF_REG_1][0];
1287
-	const s8 r4 = bpf2a32[BPF_REG_6][1];
1288
-	const s8 fplo = bpf2a32[BPF_REG_FP][1];
1289
-	const s8 fphi = bpf2a32[BPF_REG_FP][0];
1301
+	const s8 arm_r0 = bpf2a32[BPF_REG_0][1];
1302
+	const s8 *bpf_r1 = bpf2a32[BPF_REG_1];
1303
+	const s8 *bpf_fp = bpf2a32[BPF_REG_FP];
1290
1304
 	const s8 *tcc = bpf2a32[TCALL_CNT];
1291
1305
 
1292
1306
 	/* Save callee saved registers. */
..
..
@@ -1299,8 +1313,10 @@
1299
1313
 	emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx);
1300
1314
 	emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx);
1301
1315
 #endif
1302
-	/* Save frame pointer for later */
1303
-	emit(ARM_SUB_I(ARM_IP, ARM_SP, SCRATCH_SIZE), ctx);
1316
+	/* mov r3, #0 */
1317
+	/* sub r2, sp, #SCRATCH_SIZE */
1318
+	emit(ARM_MOV_I(bpf_r1[0], 0), ctx);
1319
+	emit(ARM_SUB_I(bpf_r1[1], ARM_SP, SCRATCH_SIZE), ctx);
1304
1320
 
1305
1321
 	ctx->stack_size = imm8m(STACK_SIZE);
1306
1322
 
..
..
@@ -1308,18 +1324,15 @@
1308
1324
 	emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx);
1309
1325
 
1310
1326
 	/* Set up BPF prog stack base register */
1311
-	emit_a32_mov_r(fplo, ARM_IP, ctx);
1312
-	emit_a32_mov_i(fphi, 0, ctx);
1327
+	emit_a32_mov_r64(true, bpf_fp, bpf_r1, ctx);
1313
1328
 
1314
-	/* mov r4, 0 */
1315
-	emit(ARM_MOV_I(r4, 0), ctx);
1329
+	/* Initialize Tail Count */
1330
+	emit(ARM_MOV_I(bpf_r1[1], 0), ctx);
1331
+	emit_a32_mov_r64(true, tcc, bpf_r1, ctx);
1316
1332
 
1317
1333
 	/* Move BPF_CTX to BPF_R1 */
1318
-	emit(ARM_MOV_R(r3, r4), ctx);
1319
-	emit(ARM_MOV_R(r2, r0), ctx);
1320
-	/* Initialize Tail Count */
1321
-	emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[0])), ctx);
1322
-	emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[1])), ctx);
1334
+	emit(ARM_MOV_R(bpf_r1[1], arm_r0), ctx);
1335
+
1323
1336
 	/* end of prologue */
1324
1337
 }
1325
1338
 
..
..
@@ -1382,6 +1395,11 @@
1382
1395
 	case BPF_ALU64 | BPF_MOV | BPF_X:
1383
1396
 		switch (BPF_SRC(code)) {
1384
1397
 		case BPF_X:
1398
+			if (imm == 1) {
1399
+				/* Special mov32 for zext */
1400
+				emit_a32_mov_i(dst_hi, 0, ctx);
1401
+				break;
1402
+			}
1385
1403
 			emit_a32_mov_r64(is64, dst, src, ctx);
1386
1404
 			break;
1387
1405
 		case BPF_K:
..
..
@@ -1412,7 +1430,6 @@
1412
1430
 	case BPF_ALU | BPF_MUL | BPF_X:
1413
1431
 	case BPF_ALU | BPF_LSH | BPF_X:
1414
1432
 	case BPF_ALU | BPF_RSH | BPF_X:
1415
-	case BPF_ALU | BPF_ARSH | BPF_K:
1416
1433
 	case BPF_ALU | BPF_ARSH | BPF_X:
1417
1434
 	case BPF_ALU64 | BPF_ADD | BPF_K:
1418
1435
 	case BPF_ALU64 | BPF_ADD | BPF_X:
..
..
@@ -1461,22 +1478,26 @@
1461
1478
 		}
1462
1479
 		emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code));
1463
1480
 		arm_bpf_put_reg32(dst_lo, rd_lo, ctx);
1464
-		emit_a32_mov_i(dst_hi, 0, ctx);
1481
+		if (!ctx->prog->aux->verifier_zext)
1482
+			emit_a32_mov_i(dst_hi, 0, ctx);
1465
1483
 		break;
1466
1484
 	case BPF_ALU64 | BPF_DIV | BPF_K:
1467
1485
 	case BPF_ALU64 | BPF_DIV | BPF_X:
1468
1486
 	case BPF_ALU64 | BPF_MOD | BPF_K:
1469
1487
 	case BPF_ALU64 | BPF_MOD | BPF_X:
1470
1488
 		goto notyet;
1471
-	/* dst = dst >> imm */
1472
1489
 	/* dst = dst << imm */
1473
-	case BPF_ALU | BPF_RSH | BPF_K:
1490
+	/* dst = dst >> imm */
1491
+	/* dst = dst >> imm (signed) */
1474
1492
 	case BPF_ALU | BPF_LSH | BPF_K:
1493
+	case BPF_ALU | BPF_RSH | BPF_K:
1494
+	case BPF_ALU | BPF_ARSH | BPF_K:
1475
1495
 		if (unlikely(imm > 31))
1476
1496
 			return -EINVAL;
1477
1497
 		if (imm)
1478
1498
 			emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code));
1479
-		emit_a32_mov_i(dst_hi, 0, ctx);
1499
+		if (!ctx->prog->aux->verifier_zext)
1500
+			emit_a32_mov_i(dst_hi, 0, ctx);
1480
1501
 		break;
1481
1502
 	/* dst = dst << imm */
1482
1503
 	case BPF_ALU64 | BPF_LSH | BPF_K:
..
..
@@ -1511,7 +1532,8 @@
1511
1532
 	/* dst = ~dst */
1512
1533
 	case BPF_ALU | BPF_NEG:
1513
1534
 		emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code));
1514
-		emit_a32_mov_i(dst_hi, 0, ctx);
1535
+		if (!ctx->prog->aux->verifier_zext)
1536
+			emit_a32_mov_i(dst_hi, 0, ctx);
1515
1537
 		break;
1516
1538
 	/* dst = ~dst (64 bit) */
1517
1539
 	case BPF_ALU64 | BPF_NEG:
..
..
@@ -1567,11 +1589,13 @@
1567
1589
 #else /* ARMv6+ */
1568
1590
 			emit(ARM_UXTH(rd[1], rd[1]), ctx);
1569
1591
 #endif
1570
-			emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
1592
+			if (!ctx->prog->aux->verifier_zext)
1593
+				emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
1571
1594
 			break;
1572
1595
 		case 32:
1573
1596
 			/* zero-extend 32 bits into 64 bits */
1574
-			emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
1597
+			if (!ctx->prog->aux->verifier_zext)
1598
+				emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
1575
1599
 			break;
1576
1600
 		case 64:
1577
1601
 			/* nop */
..
..
@@ -1653,6 +1677,17 @@
1653
1677
 	case BPF_JMP | BPF_JLT | BPF_X:
1654
1678
 	case BPF_JMP | BPF_JSLT | BPF_X:
1655
1679
 	case BPF_JMP | BPF_JSLE | BPF_X:
1680
+	case BPF_JMP32 | BPF_JEQ | BPF_X:
1681
+	case BPF_JMP32 | BPF_JGT | BPF_X:
1682
+	case BPF_JMP32 | BPF_JGE | BPF_X:
1683
+	case BPF_JMP32 | BPF_JNE | BPF_X:
1684
+	case BPF_JMP32 | BPF_JSGT | BPF_X:
1685
+	case BPF_JMP32 | BPF_JSGE | BPF_X:
1686
+	case BPF_JMP32 | BPF_JSET | BPF_X:
1687
+	case BPF_JMP32 | BPF_JLE | BPF_X:
1688
+	case BPF_JMP32 | BPF_JLT | BPF_X:
1689
+	case BPF_JMP32 | BPF_JSLT | BPF_X:
1690
+	case BPF_JMP32 | BPF_JSLE | BPF_X:
1656
1691
 		/* Setup source registers */
1657
1692
 		rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx);
1658
1693
 		rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
..
..
@@ -1679,6 +1714,17 @@
1679
1714
 	case BPF_JMP | BPF_JLE | BPF_K:
1680
1715
 	case BPF_JMP | BPF_JSLT | BPF_K:
1681
1716
 	case BPF_JMP | BPF_JSLE | BPF_K:
1717
+	case BPF_JMP32 | BPF_JEQ | BPF_K:
1718
+	case BPF_JMP32 | BPF_JGT | BPF_K:
1719
+	case BPF_JMP32 | BPF_JGE | BPF_K:
1720
+	case BPF_JMP32 | BPF_JNE | BPF_K:
1721
+	case BPF_JMP32 | BPF_JSGT | BPF_K:
1722
+	case BPF_JMP32 | BPF_JSGE | BPF_K:
1723
+	case BPF_JMP32 | BPF_JSET | BPF_K:
1724
+	case BPF_JMP32 | BPF_JLT | BPF_K:
1725
+	case BPF_JMP32 | BPF_JLE | BPF_K:
1726
+	case BPF_JMP32 | BPF_JSLT | BPF_K:
1727
+	case BPF_JMP32 | BPF_JSLE | BPF_K:
1682
1728
 		if (off == 0)
1683
1729
 			break;
1684
1730
 		rm = tmp2[0];
..
..
@@ -1690,7 +1736,8 @@
1690
1736
 		rd = arm_bpf_get_reg64(dst, tmp, ctx);
1691
1737
 
1692
1738
 		/* Check for the condition */
1693
-		emit_ar_r(rd[0], rd[1], rm, rn, ctx, BPF_OP(code));
1739
+		emit_ar_r(rd[0], rd[1], rm, rn, ctx, BPF_OP(code),
1740
+			  BPF_CLASS(code) == BPF_JMP);
1694
1741
 
1695
1742
 		/* Setup JUMP instruction */
1696
1743
 		jmp_offset = bpf2a32_offset(i+off, i, ctx);
..
..
@@ -1841,6 +1888,11 @@
1841
1888
 	/* Nothing to do here. We support Internal BPF. */
1842
1889
 }
1843
1890
 
1891
+bool bpf_jit_needs_zext(void)
1892
+{
1893
+	return true;
1894
+}
1895
+
1844
1896
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1845
1897
 {
1846
1898
 	struct bpf_prog *tmp, *orig_prog = prog;