forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/arch/x86/net/bpf_jit_comp32.c
....@@ -15,6 +15,7 @@
1515 #include <asm/cacheflush.h>
1616 #include <asm/set_memory.h>
1717 #include <asm/nospec-branch.h>
18
+#include <asm/asm-prototypes.h>
1819 #include <linux/bpf.h>
1920 
2021 /*
....@@ -253,13 +254,14 @@
253254 /* dst = src */
254255 static inline void emit_ia32_mov_r64(const bool is64, const u8 dst[],
255256 				     const u8 src[], bool dstk,
256
-				     bool sstk, u8 **pprog)
257
+				     bool sstk, u8 **pprog,
258
+				     const struct bpf_prog_aux *aux)
257259 {
258260 	emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog);
259261 	if (is64)
260262 		/* complete 8 byte move */
261263 		emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog);
262
-	else
264
+	else if (!aux->verifier_zext)
263265 		/* zero out high 4 bytes */
264266 		emit_ia32_mov_i(dst_hi, 0, dstk, pprog);
265267 }
....@@ -313,7 +315,8 @@
313315 }
314316 
315317 static inline void emit_ia32_to_le_r64(const u8 dst[], s32 val,
316
-					 bool dstk, u8 **pprog)
318
+					 bool dstk, u8 **pprog,
319
+					 const struct bpf_prog_aux *aux)
317320 {
318321 	u8 *prog = *pprog;
319322 	int cnt = 0;
....@@ -334,12 +337,14 @@
334337 		 */
335338 		EMIT2(0x0F, 0xB7);
336339 		EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));
337
-		/* xor dreg_hi,dreg_hi */
338
-		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
340
+		if (!aux->verifier_zext)
341
+			/* xor dreg_hi,dreg_hi */
342
+			EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
339343 		break;
340344 	case 32:
341
-		/* xor dreg_hi,dreg_hi */
342
-		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
345
+		if (!aux->verifier_zext)
346
+			/* xor dreg_hi,dreg_hi */
347
+			EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
343348 		break;
344349 	case 64:
345350 		/* nop */
....@@ -358,7 +363,8 @@
358363 }
359364 
360365 static inline void emit_ia32_to_be_r64(const u8 dst[], s32 val,
361
-				       bool dstk, u8 **pprog)
366
+				       bool dstk, u8 **pprog,
367
+				       const struct bpf_prog_aux *aux)
362368 {
363369 	u8 *prog = *pprog;
364370 	int cnt = 0;
....@@ -380,16 +386,18 @@
380386 		EMIT2(0x0F, 0xB7);
381387 		EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo));
382388 
383
-		/* xor dreg_hi,dreg_hi */
384
-		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
389
+		if (!aux->verifier_zext)
390
+			/* xor dreg_hi,dreg_hi */
391
+			EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
385392 		break;
386393 	case 32:
387394 		/* Emit 'bswap eax' to swap lower 4 bytes */
388395 		EMIT1(0x0F);
389396 		EMIT1(add_1reg(0xC8, dreg_lo));
390397 
391
-		/* xor dreg_hi,dreg_hi */
392
-		EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
398
+		if (!aux->verifier_zext)
399
+			/* xor dreg_hi,dreg_hi */
400
+			EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi));
393401 		break;
394402 	case 64:
395403 		/* Emit 'bswap eax' to swap lower 4 bytes */
....@@ -569,7 +577,7 @@
569577 static inline void emit_ia32_alu_r64(const bool is64, const u8 op,
570578 				     const u8 dst[], const u8 src[],
571579 				     bool dstk,  bool sstk,
572
-				     u8 **pprog)
580
+				     u8 **pprog, const struct bpf_prog_aux *aux)
573581 {
574582 	u8 *prog = *pprog;
575583 
....@@ -577,7 +585,7 @@
577585 	if (is64)
578586 		emit_ia32_alu_r(is64, true, op, dst_hi, src_hi, dstk, sstk,
579587 				&prog);
580
-	else
588
+	else if (!aux->verifier_zext)
581589 		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
582590 	*pprog = prog;
583591 }
....@@ -668,7 +676,8 @@
668676 /* ALU operation (64 bit) */
669677 static inline void emit_ia32_alu_i64(const bool is64, const u8 op,
670678 				     const u8 dst[], const u32 val,
671
-				     bool dstk, u8 **pprog)
679
+				     bool dstk, u8 **pprog,
680
+				     const struct bpf_prog_aux *aux)
672681 {
673682 	u8 *prog = *pprog;
674683 	u32 hi = 0;
....@@ -679,7 +688,7 @@
679688 	emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog);
680689 	if (is64)
681690 		emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog);
682
-	else
691
+	else if (!aux->verifier_zext)
683692 		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
684693 
685694 	*pprog = prog;
....@@ -1259,6 +1268,21 @@
12591268 	*pprog = prog;
12601269 }
12611270 
1271
+static int emit_jmp_edx(u8 **pprog, u8 *ip)
1272
+{
1273
+	u8 *prog = *pprog;
1274
+	int cnt = 0;
1275
+
1276
+#ifdef CONFIG_RETPOLINE
1277
+	EMIT1_off32(0xE9, (u8 *)__x86_indirect_thunk_edx - (ip + 5));
1278
+#else
1279
+	EMIT2(0xFF, 0xE2);
1280
+#endif
1281
+	*pprog = prog;
1282
+
1283
+	return cnt;
1284
+}
1285
+
12621286 /*
12631287  * Generate the following code:
12641288  * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
....@@ -1272,7 +1296,7 @@
12721296  *   goto *(prog->bpf_func + prologue_size);
12731297  * out:
12741298  */
1275
-static void emit_bpf_tail_call(u8 **pprog)
1299
+static void emit_bpf_tail_call(u8 **pprog, u8 *ip)
12761300 {
12771301 	u8 *prog = *pprog;
12781302 	int cnt = 0;
....@@ -1354,7 +1378,7 @@
13541378 	 * eax == ctx (1st arg)
13551379 	 * edx == prog->bpf_func + prologue_size
13561380 	 */
1357
-	RETPOLINE_EDX_BPF_JIT();
1381
+	cnt += emit_jmp_edx(&prog, ip + cnt);
13581382 
13591383 	if (jmp_label1 == -1)
13601384 		jmp_label1 = cnt;
....@@ -1467,8 +1491,8 @@
14671491 	for (i = 0; i < insn_cnt; i++, insn++) {
14681492 		const s32 imm32 = insn->imm;
14691493 		const bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
1470
-		const bool dstk = insn->dst_reg == BPF_REG_AX ? false : true;
1471
-		const bool sstk = insn->src_reg == BPF_REG_AX ? false : true;
1494
+		const bool dstk = insn->dst_reg != BPF_REG_AX;
1495
+		const bool sstk = insn->src_reg != BPF_REG_AX;
14721496 		const u8 code = insn->code;
14731497 		const u8 *dst = bpf2ia32[insn->dst_reg];
14741498 		const u8 *src = bpf2ia32[insn->src_reg];
....@@ -1487,8 +1511,13 @@
14871511 		case BPF_ALU64 | BPF_MOV | BPF_X:
14881512 			switch (BPF_SRC(code)) {
14891513 			case BPF_X:
1490
-				emit_ia32_mov_r64(is64, dst, src, dstk,
1491
-						  sstk, &prog);
1514
+				if (imm32 == 1) {
1515
+					/* Special mov32 for zext. */
1516
+					emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1517
+					break;
1518
+				}
1519
+				emit_ia32_mov_r64(is64, dst, src, dstk, sstk,
1520
+						  &prog, bpf_prog->aux);
14921521 				break;
14931522 			case BPF_K:
14941523 				/* Sign-extend immediate value to dst reg */
....@@ -1528,11 +1557,13 @@
15281557 			switch (BPF_SRC(code)) {
15291558 			case BPF_X:
15301559 				emit_ia32_alu_r64(is64, BPF_OP(code), dst,
1531
-						  src, dstk, sstk, &prog);
1560
+						  src, dstk, sstk, &prog,
1561
+						  bpf_prog->aux);
15321562 				break;
15331563 			case BPF_K:
15341564 				emit_ia32_alu_i64(is64, BPF_OP(code), dst,
1535
-						  imm32, dstk, &prog);
1565
+						  imm32, dstk, &prog,
1566
+						  bpf_prog->aux);
15361567 				break;
15371568 			}
15381569 			break;
....@@ -1551,7 +1582,8 @@
15511582 						false, &prog);
15521583 				break;
15531584 			}
1554
-			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1585
+			if (!bpf_prog->aux->verifier_zext)
1586
+				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
15551587 			break;
15561588 		case BPF_ALU | BPF_LSH | BPF_X:
15571589 		case BPF_ALU | BPF_RSH | BPF_X:
....@@ -1571,7 +1603,8 @@
15711603 						  &prog);
15721604 				break;
15731605 			}
1574
-			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1606
+			if (!bpf_prog->aux->verifier_zext)
1607
+				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
15751608 			break;
15761609 		/* dst = dst / src(imm) */
15771610 		/* dst = dst % src(imm) */
....@@ -1593,7 +1626,8 @@
15931626 						    &prog);
15941627 				break;
15951628 			}
1596
-			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1629
+			if (!bpf_prog->aux->verifier_zext)
1630
+				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
15971631 			break;
15981632 		case BPF_ALU64 | BPF_DIV | BPF_K:
15991633 		case BPF_ALU64 | BPF_DIV | BPF_X:
....@@ -1610,7 +1644,8 @@
16101644 			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
16111645 			emit_ia32_shift_r(BPF_OP(code), dst_lo, IA32_ECX, dstk,
16121646 					  false, &prog);
1613
-			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1647
+			if (!bpf_prog->aux->verifier_zext)
1648
+				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
16141649 			break;
16151650 		/* dst = dst << imm */
16161651 		case BPF_ALU64 | BPF_LSH | BPF_K:
....@@ -1646,7 +1681,8 @@
16461681 		case BPF_ALU | BPF_NEG:
16471682 			emit_ia32_alu_i(is64, false, BPF_OP(code),
16481683 					dst_lo, 0, dstk, &prog);
1649
-			emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
1684
+			if (!bpf_prog->aux->verifier_zext)
1685
+				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);
16501686 			break;
16511687 		/* dst = ~dst (64 bit) */
16521688 		case BPF_ALU64 | BPF_NEG:
....@@ -1666,11 +1702,13 @@
16661702 			break;
16671703 		/* dst = htole(dst) */
16681704 		case BPF_ALU | BPF_END | BPF_FROM_LE:
1669
-			emit_ia32_to_le_r64(dst, imm32, dstk, &prog);
1705
+			emit_ia32_to_le_r64(dst, imm32, dstk, &prog,
1706
+					    bpf_prog->aux);
16701707 			break;
16711708 		/* dst = htobe(dst) */
16721709 		case BPF_ALU | BPF_END | BPF_FROM_BE:
1673
-			emit_ia32_to_be_r64(dst, imm32, dstk, &prog);
1710
+			emit_ia32_to_be_r64(dst, imm32, dstk, &prog,
1711
+					    bpf_prog->aux);
16741712 			break;
16751713 		/* dst = imm64 */
16761714 		case BPF_LD | BPF_IMM | BPF_DW: {
....@@ -1831,6 +1869,8 @@
18311869 			case BPF_B:
18321870 			case BPF_H:
18331871 			case BPF_W:
1872
+				if (bpf_prog->aux->verifier_zext)
1873
+					break;
18341874 				if (dstk) {
18351875 					EMIT3(0xC7, add_1reg(0x40, IA32_EBP),
18361876 					      STACK_VAR(dst_hi));
....@@ -1905,7 +1945,7 @@
19051945 			break;
19061946 		}
19071947 		case BPF_JMP | BPF_TAIL_CALL:
1908
-			emit_bpf_tail_call(&prog);
1948
+			emit_bpf_tail_call(&prog, image + addrs[i - 1]);
19091949 			break;
19101950 
19111951 		/* cond jump */
....@@ -1914,7 +1954,18 @@
19141954 		case BPF_JMP | BPF_JGT | BPF_X:
19151955 		case BPF_JMP | BPF_JLT | BPF_X:
19161956 		case BPF_JMP | BPF_JGE | BPF_X:
1917
-		case BPF_JMP | BPF_JLE | BPF_X: {
1957
+		case BPF_JMP | BPF_JLE | BPF_X:
1958
+		case BPF_JMP32 | BPF_JEQ | BPF_X:
1959
+		case BPF_JMP32 | BPF_JNE | BPF_X:
1960
+		case BPF_JMP32 | BPF_JGT | BPF_X:
1961
+		case BPF_JMP32 | BPF_JLT | BPF_X:
1962
+		case BPF_JMP32 | BPF_JGE | BPF_X:
1963
+		case BPF_JMP32 | BPF_JLE | BPF_X:
1964
+		case BPF_JMP32 | BPF_JSGT | BPF_X:
1965
+		case BPF_JMP32 | BPF_JSLE | BPF_X:
1966
+		case BPF_JMP32 | BPF_JSLT | BPF_X:
1967
+		case BPF_JMP32 | BPF_JSGE | BPF_X: {
1968
+			bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
19181969 			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
19191970 			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
19201971 			u8 sreg_lo = sstk ? IA32_ECX : src_lo;
....@@ -1923,20 +1974,28 @@
19231974 			if (dstk) {
19241975 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
19251976 				      STACK_VAR(dst_lo));
1926
-				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
1927
-				      STACK_VAR(dst_hi));
1977
+				if (is_jmp64)
1978
+					EMIT3(0x8B,
1979
+					      add_2reg(0x40, IA32_EBP,
1980
+						       IA32_EDX),
1981
+					      STACK_VAR(dst_hi));
19281982 			}
19291983 
19301984 			if (sstk) {
19311985 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
19321986 				      STACK_VAR(src_lo));
1933
-				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX),
1934
-				      STACK_VAR(src_hi));
1987
+				if (is_jmp64)
1988
+					EMIT3(0x8B,
1989
+					      add_2reg(0x40, IA32_EBP,
1990
+						       IA32_EBX),
1991
+					      STACK_VAR(src_hi));
19351992 			}
19361993 
1937
-			/* cmp dreg_hi,sreg_hi */
1938
-			EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
1939
-			EMIT2(IA32_JNE, 2);
1994
+			if (is_jmp64) {
1995
+				/* cmp dreg_hi,sreg_hi */
1996
+				EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
1997
+				EMIT2(IA32_JNE, 2);
1998
+			}
19401999 			/* cmp dreg_lo,sreg_lo */
19412000 			EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
19422001 			goto emit_cond_jmp;
....@@ -1975,7 +2034,9 @@
19752034 			EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
19762035 			goto emit_cond_jmp_signed;
19772036 		}
1978
-		case BPF_JMP | BPF_JSET | BPF_X: {
2037
+		case BPF_JMP | BPF_JSET | BPF_X:
2038
+		case BPF_JMP32 | BPF_JSET | BPF_X: {
2039
+			bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
19792040 			u8 dreg_lo = IA32_EAX;
19802041 			u8 dreg_hi = IA32_EDX;
19812042 			u8 sreg_lo = sstk ? IA32_ECX : src_lo;
....@@ -1984,62 +2045,79 @@
19842045 			if (dstk) {
19852046 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
19862047 				      STACK_VAR(dst_lo));
1987
-				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
1988
-				      STACK_VAR(dst_hi));
2048
+				if (is_jmp64)
2049
+					EMIT3(0x8B,
2050
+					      add_2reg(0x40, IA32_EBP,
2051
+						       IA32_EDX),
2052
+					      STACK_VAR(dst_hi));
19892053 			} else {
19902054 				/* mov dreg_lo,dst_lo */
19912055 				EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo));
1992
-				/* mov dreg_hi,dst_hi */
1993
-				EMIT2(0x89,
1994
-				      add_2reg(0xC0, dreg_hi, dst_hi));
2056
+				if (is_jmp64)
2057
+					/* mov dreg_hi,dst_hi */
2058
+					EMIT2(0x89,
2059
+					      add_2reg(0xC0, dreg_hi, dst_hi));
19952060 			}
19962061 
19972062 			if (sstk) {
19982063 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),
19992064 				      STACK_VAR(src_lo));
2000
-				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX),
2001
-				      STACK_VAR(src_hi));
2065
+				if (is_jmp64)
2066
+					EMIT3(0x8B,
2067
+					      add_2reg(0x40, IA32_EBP,
2068
+						       IA32_EBX),
2069
+					      STACK_VAR(src_hi));
20022070 			}
20032071 			/* and dreg_lo,sreg_lo */
20042072 			EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo));
2005
-			/* and dreg_hi,sreg_hi */
2006
-			EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
2007
-			/* or dreg_lo,dreg_hi */
2008
-			EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
2073
+			if (is_jmp64) {
2074
+				/* and dreg_hi,sreg_hi */
2075
+				EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
2076
+				/* or dreg_lo,dreg_hi */
2077
+				EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
2078
+			}
20092079 			goto emit_cond_jmp;
20102080 		}
2011
-		case BPF_JMP | BPF_JSET | BPF_K: {
2012
-			u32 hi;
2081
+		case BPF_JMP | BPF_JSET | BPF_K:
2082
+		case BPF_JMP32 | BPF_JSET | BPF_K: {
2083
+			bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
20132084 			u8 dreg_lo = IA32_EAX;
20142085 			u8 dreg_hi = IA32_EDX;
20152086 			u8 sreg_lo = IA32_ECX;
20162087 			u8 sreg_hi = IA32_EBX;
2088
+			u32 hi;
20172089 
20182090 			if (dstk) {
20192091 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
20202092 				      STACK_VAR(dst_lo));
2021
-				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
2022
-				      STACK_VAR(dst_hi));
2093
+				if (is_jmp64)
2094
+					EMIT3(0x8B,
2095
+					      add_2reg(0x40, IA32_EBP,
2096
+						       IA32_EDX),
2097
+					      STACK_VAR(dst_hi));
20232098 			} else {
20242099 				/* mov dreg_lo,dst_lo */
20252100 				EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo));
2026
-				/* mov dreg_hi,dst_hi */
2027
-				EMIT2(0x89,
2028
-				      add_2reg(0xC0, dreg_hi, dst_hi));
2101
+				if (is_jmp64)
2102
+					/* mov dreg_hi,dst_hi */
2103
+					EMIT2(0x89,
2104
+					      add_2reg(0xC0, dreg_hi, dst_hi));
20292105 			}
2030
-			hi = imm32 & (1<<31) ? (u32)~0 : 0;
20312106 
20322107 			/* mov ecx,imm32 */
2033
-			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
2034
-			/* mov ebx,imm32 */
2035
-			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi);
2108
+			EMIT2_off32(0xC7, add_1reg(0xC0, sreg_lo), imm32);
20362109 
20372110 			/* and dreg_lo,sreg_lo */
20382111 			EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo));
2039
-			/* and dreg_hi,sreg_hi */
2040
-			EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
2041
-			/* or dreg_lo,dreg_hi */
2042
-			EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
2112
+			if (is_jmp64) {
2113
+				hi = imm32 & (1 << 31) ? (u32)~0 : 0;
2114
+				/* mov ebx,imm32 */
2115
+				EMIT2_off32(0xC7, add_1reg(0xC0, sreg_hi), hi);
2116
+				/* and dreg_hi,sreg_hi */
2117
+				EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
2118
+				/* or dreg_lo,dreg_hi */
2119
+				EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
2120
+			}
20432121 			goto emit_cond_jmp;
20442122 		}
20452123 		case BPF_JMP | BPF_JEQ | BPF_K:
....@@ -2047,29 +2125,44 @@
20472125 		case BPF_JMP | BPF_JGT | BPF_K:
20482126 		case BPF_JMP | BPF_JLT | BPF_K:
20492127 		case BPF_JMP | BPF_JGE | BPF_K:
2050
-		case BPF_JMP | BPF_JLE | BPF_K: {
2051
-			u32 hi;
2128
+		case BPF_JMP | BPF_JLE | BPF_K:
2129
+		case BPF_JMP32 | BPF_JEQ | BPF_K:
2130
+		case BPF_JMP32 | BPF_JNE | BPF_K:
2131
+		case BPF_JMP32 | BPF_JGT | BPF_K:
2132
+		case BPF_JMP32 | BPF_JLT | BPF_K:
2133
+		case BPF_JMP32 | BPF_JGE | BPF_K:
2134
+		case BPF_JMP32 | BPF_JLE | BPF_K:
2135
+		case BPF_JMP32 | BPF_JSGT | BPF_K:
2136
+		case BPF_JMP32 | BPF_JSLE | BPF_K:
2137
+		case BPF_JMP32 | BPF_JSLT | BPF_K:
2138
+		case BPF_JMP32 | BPF_JSGE | BPF_K: {
2139
+			bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
20522140 			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
20532141 			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
20542142 			u8 sreg_lo = IA32_ECX;
20552143 			u8 sreg_hi = IA32_EBX;
2144
+			u32 hi;
20562145 
20572146 			if (dstk) {
20582147 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),
20592148 				      STACK_VAR(dst_lo));
2060
-				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),
2061
-				      STACK_VAR(dst_hi));
2149
+				if (is_jmp64)
2150
+					EMIT3(0x8B,
2151
+					      add_2reg(0x40, IA32_EBP,
2152
+						       IA32_EDX),
2153
+					      STACK_VAR(dst_hi));
20622154 			}
20632155 
2064
-			hi = imm32 & (1<<31) ? (u32)~0 : 0;
20652156 			/* mov ecx,imm32 */
20662157 			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32);
2067
-			/* mov ebx,imm32 */
2068
-			EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi);
2069
-
2070
-			/* cmp dreg_hi,sreg_hi */
2071
-			EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
2072
-			EMIT2(IA32_JNE, 2);
2158
+			if (is_jmp64) {
2159
+				hi = imm32 & (1 << 31) ? (u32)~0 : 0;
2160
+				/* mov ebx,imm32 */
2161
+				EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi);
2162
+				/* cmp dreg_hi,sreg_hi */
2163
+				EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi));
2164
+				EMIT2(IA32_JNE, 2);
2165
+			}
20732166 			/* cmp dreg_lo,sreg_lo */
20742167 			EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo));
20752168 
....@@ -2229,6 +2322,11 @@
22292322 	return proglen;
22302323 }
22312324 
2325
+bool bpf_jit_needs_zext(void)
2326
+{
2327
+	return true;
2328
+}
2329
+
22322330 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
22332331 {
22342332 	struct bpf_binary_header *header = NULL;