forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/arch/powerpc/net/bpf_jit_comp64.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * bpf_jit_comp64.c: eBPF JIT compiler
34  *
....@@ -5,11 +6,6 @@
56  *		  IBM Corporation
67  *
78  * Based on the powerpc classic BPF JIT compiler by Matt Evans
8
- *
9
- * This program is free software; you can redistribute it and/or
10
- * modify it under the terms of the GNU General Public License
11
- * as published by the Free Software Foundation; version 2
12
- * of the License.
139  */
1410 #include <linux/moduleloader.h>
1511 #include <asm/cacheflush.h>
....@@ -100,12 +96,12 @@
10096 	 * invoked through a tail call.
10197 	 */
10298 	if (ctx->seen & SEEN_TAILCALL) {
103
-		PPC_LI(b2p[TMP_REG_1], 0);
99
+		EMIT(PPC_RAW_LI(b2p[TMP_REG_1], 0));
104100 		/* this goes in the redzone */
105101 		PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8));
106102 	} else {
107
-		PPC_NOP();
108
-		PPC_NOP();
103
+		EMIT(PPC_RAW_NOP());
104
+		EMIT(PPC_RAW_NOP());
109105 	}
110106 
111107 #define BPF_TAILCALL_PROLOGUE_SIZE	8
....@@ -134,8 +130,8 @@
134130 
135131 	/* Setup frame pointer to point to the bpf stack area */
136132 	if (bpf_is_seen_register(ctx, BPF_REG_FP))
137
-		PPC_ADDI(b2p[BPF_REG_FP], 1,
138
-				STACK_FRAME_MIN_SIZE + ctx->stack_size);
133
+		EMIT(PPC_RAW_ADDI(b2p[BPF_REG_FP], 1,
134
+				STACK_FRAME_MIN_SIZE + ctx->stack_size));
139135 }
140136 
141137 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
....@@ -149,10 +145,10 @@
149145 
150146 	/* Tear down our stack frame */
151147 	if (bpf_has_stack_frame(ctx)) {
152
-		PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size);
148
+		EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size));
153149 		if (ctx->seen & SEEN_FUNC) {
154150 			PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
155
-			PPC_MTLR(0);
151
+			EMIT(PPC_RAW_MTLR(0));
156152 		}
157153 	}
158154 }
....@@ -162,12 +158,38 @@
162158 	bpf_jit_emit_common_epilogue(image, ctx);
163159 
164160 	/* Move result to r3 */
165
-	PPC_MR(3, b2p[BPF_REG_0]);
161
+	EMIT(PPC_RAW_MR(3, b2p[BPF_REG_0]));
166162 
167
-	PPC_BLR();
163
+	EMIT(PPC_RAW_BLR());
168164 }
169165 
170
-static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func)
166
+static void bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx,
167
+				       u64 func)
168
+{
169
+#ifdef PPC64_ELF_ABI_v1
170
+	/* func points to the function descriptor */
171
+	PPC_LI64(b2p[TMP_REG_2], func);
172
+	/* Load actual entry point from function descriptor */
173
+	PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
174
+	/* ... and move it to LR */
175
+	EMIT(PPC_RAW_MTLR(b2p[TMP_REG_1]));
176
+	/*
177
+	 * Load TOC from function descriptor at offset 8.
178
+	 * We can clobber r2 since we get called through a
179
+	 * function pointer (so caller will save/restore r2)
180
+	 * and since we don't use a TOC ourself.
181
+	 */
182
+	PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
183
+#else
184
+	/* We can clobber r12 */
185
+	PPC_FUNC_ADDR(12, func);
186
+	EMIT(PPC_RAW_MTLR(12));
187
+#endif
188
+	EMIT(PPC_RAW_BLRL());
189
+}
190
+
191
+static void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx,
192
+				       u64 func)
171193 {
172194 	unsigned int i, ctx_idx = ctx->idx;
173195 
....@@ -185,7 +207,7 @@
185207 	 * that PPC_LI64() can emit.
186208 	 */
187209 	for (i = ctx->idx - ctx_idx; i < 5; i++)
188
-		PPC_NOP();
210
+		EMIT(PPC_RAW_NOP());
189211 
190212 #ifdef PPC64_ELF_ABI_v1
191213 	/*
....@@ -199,8 +221,8 @@
199221 	PPC_BPF_LL(12, 12, 0);
200222 #endif
201223 
202
-	PPC_MTLR(12);
203
-	PPC_BLRL();
224
+	EMIT(PPC_RAW_MTLR(12));
225
+	EMIT(PPC_RAW_BLRL());
204226 }
205227 
206228 static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
....@@ -218,9 +240,9 @@
218240 	 * if (index >= array->map.max_entries)
219241 	 *   goto out;
220242 	 */
221
-	PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries));
222
-	PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31);
223
-	PPC_CMPLW(b2p_index, b2p[TMP_REG_1]);
243
+	EMIT(PPC_RAW_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
244
+	EMIT(PPC_RAW_RLWINM(b2p_index, b2p_index, 0, 0, 31));
245
+	EMIT(PPC_RAW_CMPLW(b2p_index, b2p[TMP_REG_1]));
224246 	PPC_BCC(COND_GE, out);
225247 
226248 	/*
....@@ -228,42 +250,42 @@
228250 	 *   goto out;
229251 	 */
230252 	PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
231
-	PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT);
253
+	EMIT(PPC_RAW_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT));
232254 	PPC_BCC(COND_GT, out);
233255 
234256 	/*
235257 	 * tail_call_cnt++;
236258 	 */
237
-	PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1);
259
+	EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1));
238260 	PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
239261 
240262 	/* prog = array->ptrs[index]; */
241
-	PPC_MULI(b2p[TMP_REG_1], b2p_index, 8);
242
-	PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array);
263
+	EMIT(PPC_RAW_MULI(b2p[TMP_REG_1], b2p_index, 8));
264
+	EMIT(PPC_RAW_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array));
243265 	PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
244266 
245267 	/*
246268 	 * if (prog == NULL)
247269 	 *   goto out;
248270 	 */
249
-	PPC_CMPLDI(b2p[TMP_REG_1], 0);
271
+	EMIT(PPC_RAW_CMPLDI(b2p[TMP_REG_1], 0));
250272 	PPC_BCC(COND_EQ, out);
251273 
252274 	/* goto *(prog->bpf_func + prologue_size); */
253275 	PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
254276 #ifdef PPC64_ELF_ABI_v1
255277 	/* skip past the function descriptor */
256
-	PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
257
-			FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE);
278
+	EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
279
+			FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE));
258280 #else
259
-	PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE);
281
+	EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE));
260282 #endif
261
-	PPC_MTCTR(b2p[TMP_REG_1]);
283
+	EMIT(PPC_RAW_MTCTR(b2p[TMP_REG_1]));
262284 
263285 	/* tear down stack, restore NVRs, ... */
264286 	bpf_jit_emit_common_epilogue(image, ctx);
265287 
266
-	PPC_BCTR();
288
+	EMIT(PPC_RAW_BCTR());
267289 
268290 	/* out: */
269291 	return 0;
....@@ -310,8 +332,9 @@
310332 		u32 src_reg = b2p[insn[i].src_reg];
311333 		s16 off = insn[i].off;
312334 		s32 imm = insn[i].imm;
335
+		bool func_addr_fixed;
336
+		u64 func_addr;
313337 		u64 imm64;
314
-		u8 *func;
315338 		u32 true_cond;
316339 		u32 tmp_idx;
317340 
....@@ -343,21 +366,21 @@
343366 		 */
344367 		case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
345368 		case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
346
-			PPC_ADD(dst_reg, dst_reg, src_reg);
369
+			EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
347370 			goto bpf_alu32_trunc;
348371 		case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
349372 		case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
350
-			PPC_SUB(dst_reg, dst_reg, src_reg);
373
+			EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
351374 			goto bpf_alu32_trunc;
352375 		case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
353376 		case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
354377 			if (!imm) {
355378 				goto bpf_alu32_trunc;
356379 			} else if (imm >= -32768 && imm < 32768) {
357
-				PPC_ADDI(dst_reg, dst_reg, IMM_L(imm));
380
+				EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
358381 			} else {
359382 				PPC_LI32(b2p[TMP_REG_1], imm);
360
-				PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]);
383
+				EMIT(PPC_RAW_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]));
361384 			}
362385 			goto bpf_alu32_trunc;
363386 		case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
....@@ -365,52 +388,52 @@
365388 			if (!imm) {
366389 				goto bpf_alu32_trunc;
367390 			} else if (imm > -32768 && imm <= 32768) {
368
-				PPC_ADDI(dst_reg, dst_reg, IMM_L(-imm));
391
+				EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(-imm)));
369392 			} else {
370393 				PPC_LI32(b2p[TMP_REG_1], imm);
371
-				PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
394
+				EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
372395 			}
373396 			goto bpf_alu32_trunc;
374397 		case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
375398 		case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
376399 			if (BPF_CLASS(code) == BPF_ALU)
377
-				PPC_MULW(dst_reg, dst_reg, src_reg);
400
+				EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
378401 			else
379
-				PPC_MULD(dst_reg, dst_reg, src_reg);
402
+				EMIT(PPC_RAW_MULD(dst_reg, dst_reg, src_reg));
380403 			goto bpf_alu32_trunc;
381404 		case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
382405 		case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
383406 			if (imm >= -32768 && imm < 32768)
384
-				PPC_MULI(dst_reg, dst_reg, IMM_L(imm));
407
+				EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm)));
385408 			else {
386409 				PPC_LI32(b2p[TMP_REG_1], imm);
387410 				if (BPF_CLASS(code) == BPF_ALU)
388
-					PPC_MULW(dst_reg, dst_reg,
389
-							b2p[TMP_REG_1]);
411
+					EMIT(PPC_RAW_MULW(dst_reg, dst_reg,
412
+							b2p[TMP_REG_1]));
390413 				else
391
-					PPC_MULD(dst_reg, dst_reg,
392
-							b2p[TMP_REG_1]);
414
+					EMIT(PPC_RAW_MULD(dst_reg, dst_reg,
415
+							b2p[TMP_REG_1]));
393416 			}
394417 			goto bpf_alu32_trunc;
395418 		case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
396419 		case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
397420 			if (BPF_OP(code) == BPF_MOD) {
398
-				PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg);
399
-				PPC_MULW(b2p[TMP_REG_1], src_reg,
400
-						b2p[TMP_REG_1]);
401
-				PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
421
+				EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg));
422
+				EMIT(PPC_RAW_MULW(b2p[TMP_REG_1], src_reg,
423
+						b2p[TMP_REG_1]));
424
+				EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
402425 			} else
403
-				PPC_DIVWU(dst_reg, dst_reg, src_reg);
426
+				EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
404427 			goto bpf_alu32_trunc;
405428 		case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
406429 		case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
407430 			if (BPF_OP(code) == BPF_MOD) {
408
-				PPC_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg);
409
-				PPC_MULD(b2p[TMP_REG_1], src_reg,
410
-						b2p[TMP_REG_1]);
411
-				PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
431
+				EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg));
432
+				EMIT(PPC_RAW_MULD(b2p[TMP_REG_1], src_reg,
433
+						b2p[TMP_REG_1]));
434
+				EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
412435 			} else
413
-				PPC_DIVDU(dst_reg, dst_reg, src_reg);
436
+				EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg));
414437 			break;
415438 		case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
416439 		case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
....@@ -422,7 +445,7 @@
422445 				if (BPF_OP(code) == BPF_DIV) {
423446 					goto bpf_alu32_trunc;
424447 				} else {
425
-					PPC_LI(dst_reg, 0);
448
+					EMIT(PPC_RAW_LI(dst_reg, 0));
426449 					break;
427450 				}
428451 			}
....@@ -431,35 +454,37 @@
431454 			switch (BPF_CLASS(code)) {
432455 			case BPF_ALU:
433456 				if (BPF_OP(code) == BPF_MOD) {
434
-					PPC_DIVWU(b2p[TMP_REG_2], dst_reg,
435
-							b2p[TMP_REG_1]);
436
-					PPC_MULW(b2p[TMP_REG_1],
457
+					EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_2],
458
+							dst_reg,
459
+							b2p[TMP_REG_1]));
460
+					EMIT(PPC_RAW_MULW(b2p[TMP_REG_1],
437461 							b2p[TMP_REG_1],
438
-							b2p[TMP_REG_2]);
439
-					PPC_SUB(dst_reg, dst_reg,
440
-							b2p[TMP_REG_1]);
462
+							b2p[TMP_REG_2]));
463
+					EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
464
+							b2p[TMP_REG_1]));
441465 				} else
442
-					PPC_DIVWU(dst_reg, dst_reg,
443
-							b2p[TMP_REG_1]);
466
+					EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg,
467
+							b2p[TMP_REG_1]));
444468 				break;
445469 			case BPF_ALU64:
446470 				if (BPF_OP(code) == BPF_MOD) {
447
-					PPC_DIVDU(b2p[TMP_REG_2], dst_reg,
448
-							b2p[TMP_REG_1]);
449
-					PPC_MULD(b2p[TMP_REG_1],
471
+					EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_2],
472
+							dst_reg,
473
+							b2p[TMP_REG_1]));
474
+					EMIT(PPC_RAW_MULD(b2p[TMP_REG_1],
450475 							b2p[TMP_REG_1],
451
-							b2p[TMP_REG_2]);
452
-					PPC_SUB(dst_reg, dst_reg,
453
-							b2p[TMP_REG_1]);
476
+							b2p[TMP_REG_2]));
477
+					EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
478
+							b2p[TMP_REG_1]));
454479 				} else
455
-					PPC_DIVDU(dst_reg, dst_reg,
456
-							b2p[TMP_REG_1]);
480
+					EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg,
481
+							b2p[TMP_REG_1]));
457482 				break;
458483 			}
459484 			goto bpf_alu32_trunc;
460485 		case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
461486 		case BPF_ALU64 | BPF_NEG: /* dst = -dst */
462
-			PPC_NEG(dst_reg, dst_reg);
487
+			EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
463488 			goto bpf_alu32_trunc;
464489 
465490 		/*
....@@ -467,86 +492,101 @@
467492 		 */
468493 		case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
469494 		case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
470
-			PPC_AND(dst_reg, dst_reg, src_reg);
495
+			EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
471496 			goto bpf_alu32_trunc;
472497 		case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
473498 		case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
474499 			if (!IMM_H(imm))
475
-				PPC_ANDI(dst_reg, dst_reg, IMM_L(imm));
500
+				EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
476501 			else {
477502 				/* Sign-extended */
478503 				PPC_LI32(b2p[TMP_REG_1], imm);
479
-				PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]);
504
+				EMIT(PPC_RAW_AND(dst_reg, dst_reg, b2p[TMP_REG_1]));
480505 			}
481506 			goto bpf_alu32_trunc;
482507 		case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
483508 		case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
484
-			PPC_OR(dst_reg, dst_reg, src_reg);
509
+			EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
485510 			goto bpf_alu32_trunc;
486511 		case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
487512 		case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
488513 			if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
489514 				/* Sign-extended */
490515 				PPC_LI32(b2p[TMP_REG_1], imm);
491
-				PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]);
516
+				EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_1]));
492517 			} else {
493518 				if (IMM_L(imm))
494
-					PPC_ORI(dst_reg, dst_reg, IMM_L(imm));
519
+					EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
495520 				if (IMM_H(imm))
496
-					PPC_ORIS(dst_reg, dst_reg, IMM_H(imm));
521
+					EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
497522 			}
498523 			goto bpf_alu32_trunc;
499524 		case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
500525 		case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
501
-			PPC_XOR(dst_reg, dst_reg, src_reg);
526
+			EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
502527 			goto bpf_alu32_trunc;
503528 		case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
504529 		case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
505530 			if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
506531 				/* Sign-extended */
507532 				PPC_LI32(b2p[TMP_REG_1], imm);
508
-				PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]);
533
+				EMIT(PPC_RAW_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]));
509534 			} else {
510535 				if (IMM_L(imm))
511
-					PPC_XORI(dst_reg, dst_reg, IMM_L(imm));
536
+					EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
512537 				if (IMM_H(imm))
513
-					PPC_XORIS(dst_reg, dst_reg, IMM_H(imm));
538
+					EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
514539 			}
515540 			goto bpf_alu32_trunc;
516541 		case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
517542 			/* slw clears top 32 bits */
518
-			PPC_SLW(dst_reg, dst_reg, src_reg);
543
+			EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
544
+			/* skip zero extension move, but set address map. */
545
+			if (insn_is_zext(&insn[i + 1]))
546
+				addrs[++i] = ctx->idx * 4;
519547 			break;
520548 		case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
521
-			PPC_SLD(dst_reg, dst_reg, src_reg);
549
+			EMIT(PPC_RAW_SLD(dst_reg, dst_reg, src_reg));
522550 			break;
523551 		case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
524552 			/* with imm 0, we still need to clear top 32 bits */
525
-			PPC_SLWI(dst_reg, dst_reg, imm);
553
+			EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
554
+			if (insn_is_zext(&insn[i + 1]))
555
+				addrs[++i] = ctx->idx * 4;
526556 			break;
527557 		case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
528558 			if (imm != 0)
529
-				PPC_SLDI(dst_reg, dst_reg, imm);
559
+				EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, imm));
530560 			break;
531561 		case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
532
-			PPC_SRW(dst_reg, dst_reg, src_reg);
562
+			EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
563
+			if (insn_is_zext(&insn[i + 1]))
564
+				addrs[++i] = ctx->idx * 4;
533565 			break;
534566 		case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
535
-			PPC_SRD(dst_reg, dst_reg, src_reg);
567
+			EMIT(PPC_RAW_SRD(dst_reg, dst_reg, src_reg));
536568 			break;
537569 		case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
538
-			PPC_SRWI(dst_reg, dst_reg, imm);
570
+			EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
571
+			if (insn_is_zext(&insn[i + 1]))
572
+				addrs[++i] = ctx->idx * 4;
539573 			break;
540574 		case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
541575 			if (imm != 0)
542
-				PPC_SRDI(dst_reg, dst_reg, imm);
576
+				EMIT(PPC_RAW_SRDI(dst_reg, dst_reg, imm));
543577 			break;
578
+		case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
579
+			EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
580
+			goto bpf_alu32_trunc;
544581 		case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
545
-			PPC_SRAD(dst_reg, dst_reg, src_reg);
582
+			EMIT(PPC_RAW_SRAD(dst_reg, dst_reg, src_reg));
546583 			break;
584
+		case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
585
+			EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
586
+			goto bpf_alu32_trunc;
547587 		case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
548588 			if (imm != 0)
549
-				PPC_SRADI(dst_reg, dst_reg, imm);
589
+				EMIT(PPC_RAW_SRADI(dst_reg, dst_reg, imm));
550590 			break;
551591 
552592 		/*
....@@ -554,19 +594,26 @@
554594 		 */
555595 		case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
556596 		case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
557
-			PPC_MR(dst_reg, src_reg);
597
+			if (imm == 1) {
598
+				/* special mov32 for zext */
599
+				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
600
+				break;
601
+			}
602
+			EMIT(PPC_RAW_MR(dst_reg, src_reg));
558603 			goto bpf_alu32_trunc;
559604 		case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
560605 		case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
561606 			PPC_LI32(dst_reg, imm);
562607 			if (imm < 0)
563608 				goto bpf_alu32_trunc;
609
+			else if (insn_is_zext(&insn[i + 1]))
610
+				addrs[++i] = ctx->idx * 4;
564611 			break;
565612 
566613 bpf_alu32_trunc:
567614 		/* Truncate to 32-bits */
568
-		if (BPF_CLASS(code) == BPF_ALU)
569
-			PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
615
+		if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)
616
+			EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
570617 		break;
571618 
572619 		/*
....@@ -584,11 +631,11 @@
584631 			switch (imm) {
585632 			case 16:
586633 				/* Rotate 8 bits left & mask with 0x0000ff00 */
587
-				PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23);
634
+				EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23));
588635 				/* Rotate 8 bits right & insert LSB to reg */
589
-				PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31);
636
+				EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31));
590637 				/* Move result back to dst_reg */
591
-				PPC_MR(dst_reg, b2p[TMP_REG_1]);
638
+				EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
592639 				break;
593640 			case 32:
594641 				/*
....@@ -596,25 +643,29 @@
596643 				 * 2 bytes are already in their final position
597644 				 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
598645 				 */
599
-				PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31);
646
+				EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31));
600647 				/* Rotate 24 bits and insert byte 1 */
601
-				PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7);
648
+				EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7));
602649 				/* Rotate 24 bits and insert byte 3 */
603
-				PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23);
604
-				PPC_MR(dst_reg, b2p[TMP_REG_1]);
650
+				EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23));
651
+				EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
605652 				break;
606653 			case 64:
607
-				/*
608
-				 * Way easier and faster(?) to store the value
609
-				 * into stack and then use ldbrx
610
-				 *
611
-				 * ctx->seen will be reliable in pass2, but
612
-				 * the instructions generated will remain the
613
-				 * same across all passes
614
-				 */
654
+				/* Store the value to stack and then use byte-reverse loads */
615655 				PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx));
616
-				PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
617
-				PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
656
+				EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)));
657
+				if (cpu_has_feature(CPU_FTR_ARCH_206)) {
658
+					EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1]));
659
+				} else {
660
+					EMIT(PPC_RAW_LWBRX(dst_reg, 0, b2p[TMP_REG_1]));
661
+					if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
662
+						EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, 32));
663
+					EMIT(PPC_RAW_LI(b2p[TMP_REG_2], 4));
664
+					EMIT(PPC_RAW_LWBRX(b2p[TMP_REG_2], b2p[TMP_REG_2], b2p[TMP_REG_1]));
665
+					if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
666
+						EMIT(PPC_RAW_SLDI(b2p[TMP_REG_2], b2p[TMP_REG_2], 32));
667
+					EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_2]));
668
+				}
618669 				break;
619670 			}
620671 			break;
....@@ -623,11 +674,14 @@
623674 			switch (imm) {
624675 			case 16:
625676 				/* zero-extend 16 bits into 64 bits */
626
-				PPC_RLDICL(dst_reg, dst_reg, 0, 48);
677
+				EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 48));
678
+				if (insn_is_zext(&insn[i + 1]))
679
+					addrs[++i] = ctx->idx * 4;
627680 				break;
628681 			case 32:
629
-				/* zero-extend 32 bits into 64 bits */
630
-				PPC_RLDICL(dst_reg, dst_reg, 0, 32);
682
+				if (!fp->aux->verifier_zext)
683
+					/* zero-extend 32 bits into 64 bits */
684
+					EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 32));
631685 				break;
632686 			case 64:
633687 				/* nop */
....@@ -641,7 +695,7 @@
641695 		case BPF_ST | BPF_NOSPEC:
642696 			if (!security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) ||
643697 					(!security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) &&
644
-					(!security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) || !cpu_has_feature(CPU_FTR_HVMODE))))
698
+					 (!security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) || !cpu_has_feature(CPU_FTR_HVMODE))))
645699 				break;
646700 
647701 			switch (stf_barrier) {
....@@ -650,15 +704,15 @@
650704 				break;
651705 			case STF_BARRIER_SYNC_ORI:
652706 				EMIT(PPC_INST_SYNC);
653
-				PPC_LD(b2p[TMP_REG_1], 13, 0);
654
-				PPC_ORI(31, 31, 0);
707
+				EMIT(PPC_RAW_LD(b2p[TMP_REG_1], 13, 0));
708
+				EMIT(PPC_RAW_ORI(31, 31, 0));
655709 				break;
656710 			case STF_BARRIER_FALLBACK:
657711 				EMIT(PPC_INST_MFLR | ___PPC_RT(b2p[TMP_REG_1]));
658712 				PPC_LI64(12, dereference_kernel_function_descriptor(bpf_stf_barrier));
659
-				PPC_MTCTR(12);
713
+				EMIT(PPC_RAW_MTCTR(12));
660714 				EMIT(PPC_INST_BCTR | 0x1);
661
-				PPC_MTLR(b2p[TMP_REG_1]);
715
+				EMIT(PPC_RAW_MTLR(b2p[TMP_REG_1]));
662716 				break;
663717 			case STF_BARRIER_NONE:
664718 				break;
....@@ -671,18 +725,18 @@
671725 		case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
672726 		case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
673727 			if (BPF_CLASS(code) == BPF_ST) {
674
-				PPC_LI(b2p[TMP_REG_1], imm);
728
+				EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
675729 				src_reg = b2p[TMP_REG_1];
676730 			}
677
-			PPC_STB(src_reg, dst_reg, off);
731
+			EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
678732 			break;
679733 		case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
680734 		case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
681735 			if (BPF_CLASS(code) == BPF_ST) {
682
-				PPC_LI(b2p[TMP_REG_1], imm);
736
+				EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
683737 				src_reg = b2p[TMP_REG_1];
684738 			}
685
-			PPC_STH(src_reg, dst_reg, off);
739
+			EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
686740 			break;
687741 		case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
688742 		case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
....@@ -690,7 +744,7 @@
690744 				PPC_LI32(b2p[TMP_REG_1], imm);
691745 				src_reg = b2p[TMP_REG_1];
692746 			}
693
-			PPC_STW(src_reg, dst_reg, off);
747
+			EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
694748 			break;
695749 		case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
696750 		case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
....@@ -707,24 +761,24 @@
707761 		/* *(u32 *)(dst + off) += src */
708762 		case BPF_STX | BPF_XADD | BPF_W:
709763 			/* Get EA into TMP_REG_1 */
710
-			PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
764
+			EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
711765 			tmp_idx = ctx->idx * 4;
712766 			/* load value from memory into TMP_REG_2 */
713
-			PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
767
+			EMIT(PPC_RAW_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
714768 			/* add value from src_reg into this */
715
-			PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
769
+			EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
716770 			/* store result back */
717
-			PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
771
+			EMIT(PPC_RAW_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
718772 			/* we're done if this succeeded */
719773 			PPC_BCC_SHORT(COND_NE, tmp_idx);
720774 			break;
721775 		/* *(u64 *)(dst + off) += src */
722776 		case BPF_STX | BPF_XADD | BPF_DW:
723
-			PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
777
+			EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
724778 			tmp_idx = ctx->idx * 4;
725
-			PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
726
-			PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
727
-			PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
779
+			EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
780
+			EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
781
+			EMIT(PPC_RAW_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
728782 			PPC_BCC_SHORT(COND_NE, tmp_idx);
729783 			break;
730784 
....@@ -733,15 +787,21 @@
733787 		 */
734788 		/* dst = *(u8 *)(ul) (src + off) */
735789 		case BPF_LDX | BPF_MEM | BPF_B:
736
-			PPC_LBZ(dst_reg, src_reg, off);
790
+			EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
791
+			if (insn_is_zext(&insn[i + 1]))
792
+				addrs[++i] = ctx->idx * 4;
737793 			break;
738794 		/* dst = *(u16 *)(ul) (src + off) */
739795 		case BPF_LDX | BPF_MEM | BPF_H:
740
-			PPC_LHZ(dst_reg, src_reg, off);
796
+			EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
797
+			if (insn_is_zext(&insn[i + 1]))
798
+				addrs[++i] = ctx->idx * 4;
741799 			break;
742800 		/* dst = *(u32 *)(ul) (src + off) */
743801 		case BPF_LDX | BPF_MEM | BPF_W:
744
-			PPC_LWZ(dst_reg, src_reg, off);
802
+			EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
803
+			if (insn_is_zext(&insn[i + 1]))
804
+				addrs[++i] = ctx->idx * 4;
745805 			break;
746806 		/* dst = *(u64 *)(ul) (src + off) */
747807 		case BPF_LDX | BPF_MEM | BPF_DW:
....@@ -780,25 +840,17 @@
780840 		case BPF_JMP | BPF_CALL:
781841 			ctx->seen |= SEEN_FUNC;
782842 
783
-			/* bpf function call */
784
-			if (insn[i].src_reg == BPF_PSEUDO_CALL)
785
-				if (!extra_pass)
786
-					func = NULL;
787
-				else if (fp->aux->func && off < fp->aux->func_cnt)
788
-					/* use the subprog id from the off
789
-					 * field to lookup the callee address
790
-					 */
791
-					func = (u8 *) fp->aux->func[off]->bpf_func;
792
-				else
793
-					return -EINVAL;
794
-			/* kernel helper call */
843
+			ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
844
+						    &func_addr, &func_addr_fixed);
845
+			if (ret < 0)
846
+				return ret;
847
+
848
+			if (func_addr_fixed)
849
+				bpf_jit_emit_func_call_hlp(image, ctx, func_addr);
795850 			else
796
-				func = (u8 *) __bpf_call_base + imm;
797
-
798
-			bpf_jit_emit_func_call(image, ctx, (u64)func);
799
-
851
+				bpf_jit_emit_func_call_rel(image, ctx, func_addr);
800852 			/* move return value from r3 to BPF_REG_0 */
801
-			PPC_MR(b2p[BPF_REG_0], 3);
853
+			EMIT(PPC_RAW_MR(b2p[BPF_REG_0], 3));
802854 			break;
803855 
804856 		/*
....@@ -812,36 +864,58 @@
812864 		case BPF_JMP | BPF_JGT | BPF_X:
813865 		case BPF_JMP | BPF_JSGT | BPF_K:
814866 		case BPF_JMP | BPF_JSGT | BPF_X:
867
+		case BPF_JMP32 | BPF_JGT | BPF_K:
868
+		case BPF_JMP32 | BPF_JGT | BPF_X:
869
+		case BPF_JMP32 | BPF_JSGT | BPF_K:
870
+		case BPF_JMP32 | BPF_JSGT | BPF_X:
815871 			true_cond = COND_GT;
816872 			goto cond_branch;
817873 		case BPF_JMP | BPF_JLT | BPF_K:
818874 		case BPF_JMP | BPF_JLT | BPF_X:
819875 		case BPF_JMP | BPF_JSLT | BPF_K:
820876 		case BPF_JMP | BPF_JSLT | BPF_X:
877
+		case BPF_JMP32 | BPF_JLT | BPF_K:
878
+		case BPF_JMP32 | BPF_JLT | BPF_X:
879
+		case BPF_JMP32 | BPF_JSLT | BPF_K:
880
+		case BPF_JMP32 | BPF_JSLT | BPF_X:
821881 			true_cond = COND_LT;
822882 			goto cond_branch;
823883 		case BPF_JMP | BPF_JGE | BPF_K:
824884 		case BPF_JMP | BPF_JGE | BPF_X:
825885 		case BPF_JMP | BPF_JSGE | BPF_K:
826886 		case BPF_JMP | BPF_JSGE | BPF_X:
887
+		case BPF_JMP32 | BPF_JGE | BPF_K:
888
+		case BPF_JMP32 | BPF_JGE | BPF_X:
889
+		case BPF_JMP32 | BPF_JSGE | BPF_K:
890
+		case BPF_JMP32 | BPF_JSGE | BPF_X:
827891 			true_cond = COND_GE;
828892 			goto cond_branch;
829893 		case BPF_JMP | BPF_JLE | BPF_K:
830894 		case BPF_JMP | BPF_JLE | BPF_X:
831895 		case BPF_JMP | BPF_JSLE | BPF_K:
832896 		case BPF_JMP | BPF_JSLE | BPF_X:
897
+		case BPF_JMP32 | BPF_JLE | BPF_K:
898
+		case BPF_JMP32 | BPF_JLE | BPF_X:
899
+		case BPF_JMP32 | BPF_JSLE | BPF_K:
900
+		case BPF_JMP32 | BPF_JSLE | BPF_X:
833901 			true_cond = COND_LE;
834902 			goto cond_branch;
835903 		case BPF_JMP | BPF_JEQ | BPF_K:
836904 		case BPF_JMP | BPF_JEQ | BPF_X:
905
+		case BPF_JMP32 | BPF_JEQ | BPF_K:
906
+		case BPF_JMP32 | BPF_JEQ | BPF_X:
837907 			true_cond = COND_EQ;
838908 			goto cond_branch;
839909 		case BPF_JMP | BPF_JNE | BPF_K:
840910 		case BPF_JMP | BPF_JNE | BPF_X:
911
+		case BPF_JMP32 | BPF_JNE | BPF_K:
912
+		case BPF_JMP32 | BPF_JNE | BPF_X:
841913 			true_cond = COND_NE;
842914 			goto cond_branch;
843915 		case BPF_JMP | BPF_JSET | BPF_K:
844916 		case BPF_JMP | BPF_JSET | BPF_X:
917
+		case BPF_JMP32 | BPF_JSET | BPF_K:
918
+		case BPF_JMP32 | BPF_JSET | BPF_X:
845919 			true_cond = COND_NE;
846920 			/* Fall through */
847921 
....@@ -853,18 +927,44 @@
853927 			case BPF_JMP | BPF_JLE | BPF_X:
854928 			case BPF_JMP | BPF_JEQ | BPF_X:
855929 			case BPF_JMP | BPF_JNE | BPF_X:
930
+			case BPF_JMP32 | BPF_JGT | BPF_X:
931
+			case BPF_JMP32 | BPF_JLT | BPF_X:
932
+			case BPF_JMP32 | BPF_JGE | BPF_X:
933
+			case BPF_JMP32 | BPF_JLE | BPF_X:
934
+			case BPF_JMP32 | BPF_JEQ | BPF_X:
935
+			case BPF_JMP32 | BPF_JNE | BPF_X:
856936 				/* unsigned comparison */
857
-				PPC_CMPLD(dst_reg, src_reg);
937
+				if (BPF_CLASS(code) == BPF_JMP32)
938
+					EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
939
+				else
940
+					EMIT(PPC_RAW_CMPLD(dst_reg, src_reg));
858941 				break;
859942 			case BPF_JMP | BPF_JSGT | BPF_X:
860943 			case BPF_JMP | BPF_JSLT | BPF_X:
861944 			case BPF_JMP | BPF_JSGE | BPF_X:
862945 			case BPF_JMP | BPF_JSLE | BPF_X:
946
+			case BPF_JMP32 | BPF_JSGT | BPF_X:
947
+			case BPF_JMP32 | BPF_JSLT | BPF_X:
948
+			case BPF_JMP32 | BPF_JSGE | BPF_X:
949
+			case BPF_JMP32 | BPF_JSLE | BPF_X:
863950 				/* signed comparison */
864
-				PPC_CMPD(dst_reg, src_reg);
951
+				if (BPF_CLASS(code) == BPF_JMP32)
952
+					EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
953
+				else
954
+					EMIT(PPC_RAW_CMPD(dst_reg, src_reg));
865955 				break;
866956 			case BPF_JMP | BPF_JSET | BPF_X:
867
-				PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, src_reg);
957
+			case BPF_JMP32 | BPF_JSET | BPF_X:
958
+				if (BPF_CLASS(code) == BPF_JMP) {
959
+					EMIT(PPC_RAW_AND_DOT(b2p[TMP_REG_1], dst_reg,
960
+						    src_reg));
961
+				} else {
962
+					int tmp_reg = b2p[TMP_REG_1];
963
+
964
+					EMIT(PPC_RAW_AND(tmp_reg, dst_reg, src_reg));
965
+					EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0,
966
+						       31));
967
+				}
868968 				break;
869969 			case BPF_JMP | BPF_JNE | BPF_K:
870970 			case BPF_JMP | BPF_JEQ | BPF_K:
....@@ -872,43 +972,87 @@
872972 			case BPF_JMP | BPF_JLT | BPF_K:
873973 			case BPF_JMP | BPF_JGE | BPF_K:
874974 			case BPF_JMP | BPF_JLE | BPF_K:
975
+			case BPF_JMP32 | BPF_JNE | BPF_K:
976
+			case BPF_JMP32 | BPF_JEQ | BPF_K:
977
+			case BPF_JMP32 | BPF_JGT | BPF_K:
978
+			case BPF_JMP32 | BPF_JLT | BPF_K:
979
+			case BPF_JMP32 | BPF_JGE | BPF_K:
980
+			case BPF_JMP32 | BPF_JLE | BPF_K:
981
+			{
982
+				bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
983
+
875984 				/*
876985 				 * Need sign-extended load, so only positive
877986 				 * values can be used as imm in cmpldi
878987 				 */
879
-				if (imm >= 0 && imm < 32768)
880
-					PPC_CMPLDI(dst_reg, imm);
881
-				else {
988
+				if (imm >= 0 && imm < 32768) {
989
+					if (is_jmp32)
990
+						EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
991
+					else
992
+						EMIT(PPC_RAW_CMPLDI(dst_reg, imm));
993
+				} else {
882994 					/* sign-extending load */
883995 					PPC_LI32(b2p[TMP_REG_1], imm);
884996 					/* ... but unsigned comparison */
885
-					PPC_CMPLD(dst_reg, b2p[TMP_REG_1]);
997
+					if (is_jmp32)
998
+						EMIT(PPC_RAW_CMPLW(dst_reg,
999
+							  b2p[TMP_REG_1]));
1000
+					else
1001
+						EMIT(PPC_RAW_CMPLD(dst_reg,
1002
+							  b2p[TMP_REG_1]));
8861003 				}
8871004 				break;
1005
+			}
8881006 			case BPF_JMP | BPF_JSGT | BPF_K:
8891007 			case BPF_JMP | BPF_JSLT | BPF_K:
8901008 			case BPF_JMP | BPF_JSGE | BPF_K:
8911009 			case BPF_JMP | BPF_JSLE | BPF_K:
1010
+			case BPF_JMP32 | BPF_JSGT | BPF_K:
1011
+			case BPF_JMP32 | BPF_JSLT | BPF_K:
1012
+			case BPF_JMP32 | BPF_JSGE | BPF_K:
1013
+			case BPF_JMP32 | BPF_JSLE | BPF_K:
1014
+			{
1015
+				bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
1016
+
8921017 				/*
8931018 				 * signed comparison, so any 16-bit value
8941019 				 * can be used in cmpdi
8951020 				 */
896
-				if (imm >= -32768 && imm < 32768)
897
-					PPC_CMPDI(dst_reg, imm);
898
-				else {
1021
+				if (imm >= -32768 && imm < 32768) {
1022
+					if (is_jmp32)
1023
+						EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1024
+					else
1025
+						EMIT(PPC_RAW_CMPDI(dst_reg, imm));
1026
+				} else {
8991027 					PPC_LI32(b2p[TMP_REG_1], imm);
900
-					PPC_CMPD(dst_reg, b2p[TMP_REG_1]);
1028
+					if (is_jmp32)
1029
+						EMIT(PPC_RAW_CMPW(dst_reg,
1030
+							 b2p[TMP_REG_1]));
1031
+					else
1032
+						EMIT(PPC_RAW_CMPD(dst_reg,
1033
+							 b2p[TMP_REG_1]));
9011034 				}
9021035 				break;
1036
+			}
9031037 			case BPF_JMP | BPF_JSET | BPF_K:
1038
+			case BPF_JMP32 | BPF_JSET | BPF_K:
9041039 				/* andi does not sign-extend the immediate */
9051040 				if (imm >= 0 && imm < 32768)
9061041 					/* PPC_ANDI is _only/always_ dot-form */
907
-					PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm);
1042
+					EMIT(PPC_RAW_ANDI(b2p[TMP_REG_1], dst_reg, imm));
9081043 				else {
909
-					PPC_LI32(b2p[TMP_REG_1], imm);
910
-					PPC_AND_DOT(b2p[TMP_REG_1], dst_reg,
911
-						    b2p[TMP_REG_1]);
1044
+					int tmp_reg = b2p[TMP_REG_1];
1045
+
1046
+					PPC_LI32(tmp_reg, imm);
1047
+					if (BPF_CLASS(code) == BPF_JMP) {
1048
+						EMIT(PPC_RAW_AND_DOT(tmp_reg, dst_reg,
1049
+							    tmp_reg));
1050
+					} else {
1051
+						EMIT(PPC_RAW_AND(tmp_reg, dst_reg,
1052
+							tmp_reg));
1053
+						EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg,
1054
+							       0, 0, 31));
1055
+					}
9121056 				}
9131057 				break;
9141058 			}
....@@ -943,6 +1087,55 @@
9431087 	return 0;
9441088 }
9451089 
1090
+/* Fix the branch target addresses for subprog calls */
1091
+static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image,
1092
+				       struct codegen_context *ctx, u32 *addrs)
1093
+{
1094
+	const struct bpf_insn *insn = fp->insnsi;
1095
+	bool func_addr_fixed;
1096
+	u64 func_addr;
1097
+	u32 tmp_idx;
1098
+	int i, ret;
1099
+
1100
+	for (i = 0; i < fp->len; i++) {
1101
+		/*
1102
+		 * During the extra pass, only the branch target addresses for
1103
+		 * the subprog calls need to be fixed. All other instructions
1104
+		 * can left untouched.
1105
+		 *
1106
+		 * The JITed image length does not change because we already
1107
+		 * ensure that the JITed instruction sequence for these calls
1108
+		 * are of fixed length by padding them with NOPs.
1109
+		 */
1110
+		if (insn[i].code == (BPF_JMP | BPF_CALL) &&
1111
+		    insn[i].src_reg == BPF_PSEUDO_CALL) {
1112
+			ret = bpf_jit_get_func_addr(fp, &insn[i], true,
1113
+						    &func_addr,
1114
+						    &func_addr_fixed);
1115
+			if (ret < 0)
1116
+				return ret;
1117
+
1118
+			/*
1119
+			 * Save ctx->idx as this would currently point to the
1120
+			 * end of the JITed image and set it to the offset of
1121
+			 * the instruction sequence corresponding to the
1122
+			 * subprog call temporarily.
1123
+			 */
1124
+			tmp_idx = ctx->idx;
1125
+			ctx->idx = addrs[i] / 4;
1126
+			bpf_jit_emit_func_call_rel(image, ctx, func_addr);
1127
+
1128
+			/*
1129
+			 * Restore ctx->idx here. This is safe as the length
1130
+			 * of the JITed sequence remains unchanged.
1131
+			 */
1132
+			ctx->idx = tmp_idx;
1133
+		}
1134
+	}
1135
+
1136
+	return 0;
1137
+}
1138
+
9461139 struct powerpc64_jit_data {
9471140 	struct bpf_binary_header *header;
9481141 	u32 *addrs;
....@@ -950,6 +1143,11 @@
9501143 	u32 proglen;
9511144 	struct codegen_context ctx;
9521145 };
1146
+
1147
+bool bpf_jit_needs_zext(void)
1148
+{
1149
+	return true;
1150
+}
9531151 
9541152 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
9551153 {
....@@ -1054,6 +1252,22 @@
10541252 skip_init_ctx:
10551253 	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
10561254 
1255
+	if (extra_pass) {
1256
+		/*
1257
+		 * Do not touch the prologue and epilogue as they will remain
1258
+		 * unchanged. Only fix the branch target address for subprog
1259
+		 * calls in the body.
1260
+		 *
1261
+		 * This does not change the offsets and lengths of the subprog
1262
+		 * call instruction sequences and hence, the size of the JITed
1263
+		 * image as well.
1264
+		 */
1265
+		bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs);
1266
+
1267
+		/* There is no need to perform the usual passes. */
1268
+		goto skip_codegen_passes;
1269
+	}
1270
+
10571271 	/* Code generation passes 1-2 */
10581272 	for (pass = 1; pass < 3; pass++) {
10591273 		/* Now build the prologue, body code & epilogue for real. */
....@@ -1067,6 +1281,7 @@
10671281 				proglen - (cgctx.idx * 4), cgctx.seen);
10681282 	}
10691283 
1284
+skip_codegen_passes:
10701285 	if (bpf_jit_enable > 1)
10711286 		/*
10721287 		 * Note that we output the base address of the code_base
....@@ -1086,6 +1301,7 @@
10861301 
10871302 	bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
10881303 	if (!fp->is_func || extra_pass) {
1304
+		bpf_prog_fill_jited_linfo(fp, addrs);
10891305 out_addrs:
10901306 		kfree(addrs);
10911307 		kfree(jit_data);