forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/arch/mips/net/ebpf_jit.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Just-In-Time compiler for eBPF filters on MIPS
34  *
....@@ -7,10 +8,6 @@
78  *
89  * Copyright (c) 2014 Imagination Technologies Ltd.
910  * Author: Markos Chandras <markos.chandras@imgtec.com>
10
- *
11
- * This program is free software; you can redistribute it and/or modify it
12
- * under the terms of the GNU General Public License as published by the
13
- * Free Software Foundation; version 2 of the License.
1411  */
1512 
1613 #include <linux/bitops.h>
....@@ -22,6 +19,7 @@
2219 #include <asm/byteorder.h>
2320 #include <asm/cacheflush.h>
2421 #include <asm/cpu-features.h>
22
+#include <asm/isa-rev.h>
2523 #include <asm/uasm.h>
2624 
2725 /* Registers used by JIT */
....@@ -79,8 +77,6 @@
7977 	REG_64BIT_32BIT,
8078 	/* 32-bit compatible, need truncation for 64-bit ops. */
8179 	REG_32BIT,
82
-	/* 32-bit zero extended. */
83
-	REG_32BIT_ZERO_EX,
8480 	/* 32-bit no sign/zero extension needed. */
8581 	REG_32BIT_POS
8682 };
....@@ -127,14 +123,20 @@
127123 }
128124 
129125 /* Simply emit the instruction if the JIT memory space has been allocated */
130
-#define emit_instr(ctx, func, ...)			\
131
-do {							\
132
-	if ((ctx)->target != NULL) {			\
133
-		u32 *p = &(ctx)->target[ctx->idx];	\
134
-		uasm_i_##func(&p, ##__VA_ARGS__);	\
135
-	}						\
136
-	(ctx)->idx++;					\
126
+#define emit_instr_long(ctx, func64, func32, ...)		\
127
+do {								\
128
+	if ((ctx)->target != NULL) {				\
129
+		u32 *p = &(ctx)->target[ctx->idx];		\
130
+		if (IS_ENABLED(CONFIG_64BIT))			\
131
+			uasm_i_##func64(&p, ##__VA_ARGS__);	\
132
+		else						\
133
+			uasm_i_##func32(&p, ##__VA_ARGS__);	\
134
+	}							\
135
+	(ctx)->idx++;						\
137136 } while (0)
137
+
138
+#define emit_instr(ctx, func, ...)				\
139
+	emit_instr_long(ctx, func, func, ##__VA_ARGS__)
138140 
139141 static unsigned int j_target(struct jit_ctx *ctx, int target_idx)
140142 {
....@@ -188,8 +190,9 @@
188190  * separate frame pointer, so BPF_REG_10 relative accesses are
189191  * adjusted to be $sp relative.
190192  */
191
-int ebpf_to_mips_reg(struct jit_ctx *ctx, const struct bpf_insn *insn,
192
-		     enum which_ebpf_reg w)
193
+static int ebpf_to_mips_reg(struct jit_ctx *ctx,
194
+			    const struct bpf_insn *insn,
195
+			    enum which_ebpf_reg w)
193196 {
194197 	int ebpf_reg = (w == src_reg || w == src_reg_no_fp) ?
195198 		insn->src_reg : insn->dst_reg;
....@@ -275,17 +278,17 @@
275278 		 * If RA we are doing a function call and may need
276279 		 * extra 8-byte tmp area.
277280 		 */
278
-		stack_adjust += 16;
281
+		stack_adjust += 2 * sizeof(long);
279282 	if (ctx->flags & EBPF_SAVE_S0)
280
-		stack_adjust += 8;
283
+		stack_adjust += sizeof(long);
281284 	if (ctx->flags & EBPF_SAVE_S1)
282
-		stack_adjust += 8;
285
+		stack_adjust += sizeof(long);
283286 	if (ctx->flags & EBPF_SAVE_S2)
284
-		stack_adjust += 8;
287
+		stack_adjust += sizeof(long);
285288 	if (ctx->flags & EBPF_SAVE_S3)
286
-		stack_adjust += 8;
289
+		stack_adjust += sizeof(long);
287290 	if (ctx->flags & EBPF_SAVE_S4)
288
-		stack_adjust += 8;
291
+		stack_adjust += sizeof(long);
289292 
290293 	BUILD_BUG_ON(MAX_BPF_STACK & 7);
291294 	locals_size = (ctx->flags & EBPF_SEEN_FP) ? MAX_BPF_STACK : 0;
....@@ -299,41 +302,49 @@
299302 	 * On tail call we skip this instruction, and the TCC is
300303 	 * passed in $v1 from the caller.
301304 	 */
302
-	emit_instr(ctx, daddiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
305
+	emit_instr(ctx, addiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
303306 	if (stack_adjust)
304
-		emit_instr(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack_adjust);
307
+		emit_instr_long(ctx, daddiu, addiu,
308
+					MIPS_R_SP, MIPS_R_SP, -stack_adjust);
305309 	else
306310 		return 0;
307311 
308
-	store_offset = stack_adjust - 8;
312
+	store_offset = stack_adjust - sizeof(long);
309313 
310314 	if (ctx->flags & EBPF_SAVE_RA) {
311
-		emit_instr(ctx, sd, MIPS_R_RA, store_offset, MIPS_R_SP);
312
-		store_offset -= 8;
315
+		emit_instr_long(ctx, sd, sw,
316
+					MIPS_R_RA, store_offset, MIPS_R_SP);
317
+		store_offset -= sizeof(long);
313318 	}
314319 	if (ctx->flags & EBPF_SAVE_S0) {
315
-		emit_instr(ctx, sd, MIPS_R_S0, store_offset, MIPS_R_SP);
316
-		store_offset -= 8;
320
+		emit_instr_long(ctx, sd, sw,
321
+					MIPS_R_S0, store_offset, MIPS_R_SP);
322
+		store_offset -= sizeof(long);
317323 	}
318324 	if (ctx->flags & EBPF_SAVE_S1) {
319
-		emit_instr(ctx, sd, MIPS_R_S1, store_offset, MIPS_R_SP);
320
-		store_offset -= 8;
325
+		emit_instr_long(ctx, sd, sw,
326
+					MIPS_R_S1, store_offset, MIPS_R_SP);
327
+		store_offset -= sizeof(long);
321328 	}
322329 	if (ctx->flags & EBPF_SAVE_S2) {
323
-		emit_instr(ctx, sd, MIPS_R_S2, store_offset, MIPS_R_SP);
324
-		store_offset -= 8;
330
+		emit_instr_long(ctx, sd, sw,
331
+					MIPS_R_S2, store_offset, MIPS_R_SP);
332
+		store_offset -= sizeof(long);
325333 	}
326334 	if (ctx->flags & EBPF_SAVE_S3) {
327
-		emit_instr(ctx, sd, MIPS_R_S3, store_offset, MIPS_R_SP);
328
-		store_offset -= 8;
335
+		emit_instr_long(ctx, sd, sw,
336
+					MIPS_R_S3, store_offset, MIPS_R_SP);
337
+		store_offset -= sizeof(long);
329338 	}
330339 	if (ctx->flags & EBPF_SAVE_S4) {
331
-		emit_instr(ctx, sd, MIPS_R_S4, store_offset, MIPS_R_SP);
332
-		store_offset -= 8;
340
+		emit_instr_long(ctx, sd, sw,
341
+					MIPS_R_S4, store_offset, MIPS_R_SP);
342
+		store_offset -= sizeof(long);
333343 	}
334344 
335345 	if ((ctx->flags & EBPF_SEEN_TC) && !(ctx->flags & EBPF_TCC_IN_V1))
336
-		emit_instr(ctx, daddu, MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);
346
+		emit_instr_long(ctx, daddu, addu,
347
+					MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);
337348 
338349 	return 0;
339350 }
....@@ -342,45 +353,52 @@
342353 {
343354 	const struct bpf_prog *prog = ctx->skf;
344355 	int stack_adjust = ctx->stack_size;
345
-	int store_offset = stack_adjust - 8;
356
+	int store_offset = stack_adjust - sizeof(long);
346357 	enum reg_val_type td;
347358 	int r0 = MIPS_R_V0;
348359 
349360 	if (dest_reg == MIPS_R_RA) {
350361 		/* Don't let zero extended value escape. */
351362 		td = get_reg_val_type(ctx, prog->len, BPF_REG_0);
352
-		if (td == REG_64BIT || td == REG_32BIT_ZERO_EX)
363
+		if (td == REG_64BIT)
353364 			emit_instr(ctx, sll, r0, r0, 0);
354365 	}
355366 
356367 	if (ctx->flags & EBPF_SAVE_RA) {
357
-		emit_instr(ctx, ld, MIPS_R_RA, store_offset, MIPS_R_SP);
358
-		store_offset -= 8;
368
+		emit_instr_long(ctx, ld, lw,
369
+					MIPS_R_RA, store_offset, MIPS_R_SP);
370
+		store_offset -= sizeof(long);
359371 	}
360372 	if (ctx->flags & EBPF_SAVE_S0) {
361
-		emit_instr(ctx, ld, MIPS_R_S0, store_offset, MIPS_R_SP);
362
-		store_offset -= 8;
373
+		emit_instr_long(ctx, ld, lw,
374
+					MIPS_R_S0, store_offset, MIPS_R_SP);
375
+		store_offset -= sizeof(long);
363376 	}
364377 	if (ctx->flags & EBPF_SAVE_S1) {
365
-		emit_instr(ctx, ld, MIPS_R_S1, store_offset, MIPS_R_SP);
366
-		store_offset -= 8;
378
+		emit_instr_long(ctx, ld, lw,
379
+					MIPS_R_S1, store_offset, MIPS_R_SP);
380
+		store_offset -= sizeof(long);
367381 	}
368382 	if (ctx->flags & EBPF_SAVE_S2) {
369
-		emit_instr(ctx, ld, MIPS_R_S2, store_offset, MIPS_R_SP);
370
-		store_offset -= 8;
383
+		emit_instr_long(ctx, ld, lw,
384
+				MIPS_R_S2, store_offset, MIPS_R_SP);
385
+		store_offset -= sizeof(long);
371386 	}
372387 	if (ctx->flags & EBPF_SAVE_S3) {
373
-		emit_instr(ctx, ld, MIPS_R_S3, store_offset, MIPS_R_SP);
374
-		store_offset -= 8;
388
+		emit_instr_long(ctx, ld, lw,
389
+					MIPS_R_S3, store_offset, MIPS_R_SP);
390
+		store_offset -= sizeof(long);
375391 	}
376392 	if (ctx->flags & EBPF_SAVE_S4) {
377
-		emit_instr(ctx, ld, MIPS_R_S4, store_offset, MIPS_R_SP);
378
-		store_offset -= 8;
393
+		emit_instr_long(ctx, ld, lw,
394
+					MIPS_R_S4, store_offset, MIPS_R_SP);
395
+		store_offset -= sizeof(long);
379396 	}
380397 	emit_instr(ctx, jr, dest_reg);
381398 
382399 	if (stack_adjust)
383
-		emit_instr(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, stack_adjust);
400
+		emit_instr_long(ctx, daddiu, addiu,
401
+					MIPS_R_SP, MIPS_R_SP, stack_adjust);
384402 	else
385403 		emit_instr(ctx, nop);
386404 
....@@ -648,6 +666,10 @@
648666 	s64 t64s;
649667 	int bpf_op = BPF_OP(insn->code);
650668 
669
+	if (IS_ENABLED(CONFIG_32BIT) && ((BPF_CLASS(insn->code) == BPF_ALU64)
670
+						|| (bpf_op == BPF_DW)))
671
+		return -EINVAL;
672
+
651673 	switch (insn->code) {
652674 	case BPF_ALU64 | BPF_ADD | BPF_K: /* ALU64_IMM */
653675 	case BPF_ALU64 | BPF_SUB | BPF_K: /* ALU64_IMM */
....@@ -680,8 +702,12 @@
680702 		if (insn->imm == 1) /* Mult by 1 is a nop */
681703 			break;
682704 		gen_imm_to_reg(insn, MIPS_R_AT, ctx);
683
-		emit_instr(ctx, dmultu, MIPS_R_AT, dst);
684
-		emit_instr(ctx, mflo, dst);
705
+		if (MIPS_ISA_REV >= 6) {
706
+			emit_instr(ctx, dmulu, dst, dst, MIPS_R_AT);
707
+		} else {
708
+			emit_instr(ctx, dmultu, MIPS_R_AT, dst);
709
+			emit_instr(ctx, mflo, dst);
710
+		}
685711 		break;
686712 	case BPF_ALU64 | BPF_NEG | BPF_K: /* ALU64_IMM */
687713 		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
....@@ -696,22 +722,26 @@
696722 		if (dst < 0)
697723 			return dst;
698724 		td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
699
-		if (td == REG_64BIT || td == REG_32BIT_ZERO_EX) {
725
+		if (td == REG_64BIT) {
700726 			/* sign extend */
701727 			emit_instr(ctx, sll, dst, dst, 0);
702728 		}
703729 		if (insn->imm == 1) /* Mult by 1 is a nop */
704730 			break;
705731 		gen_imm_to_reg(insn, MIPS_R_AT, ctx);
706
-		emit_instr(ctx, multu, dst, MIPS_R_AT);
707
-		emit_instr(ctx, mflo, dst);
732
+		if (MIPS_ISA_REV >= 6) {
733
+			emit_instr(ctx, mulu, dst, dst, MIPS_R_AT);
734
+		} else {
735
+			emit_instr(ctx, multu, dst, MIPS_R_AT);
736
+			emit_instr(ctx, mflo, dst);
737
+		}
708738 		break;
709739 	case BPF_ALU | BPF_NEG | BPF_K: /* ALU_IMM */
710740 		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
711741 		if (dst < 0)
712742 			return dst;
713743 		td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
714
-		if (td == REG_64BIT || td == REG_32BIT_ZERO_EX) {
744
+		if (td == REG_64BIT) {
715745 			/* sign extend */
716746 			emit_instr(ctx, sll, dst, dst, 0);
717747 		}
....@@ -725,7 +755,7 @@
725755 		if (dst < 0)
726756 			return dst;
727757 		td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
728
-		if (td == REG_64BIT || td == REG_32BIT_ZERO_EX)
758
+		if (td == REG_64BIT)
729759 			/* sign extend */
730760 			emit_instr(ctx, sll, dst, dst, 0);
731761 		if (insn->imm == 1) {
....@@ -735,6 +765,13 @@
735765 			break;
736766 		}
737767 		gen_imm_to_reg(insn, MIPS_R_AT, ctx);
768
+		if (MIPS_ISA_REV >= 6) {
769
+			if (bpf_op == BPF_DIV)
770
+				emit_instr(ctx, divu_r6, dst, dst, MIPS_R_AT);
771
+			else
772
+				emit_instr(ctx, modu, dst, dst, MIPS_R_AT);
773
+			break;
774
+		}
738775 		emit_instr(ctx, divu, dst, MIPS_R_AT);
739776 		if (bpf_op == BPF_DIV)
740777 			emit_instr(ctx, mflo, dst);
....@@ -757,6 +794,13 @@
757794 			break;
758795 		}
759796 		gen_imm_to_reg(insn, MIPS_R_AT, ctx);
797
+		if (MIPS_ISA_REV >= 6) {
798
+			if (bpf_op == BPF_DIV)
799
+				emit_instr(ctx, ddivu_r6, dst, dst, MIPS_R_AT);
800
+			else
801
+				emit_instr(ctx, modu, dst, dst, MIPS_R_AT);
802
+			break;
803
+		}
760804 		emit_instr(ctx, ddivu, dst, MIPS_R_AT);
761805 		if (bpf_op == BPF_DIV)
762806 			emit_instr(ctx, mflo, dst);
....@@ -822,11 +866,23 @@
822866 			emit_instr(ctx, and, dst, dst, src);
823867 			break;
824868 		case BPF_MUL:
825
-			emit_instr(ctx, dmultu, dst, src);
826
-			emit_instr(ctx, mflo, dst);
869
+			if (MIPS_ISA_REV >= 6) {
870
+				emit_instr(ctx, dmulu, dst, dst, src);
871
+			} else {
872
+				emit_instr(ctx, dmultu, dst, src);
873
+				emit_instr(ctx, mflo, dst);
874
+			}
827875 			break;
828876 		case BPF_DIV:
829877 		case BPF_MOD:
878
+			if (MIPS_ISA_REV >= 6) {
879
+				if (bpf_op == BPF_DIV)
880
+					emit_instr(ctx, ddivu_r6,
881
+							dst, dst, src);
882
+				else
883
+					emit_instr(ctx, modu, dst, dst, src);
884
+				break;
885
+			}
830886 			emit_instr(ctx, ddivu, dst, src);
831887 			if (bpf_op == BPF_DIV)
832888 				emit_instr(ctx, mflo, dst);
....@@ -858,18 +914,19 @@
858914 	case BPF_ALU | BPF_MOD | BPF_X: /* ALU_REG */
859915 	case BPF_ALU | BPF_LSH | BPF_X: /* ALU_REG */
860916 	case BPF_ALU | BPF_RSH | BPF_X: /* ALU_REG */
917
+	case BPF_ALU | BPF_ARSH | BPF_X: /* ALU_REG */
861918 		src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
862919 		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
863920 		if (src < 0 || dst < 0)
864921 			return -EINVAL;
865922 		td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
866
-		if (td == REG_64BIT || td == REG_32BIT_ZERO_EX) {
923
+		if (td == REG_64BIT) {
867924 			/* sign extend */
868925 			emit_instr(ctx, sll, dst, dst, 0);
869926 		}
870927 		did_move = false;
871928 		ts = get_reg_val_type(ctx, this_idx, insn->src_reg);
872
-		if (ts == REG_64BIT || ts == REG_32BIT_ZERO_EX) {
929
+		if (ts == REG_64BIT) {
873930 			int tmp_reg = MIPS_R_AT;
874931 
875932 			if (bpf_op == BPF_MOV) {
....@@ -905,6 +962,13 @@
905962 			break;
906963 		case BPF_DIV:
907964 		case BPF_MOD:
965
+			if (MIPS_ISA_REV >= 6) {
966
+				if (bpf_op == BPF_DIV)
967
+					emit_instr(ctx, divu_r6, dst, dst, src);
968
+				else
969
+					emit_instr(ctx, modu, dst, dst, src);
970
+				break;
971
+			}
908972 			emit_instr(ctx, divu, dst, src);
909973 			if (bpf_op == BPF_DIV)
910974 				emit_instr(ctx, mflo, dst);
....@@ -916,6 +980,9 @@
916980 			break;
917981 		case BPF_RSH:
918982 			emit_instr(ctx, srlv, dst, dst, src);
983
+			break;
984
+		case BPF_ARSH:
985
+			emit_instr(ctx, srav, dst, dst, src);
919986 			break;
920987 		default:
921988 			pr_err("ALU_REG NOT HANDLED\n");
....@@ -1005,8 +1072,15 @@
10051072 			emit_instr(ctx, dsubu, MIPS_R_T8, dst, src);
10061073 			emit_instr(ctx, sltu, MIPS_R_AT, dst, src);
10071074 			/* SP known to be non-zero, movz becomes boolean not */
1008
-			emit_instr(ctx, movz, MIPS_R_T9, MIPS_R_SP, MIPS_R_T8);
1009
-			emit_instr(ctx, movn, MIPS_R_T9, MIPS_R_ZERO, MIPS_R_T8);
1075
+			if (MIPS_ISA_REV >= 6) {
1076
+				emit_instr(ctx, seleqz, MIPS_R_T9,
1077
+						MIPS_R_SP, MIPS_R_T8);
1078
+			} else {
1079
+				emit_instr(ctx, movz, MIPS_R_T9,
1080
+						MIPS_R_SP, MIPS_R_T8);
1081
+				emit_instr(ctx, movn, MIPS_R_T9,
1082
+						MIPS_R_ZERO, MIPS_R_T8);
1083
+			}
10101084 			emit_instr(ctx, or, MIPS_R_AT, MIPS_R_T9, MIPS_R_AT);
10111085 			cmp_eq = bpf_op == BPF_JGT;
10121086 			dst = MIPS_R_AT;
....@@ -1233,7 +1307,7 @@
12331307 
12341308 	case BPF_JMP | BPF_CALL:
12351309 		ctx->flags |= EBPF_SAVE_RA;
1236
-		t64s = (s64)insn->imm + (s64)__bpf_call_base;
1310
+		t64s = (s64)insn->imm + (long)__bpf_call_base;
12371311 		emit_const_to_reg(ctx, MIPS_R_T9, (u64)t64s);
12381312 		emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
12391313 		/* delay slot */
....@@ -1254,8 +1328,7 @@
12541328 		if (insn->imm == 64 && td == REG_32BIT)
12551329 			emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
12561330 
1257
-		if (insn->imm != 64 &&
1258
-		    (td == REG_64BIT || td == REG_32BIT_ZERO_EX)) {
1331
+		if (insn->imm != 64 && td == REG_64BIT) {
12591332 			/* sign extend */
12601333 			emit_instr(ctx, sll, dst, dst, 0);
12611334 		}
....@@ -1369,6 +1442,17 @@
13691442 		if (src < 0)
13701443 			return src;
13711444 		if (BPF_MODE(insn->code) == BPF_XADD) {
1445
+			/*
1446
+			 * If mem_off does not fit within the 9 bit ll/sc
1447
+			 * instruction immediate field, use a temp reg.
1448
+			 */
1449
+			if (MIPS_ISA_REV >= 6 &&
1450
+			    (mem_off >= BIT(8) || mem_off < -BIT(8))) {
1451
+				emit_instr(ctx, daddiu, MIPS_R_T6,
1452
+						dst, mem_off);
1453
+				mem_off = 0;
1454
+				dst = MIPS_R_T6;
1455
+			}
13721456 			switch (BPF_SIZE(insn->code)) {
13731457 			case BPF_W:
13741458 				if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
....@@ -1723,7 +1807,7 @@
17231807 	unsigned int image_size;
17241808 	u8 *image_ptr;
17251809 
1726
-	if (!prog->jit_requested || !cpu_has_mips64r2)
1810
+	if (!prog->jit_requested)
17271811 		return prog;
17281812 
17291813 	tmp = bpf_jit_blind_constants(prog);