forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-10-22 8ac6c7a54ed1b98d142dce24b11c6de6a1e239a5 
 kernel/arch/arm64/net/bpf_jit_comp.c
....@@ -1,23 +1,13 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * BPF JIT compiler for ARM64
34  *
45  * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
5
- *
6
- * This program is free software; you can redistribute it and/or modify
7
- * it under the terms of the GNU General Public License version 2 as
8
- * published by the Free Software Foundation.
9
- *
10
- * This program is distributed in the hope that it will be useful,
11
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13
- * GNU General Public License for more details.
14
- *
15
- * You should have received a copy of the GNU General Public License
16
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
176  */
187 
198 #define pr_fmt(fmt) "bpf_jit: " fmt
209 
10
+#include <linux/bitfield.h>
2111 #include <linux/bpf.h>
2212 #include <linux/filter.h>
2313 #include <linux/printk.h>
....@@ -27,6 +17,7 @@
2717 #include <asm/cacheflush.h>
2818 #include <asm/debug-monitors.h>
2919 #include <asm/set_memory.h>
20
+#include <trace/hooks/memory.h>
3021 
3122 #include "bpf_jit.h"
3223 
....@@ -67,6 +58,7 @@
6758 	int idx;
6859 	int epilogue_offset;
6960 	int *offset;
61
+	int exentry_idx;
7062 	__le32 *image;
7163 	u32 stack_size;
7264 };
....@@ -134,10 +126,9 @@
134126 }
135127 
136128 /*
137
- * This is an unoptimized 64 immediate emission used for BPF to BPF call
138
- * addresses. It will always do a full 64 bit decomposition as otherwise
139
- * more complexity in the last extra pass is required since we previously
140
- * reserved 4 instructions for the address.
129
+ * Kernel addresses in the vmalloc space use at most 48 bits, and the
130
+ * remaining bits are guaranteed to be 0x1. So we can compose the address
131
+ * with a fixed length movn/movk/movk sequence.
141132  */
142133 static inline void emit_addr_mov_i64(const int reg, const u64 val,
143134 				     struct jit_ctx *ctx)
....@@ -145,22 +136,25 @@
145136 	u64 tmp = val;
146137 	int shift = 0;
147138 
148
-	emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx);
149
-	for (;shift < 48;) {
139
+	emit(A64_MOVN(1, reg, ~tmp & 0xffff, shift), ctx);
140
+	while (shift < 32) {
150141 		tmp >>= 16;
151142 		shift += 16;
152143 		emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx);
153144 	}
154145 }
155146 
156
-static inline int bpf2a64_offset(int bpf_to, int bpf_from,
147
+static inline int bpf2a64_offset(int bpf_insn, int off,
157148 				 const struct jit_ctx *ctx)
158149 {
159
-	int to = ctx->offset[bpf_to];
160
-	/* -1 to account for the Branch instruction */
161
-	int from = ctx->offset[bpf_from] - 1;
162
-
163
-	return to - from;
150
+	/* BPF JMP offset is relative to the next instruction */
151
+	bpf_insn++;
152
+	/*
153
+	 * Whereas arm64 branch instructions encode the offset
154
+	 * from the branch itself, so we must subtract 1 from the
155
+	 * instruction offset.
156
+	 */
157
+	return ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1);
164158 }
165159 
166160 static void jit_fill_hole(void *area, unsigned int size)
....@@ -179,11 +173,21 @@
179173 	return to - from;
180174 }
181175 
176
+static bool is_addsub_imm(u32 imm)
177
+{
178
+	/* Either imm12 or shifted imm12. */
179
+	return !(imm & ~0xfff) || !(imm & ~0xfff000);
180
+}
181
+
182182 /* Stack must be multiples of 16B */
183183 #define STACK_ALIGN(sz) (((sz) + 15) & ~15)
184184 
185185 /* Tail call offset to jump into */
186
+#if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)
187
+#define PROLOGUE_OFFSET 8
188
+#else
186189 #define PROLOGUE_OFFSET 7
190
+#endif
187191 
188192 static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
189193 {
....@@ -220,6 +224,10 @@
220224 	 *
221225 	 */
222226 
227
+	/* BTI landing pad */
228
+	if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
229
+		emit(A64_BTI_C, ctx);
230
+
223231 	/* Save FP and LR registers to stay align with ARM64 AAPCS */
224232 	emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
225233 	emit(A64_MOV(1, A64_FP, A64_SP), ctx);
....@@ -242,6 +250,10 @@
242250 				    cur_offset, PROLOGUE_OFFSET);
243251 			return -1;
244252 		}
253
+
254
+		/* BTI landing pad for the tail call, done with a BR */
255
+		if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
256
+			emit(A64_BTI_J, ctx);
245257 	}
246258 
247259 	ctx->stack_size = STACK_ALIGN(prog->aux->stack_depth);
....@@ -345,13 +357,75 @@
345357 	emit(A64_RET(A64_LR), ctx);
346358 }
347359 
360
+#define BPF_FIXUP_OFFSET_MASK	GENMASK(26, 0)
361
+#define BPF_FIXUP_REG_MASK	GENMASK(31, 27)
362
+
363
+int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
364
+			      struct pt_regs *regs)
365
+{
366
+	off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
367
+	int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
368
+
369
+	regs->regs[dst_reg] = 0;
370
+	regs->pc = (unsigned long)&ex->fixup - offset;
371
+	return 1;
372
+}
373
+
374
+/* For accesses to BTF pointers, add an entry to the exception table */
375
+static int add_exception_handler(const struct bpf_insn *insn,
376
+				 struct jit_ctx *ctx,
377
+				 int dst_reg)
378
+{
379
+	off_t offset;
380
+	unsigned long pc;
381
+	struct exception_table_entry *ex;
382
+
383
+	if (!ctx->image)
384
+		/* First pass */
385
+		return 0;
386
+
387
+	if (BPF_MODE(insn->code) != BPF_PROBE_MEM)
388
+		return 0;
389
+
390
+	if (!ctx->prog->aux->extable ||
391
+	    WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries))
392
+		return -EINVAL;
393
+
394
+	ex = &ctx->prog->aux->extable[ctx->exentry_idx];
395
+	pc = (unsigned long)&ctx->image[ctx->idx - 1];
396
+
397
+	offset = pc - (long)&ex->insn;
398
+	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
399
+		return -ERANGE;
400
+	ex->insn = offset;
401
+
402
+	/*
403
+	 * Since the extable follows the program, the fixup offset is always
404
+	 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
405
+	 * to keep things simple, and put the destination register in the upper
406
+	 * bits. We don't need to worry about buildtime or runtime sort
407
+	 * modifying the upper bits because the table is already sorted, and
408
+	 * isn't part of the main exception table.
409
+	 */
410
+	offset = (long)&ex->fixup - (pc + AARCH64_INSN_SIZE);
411
+	if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
412
+		return -ERANGE;
413
+
414
+	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |
415
+		    FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
416
+
417
+	ctx->exentry_idx++;
418
+	return 0;
419
+}
420
+
348421 /* JITs an eBPF instruction.
349422  * Returns:
350423  * 0  - successfully JITed an 8-byte eBPF instruction.
351424  * >0 - successfully JITed a 16-byte eBPF instruction.
352425  * <0 - failed to JIT.
353426  */
354
-static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
427
+static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
428
+		      bool extra_pass)
355429 {
356430 	const u8 code = insn->code;
357431 	const u8 dst = bpf2a64[insn->dst_reg];
....@@ -362,10 +436,13 @@
362436 	const s16 off = insn->off;
363437 	const s32 imm = insn->imm;
364438 	const int i = insn - ctx->prog->insnsi;
365
-	const bool is64 = BPF_CLASS(code) == BPF_ALU64;
439
+	const bool is64 = BPF_CLASS(code) == BPF_ALU64 ||
440
+			  BPF_CLASS(code) == BPF_JMP;
366441 	const bool isdw = BPF_SIZE(code) == BPF_DW;
367442 	u8 jmp_cond, reg;
368443 	s32 jmp_offset;
444
+	u32 a64_insn;
445
+	int ret;
369446 
370447 #define check_imm(bits, imm) do {				\
371448 	if ((((imm) > 0) && ((imm) >> (bits))) ||		\
....@@ -419,8 +496,7 @@
419496 			break;
420497 		case BPF_MOD:
421498 			emit(A64_UDIV(is64, tmp, dst, src), ctx);
422
-			emit(A64_MUL(is64, tmp, tmp, src), ctx);
423
-			emit(A64_SUB(is64, dst, dst, tmp), ctx);
499
+			emit(A64_MSUB(is64, dst, dst, tmp, src), ctx);
424500 			break;
425501 		}
426502 		break;
....@@ -489,28 +565,55 @@
489565 	/* dst = dst OP imm */
490566 	case BPF_ALU | BPF_ADD | BPF_K:
491567 	case BPF_ALU64 | BPF_ADD | BPF_K:
492
-		emit_a64_mov_i(is64, tmp, imm, ctx);
493
-		emit(A64_ADD(is64, dst, dst, tmp), ctx);
568
+		if (is_addsub_imm(imm)) {
569
+			emit(A64_ADD_I(is64, dst, dst, imm), ctx);
570
+		} else if (is_addsub_imm(-imm)) {
571
+			emit(A64_SUB_I(is64, dst, dst, -imm), ctx);
572
+		} else {
573
+			emit_a64_mov_i(is64, tmp, imm, ctx);
574
+			emit(A64_ADD(is64, dst, dst, tmp), ctx);
575
+		}
494576 		break;
495577 	case BPF_ALU | BPF_SUB | BPF_K:
496578 	case BPF_ALU64 | BPF_SUB | BPF_K:
497
-		emit_a64_mov_i(is64, tmp, imm, ctx);
498
-		emit(A64_SUB(is64, dst, dst, tmp), ctx);
579
+		if (is_addsub_imm(imm)) {
580
+			emit(A64_SUB_I(is64, dst, dst, imm), ctx);
581
+		} else if (is_addsub_imm(-imm)) {
582
+			emit(A64_ADD_I(is64, dst, dst, -imm), ctx);
583
+		} else {
584
+			emit_a64_mov_i(is64, tmp, imm, ctx);
585
+			emit(A64_SUB(is64, dst, dst, tmp), ctx);
586
+		}
499587 		break;
500588 	case BPF_ALU | BPF_AND | BPF_K:
501589 	case BPF_ALU64 | BPF_AND | BPF_K:
502
-		emit_a64_mov_i(is64, tmp, imm, ctx);
503
-		emit(A64_AND(is64, dst, dst, tmp), ctx);
590
+		a64_insn = A64_AND_I(is64, dst, dst, imm);
591
+		if (a64_insn != AARCH64_BREAK_FAULT) {
592
+			emit(a64_insn, ctx);
593
+		} else {
594
+			emit_a64_mov_i(is64, tmp, imm, ctx);
595
+			emit(A64_AND(is64, dst, dst, tmp), ctx);
596
+		}
504597 		break;
505598 	case BPF_ALU | BPF_OR | BPF_K:
506599 	case BPF_ALU64 | BPF_OR | BPF_K:
507
-		emit_a64_mov_i(is64, tmp, imm, ctx);
508
-		emit(A64_ORR(is64, dst, dst, tmp), ctx);
600
+		a64_insn = A64_ORR_I(is64, dst, dst, imm);
601
+		if (a64_insn != AARCH64_BREAK_FAULT) {
602
+			emit(a64_insn, ctx);
603
+		} else {
604
+			emit_a64_mov_i(is64, tmp, imm, ctx);
605
+			emit(A64_ORR(is64, dst, dst, tmp), ctx);
606
+		}
509607 		break;
510608 	case BPF_ALU | BPF_XOR | BPF_K:
511609 	case BPF_ALU64 | BPF_XOR | BPF_K:
512
-		emit_a64_mov_i(is64, tmp, imm, ctx);
513
-		emit(A64_EOR(is64, dst, dst, tmp), ctx);
610
+		a64_insn = A64_EOR_I(is64, dst, dst, imm);
611
+		if (a64_insn != AARCH64_BREAK_FAULT) {
612
+			emit(a64_insn, ctx);
613
+		} else {
614
+			emit_a64_mov_i(is64, tmp, imm, ctx);
615
+			emit(A64_EOR(is64, dst, dst, tmp), ctx);
616
+		}
514617 		break;
515618 	case BPF_ALU | BPF_MUL | BPF_K:
516619 	case BPF_ALU64 | BPF_MUL | BPF_K:
....@@ -526,8 +629,7 @@
526629 	case BPF_ALU64 | BPF_MOD | BPF_K:
527630 		emit_a64_mov_i(is64, tmp2, imm, ctx);
528631 		emit(A64_UDIV(is64, tmp, dst, tmp2), ctx);
529
-		emit(A64_MUL(is64, tmp, tmp, tmp2), ctx);
530
-		emit(A64_SUB(is64, dst, dst, tmp), ctx);
632
+		emit(A64_MSUB(is64, dst, dst, tmp, tmp2), ctx);
531633 		break;
532634 	case BPF_ALU | BPF_LSH | BPF_K:
533635 	case BPF_ALU64 | BPF_LSH | BPF_K:
....@@ -544,7 +646,7 @@
544646 
545647 	/* JUMP off */
546648 	case BPF_JMP | BPF_JA:
547
-		jmp_offset = bpf2a64_offset(i + off, i, ctx);
649
+		jmp_offset = bpf2a64_offset(i, off, ctx);
548650 		check_imm26(jmp_offset);
549651 		emit(A64_B(jmp_offset), ctx);
550652 		break;
....@@ -559,9 +661,19 @@
559661 	case BPF_JMP | BPF_JSLT | BPF_X:
560662 	case BPF_JMP | BPF_JSGE | BPF_X:
561663 	case BPF_JMP | BPF_JSLE | BPF_X:
562
-		emit(A64_CMP(1, dst, src), ctx);
664
+	case BPF_JMP32 | BPF_JEQ | BPF_X:
665
+	case BPF_JMP32 | BPF_JGT | BPF_X:
666
+	case BPF_JMP32 | BPF_JLT | BPF_X:
667
+	case BPF_JMP32 | BPF_JGE | BPF_X:
668
+	case BPF_JMP32 | BPF_JLE | BPF_X:
669
+	case BPF_JMP32 | BPF_JNE | BPF_X:
670
+	case BPF_JMP32 | BPF_JSGT | BPF_X:
671
+	case BPF_JMP32 | BPF_JSLT | BPF_X:
672
+	case BPF_JMP32 | BPF_JSGE | BPF_X:
673
+	case BPF_JMP32 | BPF_JSLE | BPF_X:
674
+		emit(A64_CMP(is64, dst, src), ctx);
563675 emit_cond_jmp:
564
-		jmp_offset = bpf2a64_offset(i + off, i, ctx);
676
+		jmp_offset = bpf2a64_offset(i, off, ctx);
565677 		check_imm19(jmp_offset);
566678 		switch (BPF_OP(code)) {
567679 		case BPF_JEQ:
....@@ -601,7 +713,8 @@
601713 		emit(A64_B_(jmp_cond, jmp_offset), ctx);
602714 		break;
603715 	case BPF_JMP | BPF_JSET | BPF_X:
604
-		emit(A64_TST(1, dst, src), ctx);
716
+	case BPF_JMP32 | BPF_JSET | BPF_X:
717
+		emit(A64_TST(is64, dst, src), ctx);
605718 		goto emit_cond_jmp;
606719 	/* IF (dst COND imm) JUMP off */
607720 	case BPF_JMP | BPF_JEQ | BPF_K:
....@@ -614,23 +727,47 @@
614727 	case BPF_JMP | BPF_JSLT | BPF_K:
615728 	case BPF_JMP | BPF_JSGE | BPF_K:
616729 	case BPF_JMP | BPF_JSLE | BPF_K:
617
-		emit_a64_mov_i(1, tmp, imm, ctx);
618
-		emit(A64_CMP(1, dst, tmp), ctx);
730
+	case BPF_JMP32 | BPF_JEQ | BPF_K:
731
+	case BPF_JMP32 | BPF_JGT | BPF_K:
732
+	case BPF_JMP32 | BPF_JLT | BPF_K:
733
+	case BPF_JMP32 | BPF_JGE | BPF_K:
734
+	case BPF_JMP32 | BPF_JLE | BPF_K:
735
+	case BPF_JMP32 | BPF_JNE | BPF_K:
736
+	case BPF_JMP32 | BPF_JSGT | BPF_K:
737
+	case BPF_JMP32 | BPF_JSLT | BPF_K:
738
+	case BPF_JMP32 | BPF_JSGE | BPF_K:
739
+	case BPF_JMP32 | BPF_JSLE | BPF_K:
740
+		if (is_addsub_imm(imm)) {
741
+			emit(A64_CMP_I(is64, dst, imm), ctx);
742
+		} else if (is_addsub_imm(-imm)) {
743
+			emit(A64_CMN_I(is64, dst, -imm), ctx);
744
+		} else {
745
+			emit_a64_mov_i(is64, tmp, imm, ctx);
746
+			emit(A64_CMP(is64, dst, tmp), ctx);
747
+		}
619748 		goto emit_cond_jmp;
620749 	case BPF_JMP | BPF_JSET | BPF_K:
621
-		emit_a64_mov_i(1, tmp, imm, ctx);
622
-		emit(A64_TST(1, dst, tmp), ctx);
750
+	case BPF_JMP32 | BPF_JSET | BPF_K:
751
+		a64_insn = A64_TST_I(is64, dst, imm);
752
+		if (a64_insn != AARCH64_BREAK_FAULT) {
753
+			emit(a64_insn, ctx);
754
+		} else {
755
+			emit_a64_mov_i(is64, tmp, imm, ctx);
756
+			emit(A64_TST(is64, dst, tmp), ctx);
757
+		}
623758 		goto emit_cond_jmp;
624759 	/* function call */
625760 	case BPF_JMP | BPF_CALL:
626761 	{
627762 		const u8 r0 = bpf2a64[BPF_REG_0];
628
-		const u64 func = (u64)__bpf_call_base + imm;
763
+		bool func_addr_fixed;
764
+		u64 func_addr;
629765 
630
-		if (ctx->prog->is_func)
631
-			emit_addr_mov_i64(tmp, func, ctx);
632
-		else
633
-			emit_a64_mov_i64(tmp, func, ctx);
766
+		ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
767
+					    &func_addr, &func_addr_fixed);
768
+		if (ret < 0)
769
+			return ret;
770
+		emit_addr_mov_i64(tmp, func_addr, ctx);
634771 		emit(A64_BLR(tmp), ctx);
635772 		emit(A64_MOV(1, r0, A64_R(0)), ctx);
636773 		break;
....@@ -668,6 +805,10 @@
668805 	case BPF_LDX | BPF_MEM | BPF_H:
669806 	case BPF_LDX | BPF_MEM | BPF_B:
670807 	case BPF_LDX | BPF_MEM | BPF_DW:
808
+	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
809
+	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
810
+	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
811
+	case BPF_LDX | BPF_PROBE_MEM | BPF_B:
671812 		emit_a64_mov_i(1, tmp, off, ctx);
672813 		switch (BPF_SIZE(code)) {
673814 		case BPF_W:
....@@ -683,6 +824,10 @@
683824 			emit(A64_LDR64(dst, src, tmp), ctx);
684825 			break;
685826 		}
827
+
828
+		ret = add_exception_handler(insn, ctx, dst);
829
+		if (ret)
830
+			return ret;
686831 		break;
687832 
688833 	/* speculation barrier */
....@@ -775,27 +920,43 @@
775920 	return 0;
776921 }
777922 
778
-static int build_body(struct jit_ctx *ctx)
923
+static int build_body(struct jit_ctx *ctx, bool extra_pass)
779924 {
780925 	const struct bpf_prog *prog = ctx->prog;
781926 	int i;
782927 
928
+	/*
929
+	 * - offset[0] offset of the end of prologue,
930
+	 *   start of the 1st instruction.
931
+	 * - offset[1] - offset of the end of 1st instruction,
932
+	 *   start of the 2nd instruction
933
+	 * [....]
934
+	 * - offset[3] - offset of the end of 3rd instruction,
935
+	 *   start of 4th instruction
936
+	 */
783937 	for (i = 0; i < prog->len; i++) {
784938 		const struct bpf_insn *insn = &prog->insnsi[i];
785939 		int ret;
786940 
787
-		ret = build_insn(insn, ctx);
941
+		if (ctx->image == NULL)
942
+			ctx->offset[i] = ctx->idx;
943
+		ret = build_insn(insn, ctx, extra_pass);
788944 		if (ret > 0) {
789945 			i++;
790946 			if (ctx->image == NULL)
791947 				ctx->offset[i] = ctx->idx;
792948 			continue;
793949 		}
794
-		if (ctx->image == NULL)
795
-			ctx->offset[i] = ctx->idx;
796950 		if (ret)
797951 			return ret;
798952 	}
953
+	/*
954
+	 * offset is allocated with prog->len + 1 so fill in
955
+	 * the last element with the offset after the last
956
+	 * instruction (end of program)
957
+	 */
958
+	if (ctx->image == NULL)
959
+		ctx->offset[i] = ctx->idx;
799960 
800961 	return 0;
801962 }
....@@ -810,6 +971,9 @@
810971 		if (a64_insn == AARCH64_BREAK_FAULT)
811972 			return -1;
812973 	}
974
+
975
+	if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries))
976
+		return -1;
813977 
814978 	return 0;
815979 }
....@@ -827,6 +991,7 @@
827991 
828992 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
829993 {
994
+	int image_size, prog_size, extable_size;
830995 	struct bpf_prog *tmp, *orig_prog = prog;
831996 	struct bpf_binary_header *header;
832997 	struct arm64_jit_data *jit_data;
....@@ -834,7 +999,6 @@
834999 	bool tmp_blinded = false;
8351000 	bool extra_pass = false;
8361001 	struct jit_ctx ctx;
837
-	int image_size;
8381002 	u8 *image_ptr;
8391003 
8401004 	if (!prog->jit_requested)
....@@ -865,27 +1029,30 @@
8651029 		image_ptr = jit_data->image;
8661030 		header = jit_data->header;
8671031 		extra_pass = true;
868
-		image_size = sizeof(u32) * ctx.idx;
1032
+		prog_size = sizeof(u32) * ctx.idx;
8691033 		goto skip_init_ctx;
8701034 	}
8711035 	memset(&ctx, 0, sizeof(ctx));
8721036 	ctx.prog = prog;
8731037 
874
-	ctx.offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
1038
+	ctx.offset = kcalloc(prog->len + 1, sizeof(int), GFP_KERNEL);
8751039 	if (ctx.offset == NULL) {
8761040 		prog = orig_prog;
8771041 		goto out_off;
8781042 	}
8791043 
880
-	/* 1. Initial fake pass to compute ctx->idx. */
881
-
882
-	/* Fake pass to fill in ctx->offset. */
883
-	if (build_body(&ctx)) {
1044
+	/*
1045
+	 * 1. Initial fake pass to compute ctx->idx and ctx->offset.
1046
+	 *
1047
+	 * BPF line info needs ctx->offset[i] to be the offset of
1048
+	 * instruction[i] in jited image, so build prologue first.
1049
+	 */
1050
+	if (build_prologue(&ctx, was_classic)) {
8841051 		prog = orig_prog;
8851052 		goto out_off;
8861053 	}
8871054 
888
-	if (build_prologue(&ctx, was_classic)) {
1055
+	if (build_body(&ctx, extra_pass)) {
8891056 		prog = orig_prog;
8901057 		goto out_off;
8911058 	}
....@@ -893,8 +1060,12 @@
8931060 	ctx.epilogue_offset = ctx.idx;
8941061 	build_epilogue(&ctx);
8951062 
1063
+	extable_size = prog->aux->num_exentries *
1064
+		sizeof(struct exception_table_entry);
1065
+
8961066 	/* Now we know the actual image size. */
897
-	image_size = sizeof(u32) * ctx.idx;
1067
+	prog_size = sizeof(u32) * ctx.idx;
1068
+	image_size = prog_size + extable_size;
8981069 	header = bpf_jit_binary_alloc(image_size, &image_ptr,
8991070 				      sizeof(u32), jit_fill_hole);
9001071 	if (header == NULL) {
....@@ -905,12 +1076,15 @@
9051076 	/* 2. Now, the actual pass. */
9061077 
9071078 	ctx.image = (__le32 *)image_ptr;
1079
+	if (extable_size)
1080
+		prog->aux->extable = (void *)image_ptr + prog_size;
9081081 skip_init_ctx:
9091082 	ctx.idx = 0;
1083
+	ctx.exentry_idx = 0;
9101084 
9111085 	build_prologue(&ctx, was_classic);
9121086 
913
-	if (build_body(&ctx)) {
1087
+	if (build_body(&ctx, extra_pass)) {
9141088 		bpf_jit_binary_free(header);
9151089 		prog = orig_prog;
9161090 		goto out_off;
....@@ -927,7 +1101,7 @@
9271101 
9281102 	/* And we're done. */
9291103 	if (bpf_jit_enable > 1)
930
-		bpf_jit_dump(prog->len, image_size, 2, ctx.image);
1104
+		bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
9311105 
9321106 	bpf_flush_icache(header, ctx.image + ctx.idx);
9331107 
....@@ -938,9 +1112,12 @@
9381112 			bpf_jit_binary_free(header);
9391113 			prog->bpf_func = NULL;
9401114 			prog->jited = 0;
1115
+			prog->jited_len = 0;
9411116 			goto out_off;
9421117 		}
9431118 		bpf_jit_binary_lock_ro(header);
1119
+		trace_android_vh_set_memory_ro((unsigned long)header, header->pages);
1120
+		trace_android_vh_set_memory_x((unsigned long)header, header->pages);
9441121 	} else {
9451122 		jit_data->ctx = ctx;
9461123 		jit_data->image = image_ptr;
....@@ -948,9 +1125,15 @@
9481125 	}
9491126 	prog->bpf_func = (void *)ctx.image;
9501127 	prog->jited = 1;
951
-	prog->jited_len = image_size;
1128
+	prog->jited_len = prog_size;
9521129 
9531130 	if (!prog->is_func || extra_pass) {
1131
+		int i;
1132
+
1133
+		/* offset[prog->len] is the size of program */
1134
+		for (i = 0; i <= prog->len; i++)
1135
+			ctx.offset[i] *= AARCH64_INSN_SIZE;
1136
+		bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
9541137 out_off:
9551138 		kfree(ctx.offset);
9561139 		kfree(jit_data);
....@@ -963,24 +1146,17 @@
9631146 	return prog;
9641147 }
9651148 
966
-#ifdef CONFIG_CFI_CLANG
967
-bool arch_bpf_jit_check_func(const struct bpf_prog *prog)
1149
+u64 bpf_jit_alloc_exec_limit(void)
9681150 {
969
-	const uintptr_t func = (const uintptr_t)prog->bpf_func;
970
-
971
-	/*
972
-	 * bpf_func must be correctly aligned and within the correct region.
973
-	 * module_alloc places JIT code in the module region, unless
974
-	 * ARM64_MODULE_PLTS is enabled, in which case we might end up using
975
-	 * the vmalloc region too.
976
-	 */
977
-	if (unlikely(!IS_ALIGNED(func, sizeof(u32))))
978
-		return false;
979
-
980
-	if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
981
-			is_vmalloc_addr(prog->bpf_func))
982
-		return true;
983
-
984
-	return (func >= MODULES_VADDR && func < MODULES_END);
1151
+	return VMALLOC_END - VMALLOC_START;
9851152 }
986
-#endif
1153
+
1154
+void *bpf_jit_alloc_exec(unsigned long size)
1155
+{
1156
+	return vmalloc(size);
1157
+}
1158
+
1159
+void bpf_jit_free_exec(void *addr)
1160
+{
1161
+	return vfree(addr);
1162
+}