forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
....@@ -1,43 +1,15 @@
1
-/*
2
- * Copyright (C) 2016-2018 Netronome Systems, Inc.
3
- *
4
- * This software is dual licensed under the GNU General License Version 2,
5
- * June 1991 as shown in the file COPYING in the top-level directory of this
6
- * source tree or the BSD 2-Clause License provided below.  You have the
7
- * option to license this software under the complete terms of either license.
8
- *
9
- * The BSD 2-Clause License:
10
- *
11
- *     Redistribution and use in source and binary forms, with or
12
- *     without modification, are permitted provided that the following
13
- *     conditions are met:
14
- *
15
- *      1. Redistributions of source code must retain the above
16
- *         copyright notice, this list of conditions and the following
17
- *         disclaimer.
18
- *
19
- *      2. Redistributions in binary form must reproduce the above
20
- *         copyright notice, this list of conditions and the following
21
- *         disclaimer in the documentation and/or other materials
22
- *         provided with the distribution.
23
- *
24
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31
- * SOFTWARE.
32
- */
1
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2
+/* Copyright (C) 2016-2018 Netronome Systems, Inc. */
333 
344 #include <linux/bpf.h>
355 #include <linux/bpf_verifier.h>
366 #include <linux/kernel.h>
7
+#include <linux/netdevice.h>
378 #include <linux/pkt_cls.h>
389 
3910 #include "../nfp_app.h"
4011 #include "../nfp_main.h"
12
+#include "../nfp_net.h"
4113 #include "fw.h"
4214 #include "main.h"
4315 
....@@ -46,15 +18,15 @@
4618 
4719 struct nfp_insn_meta *
4820 nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
49
-		  unsigned int insn_idx, unsigned int n_insns)
21
+		  unsigned int insn_idx)
5022 {
5123 	unsigned int forward, backward, i;
5224 
5325 	backward = meta->n - insn_idx;
5426 	forward = insn_idx - meta->n;
5527 
56
-	if (min(forward, backward) > n_insns - insn_idx - 1) {
57
-		backward = n_insns - insn_idx - 1;
28
+	if (min(forward, backward) > nfp_prog->n_insns - insn_idx - 1) {
29
+		backward = nfp_prog->n_insns - insn_idx - 1;
5830 		meta = nfp_prog_last_meta(nfp_prog);
5931 	}
6032 	if (min(forward, backward) > insn_idx && backward > insn_idx) {
....@@ -195,8 +167,9 @@
195167 }
196168 
197169 static int
198
-nfp_bpf_check_call(struct nfp_prog *nfp_prog, struct bpf_verifier_env *env,
199
-		   struct nfp_insn_meta *meta)
170
+nfp_bpf_check_helper_call(struct nfp_prog *nfp_prog,
171
+			  struct bpf_verifier_env *env,
172
+			  struct nfp_insn_meta *meta)
200173 {
201174 	const struct bpf_reg_state *reg1 = cur_regs(env) + BPF_REG_1;
202175 	const struct bpf_reg_state *reg2 = cur_regs(env) + BPF_REG_2;
....@@ -373,6 +346,9 @@
373346 			   struct bpf_verifier_env *env)
374347 {
375348 	s32 old_off, new_off;
349
+
350
+	if (reg->frameno != env->cur_state->curframe)
351
+		meta->flags |= FLAG_INSN_PTR_CALLER_STACK_FRAME;
376352 
377353 	if (!tnum_is_const(reg->var_off)) {
378354 		pr_vlog(env, "variable ptr stack access\n");
....@@ -647,13 +623,13 @@
647623 	return 0;
648624 }
649625 
650
-static int
651
-nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
626
+int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx,
627
+		    int prev_insn_idx)
652628 {
653629 	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;
654630 	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;
655631 
656
-	meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx, env->prog->len);
632
+	meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx);
657633 	nfp_prog->verifier_meta = meta;
658634 
659635 	if (!nfp_bpf_supported_opcode(meta->insn.code)) {
....@@ -668,8 +644,8 @@
668644 		return -EINVAL;
669645 	}
670646 
671
-	if (meta->insn.code == (BPF_JMP | BPF_CALL))
672
-		return nfp_bpf_check_call(nfp_prog, env, meta);
647
+	if (is_mbpf_helper_call(meta))
648
+		return nfp_bpf_check_helper_call(nfp_prog, env, meta);
673649 	if (meta->insn.code == (BPF_JMP | BPF_EXIT))
674650 		return nfp_bpf_check_exit(nfp_prog, env);
675651 
....@@ -688,6 +664,195 @@
688664 	return 0;
689665 }
690666 
691
-const struct bpf_prog_offload_ops nfp_bpf_analyzer_ops = {
692
-	.insn_hook = nfp_verify_insn,
693
-};
667
+static int
668
+nfp_assign_subprog_idx_and_regs(struct bpf_verifier_env *env,
669
+				struct nfp_prog *nfp_prog)
670
+{
671
+	struct nfp_insn_meta *meta;
672
+	int index = 0;
673
+
674
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
675
+		if (nfp_is_subprog_start(meta))
676
+			index++;
677
+		meta->subprog_idx = index;
678
+
679
+		if (meta->insn.dst_reg >= BPF_REG_6 &&
680
+		    meta->insn.dst_reg <= BPF_REG_9)
681
+			nfp_prog->subprog[index].needs_reg_push = 1;
682
+	}
683
+
684
+	if (index + 1 != nfp_prog->subprog_cnt) {
685
+		pr_vlog(env, "BUG: number of processed BPF functions is not consistent (processed %d, expected %d)\n",
686
+			index + 1, nfp_prog->subprog_cnt);
687
+		return -EFAULT;
688
+	}
689
+
690
+	return 0;
691
+}
692
+
693
+static unsigned int nfp_bpf_get_stack_usage(struct nfp_prog *nfp_prog)
694
+{
695
+	struct nfp_insn_meta *meta = nfp_prog_first_meta(nfp_prog);
696
+	unsigned int max_depth = 0, depth = 0, frame = 0;
697
+	struct nfp_insn_meta *ret_insn[MAX_CALL_FRAMES];
698
+	unsigned short frame_depths[MAX_CALL_FRAMES];
699
+	unsigned short ret_prog[MAX_CALL_FRAMES];
700
+	unsigned short idx = meta->subprog_idx;
701
+
702
+	/* Inspired from check_max_stack_depth() from kernel verifier.
703
+	 * Starting from main subprogram, walk all instructions and recursively
704
+	 * walk all callees that given subprogram can call. Since recursion is
705
+	 * prevented by the kernel verifier, this algorithm only needs a local
706
+	 * stack of MAX_CALL_FRAMES to remember callsites.
707
+	 */
708
+process_subprog:
709
+	frame_depths[frame] = nfp_prog->subprog[idx].stack_depth;
710
+	frame_depths[frame] = round_up(frame_depths[frame], STACK_FRAME_ALIGN);
711
+	depth += frame_depths[frame];
712
+	max_depth = max(max_depth, depth);
713
+
714
+continue_subprog:
715
+	for (; meta != nfp_prog_last_meta(nfp_prog) && meta->subprog_idx == idx;
716
+	     meta = nfp_meta_next(meta)) {
717
+		if (!is_mbpf_pseudo_call(meta))
718
+			continue;
719
+
720
+		/* We found a call to a subprogram. Remember instruction to
721
+		 * return to and subprog id.
722
+		 */
723
+		ret_insn[frame] = nfp_meta_next(meta);
724
+		ret_prog[frame] = idx;
725
+
726
+		/* Find the callee and start processing it. */
727
+		meta = nfp_bpf_goto_meta(nfp_prog, meta,
728
+					 meta->n + 1 + meta->insn.imm);
729
+		idx = meta->subprog_idx;
730
+		frame++;
731
+		goto process_subprog;
732
+	}
733
+	/* End of for() loop means the last instruction of the subprog was
734
+	 * reached. If we popped all stack frames, return; otherwise, go on
735
+	 * processing remaining instructions from the caller.
736
+	 */
737
+	if (frame == 0)
738
+		return max_depth;
739
+
740
+	depth -= frame_depths[frame];
741
+	frame--;
742
+	meta = ret_insn[frame];
743
+	idx = ret_prog[frame];
744
+	goto continue_subprog;
745
+}
746
+
747
+static void nfp_bpf_insn_flag_zext(struct nfp_prog *nfp_prog,
748
+				   struct bpf_insn_aux_data *aux)
749
+{
750
+	struct nfp_insn_meta *meta;
751
+
752
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
753
+		if (aux[meta->n].zext_dst)
754
+			meta->flags |= FLAG_INSN_DO_ZEXT;
755
+	}
756
+}
757
+
758
+int nfp_bpf_finalize(struct bpf_verifier_env *env)
759
+{
760
+	struct bpf_subprog_info *info;
761
+	struct nfp_prog *nfp_prog;
762
+	unsigned int max_stack;
763
+	struct nfp_net *nn;
764
+	int i;
765
+
766
+	nfp_prog = env->prog->aux->offload->dev_priv;
767
+	nfp_prog->subprog_cnt = env->subprog_cnt;
768
+	nfp_prog->subprog = kcalloc(nfp_prog->subprog_cnt,
769
+				    sizeof(nfp_prog->subprog[0]), GFP_KERNEL);
770
+	if (!nfp_prog->subprog)
771
+		return -ENOMEM;
772
+
773
+	nfp_assign_subprog_idx_and_regs(env, nfp_prog);
774
+
775
+	info = env->subprog_info;
776
+	for (i = 0; i < nfp_prog->subprog_cnt; i++) {
777
+		nfp_prog->subprog[i].stack_depth = info[i].stack_depth;
778
+
779
+		if (i == 0)
780
+			continue;
781
+
782
+		/* Account for size of return address. */
783
+		nfp_prog->subprog[i].stack_depth += REG_WIDTH;
784
+		/* Account for size of saved registers, if necessary. */
785
+		if (nfp_prog->subprog[i].needs_reg_push)
786
+			nfp_prog->subprog[i].stack_depth += BPF_REG_SIZE * 4;
787
+	}
788
+
789
+	nn = netdev_priv(env->prog->aux->offload->netdev);
790
+	max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
791
+	nfp_prog->stack_size = nfp_bpf_get_stack_usage(nfp_prog);
792
+	if (nfp_prog->stack_size > max_stack) {
793
+		pr_vlog(env, "stack too large: program %dB > FW stack %dB\n",
794
+			nfp_prog->stack_size, max_stack);
795
+		return -EOPNOTSUPP;
796
+	}
797
+
798
+	nfp_bpf_insn_flag_zext(nfp_prog, env->insn_aux_data);
799
+	return 0;
800
+}
801
+
802
+int nfp_bpf_opt_replace_insn(struct bpf_verifier_env *env, u32 off,
803
+			     struct bpf_insn *insn)
804
+{
805
+	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;
806
+	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
807
+	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;
808
+
809
+	meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx);
810
+	nfp_prog->verifier_meta = meta;
811
+
812
+	/* conditional jump to jump conversion */
813
+	if (is_mbpf_cond_jump(meta) &&
814
+	    insn->code == (BPF_JMP | BPF_JA | BPF_K)) {
815
+		unsigned int tgt_off;
816
+
817
+		tgt_off = off + insn->off + 1;
818
+
819
+		if (!insn->off) {
820
+			meta->jmp_dst = list_next_entry(meta, l);
821
+			meta->jump_neg_op = false;
822
+		} else if (meta->jmp_dst->n != aux_data[tgt_off].orig_idx) {
823
+			pr_vlog(env, "branch hard wire at %d changes target %d -> %d\n",
824
+				off, meta->jmp_dst->n,
825
+				aux_data[tgt_off].orig_idx);
826
+			return -EINVAL;
827
+		}
828
+		return 0;
829
+	}
830
+
831
+	pr_vlog(env, "unsupported instruction replacement %hhx -> %hhx\n",
832
+		meta->insn.code, insn->code);
833
+	return -EINVAL;
834
+}
835
+
836
+int nfp_bpf_opt_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
837
+{
838
+	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;
839
+	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
840
+	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;
841
+	unsigned int i;
842
+
843
+	meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx);
844
+
845
+	for (i = 0; i < cnt; i++) {
846
+		if (WARN_ON_ONCE(&meta->l == &nfp_prog->insns))
847
+			return -EINVAL;
848
+
849
+		/* doesn't count if it already has the flag */
850
+		if (meta->flags & FLAG_INSN_SKIP_VERIFIER_OPT)
851
+			i--;
852
+
853
+		meta->flags |= FLAG_INSN_SKIP_VERIFIER_OPT;
854
+		meta = list_next_entry(meta, l);
855
+	}
856
+
857
+	return 0;
858
+}