forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/include/linux/filter.h
....@@ -16,13 +16,16 @@
1616 #include <linux/workqueue.h>
1717 #include <linux/sched.h>
1818 #include <linux/capability.h>
19
-#include <linux/cryptohash.h>
2019 #include <linux/set_memory.h>
2120 #include <linux/kallsyms.h>
2221 #include <linux/if_vlan.h>
22
+#include <linux/vmalloc.h>
23
+#include <linux/sockptr.h>
24
+#include <crypto/sha.h>
2325 
2426 #include <net/sch_generic.h>
2527 
28
+#include <asm/byteorder.h>
2629 #include <uapi/linux/filter.h>
2730 #include <uapi/linux/bpf.h>
2831 
....@@ -33,6 +36,8 @@
3336 struct xdp_rxq_info;
3437 struct xdp_buff;
3538 struct sock_reuseport;
39
+struct ctl_table;
40
+struct ctl_table_header;
3641 
3742 /* ArgX, context and stack frame pointer register positions. Note,
3843  * Arg1, Arg2, Arg3, etc are used as argument mappings of function
....@@ -61,6 +66,9 @@
6166 /* unused opcode to mark special call to bpf_tail_call() helper */
6267 #define BPF_TAIL_CALL	0xf0
6368 
69
+/* unused opcode to mark special load instruction. Same as BPF_ABS */
70
+#define BPF_PROBE_MEM	0x20
71
+
6472 /* unused opcode to mark call to interpreter with arguments */
6573 #define BPF_CALL_ARGS	0xe0
6674 
....@@ -81,14 +89,6 @@
8189 /* Helper macros for filter block array initializers. */
8290 
8391 /* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */
84
-
85
-#define BPF_ALU_REG(CLASS, OP, DST, SRC)			\
86
-	((struct bpf_insn) {					\
87
-		.code  = CLASS | BPF_OP(OP) | BPF_X,		\
88
-		.dst_reg = DST,					\
89
-		.src_reg = SRC,					\
90
-		.off   = 0,					\
91
-		.imm   = 0 })
9292 
9393 #define BPF_ALU64_REG(OP, DST, SRC)				\
9494 	((struct bpf_insn) {					\
....@@ -136,14 +136,6 @@
136136 
137137 /* Short form of mov, dst_reg = src_reg */
138138 
139
-#define BPF_MOV_REG(CLASS, DST, SRC)				\
140
-	((struct bpf_insn) {					\
141
-		.code  = CLASS | BPF_MOV | BPF_X,		\
142
-		.dst_reg = DST,					\
143
-		.src_reg = SRC,					\
144
-		.off   = 0,					\
145
-		.imm   = 0 })
146
-
147139 #define BPF_MOV64_REG(DST, SRC)					\
148140 	((struct bpf_insn) {					\
149141 		.code  = BPF_ALU64 | BPF_MOV | BPF_X,		\
....@@ -178,13 +170,19 @@
178170 		.off   = 0,					\
179171 		.imm   = IMM })
180172 
181
-#define BPF_RAW_REG(insn, DST, SRC)				\
173
+/* Special form of mov32, used for doing explicit zero extension on dst. */
174
+#define BPF_ZEXT_REG(DST)					\
182175 	((struct bpf_insn) {					\
183
-		.code  = (insn).code,				\
176
+		.code  = BPF_ALU | BPF_MOV | BPF_X,		\
184177 		.dst_reg = DST,					\
185
-		.src_reg = SRC,					\
186
-		.off   = (insn).off,				\
187
-		.imm   = (insn).imm })
178
+		.src_reg = DST,					\
179
+		.off   = 0,					\
180
+		.imm   = 1 })
181
+
182
+static inline bool insn_is_zext(const struct bpf_insn *insn)
183
+{
184
+	return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1;
185
+}
188186 
189187 /* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */
190188 #define BPF_LD_IMM64(DST, IMM)					\
....@@ -306,6 +304,26 @@
306304 		.off   = OFF,					\
307305 		.imm   = IMM })
308306 
307
+/* Like BPF_JMP_REG, but with 32-bit wide operands for comparison. */
308
+
309
+#define BPF_JMP32_REG(OP, DST, SRC, OFF)			\
310
+	((struct bpf_insn) {					\
311
+		.code  = BPF_JMP32 | BPF_OP(OP) | BPF_X,	\
312
+		.dst_reg = DST,					\
313
+		.src_reg = SRC,					\
314
+		.off   = OFF,					\
315
+		.imm   = 0 })
316
+
317
+/* Like BPF_JMP_IMM, but with 32-bit wide operands for comparison. */
318
+
319
+#define BPF_JMP32_IMM(OP, DST, IMM, OFF)			\
320
+	((struct bpf_insn) {					\
321
+		.code  = BPF_JMP32 | BPF_OP(OP) | BPF_K,	\
322
+		.dst_reg = DST,					\
323
+		.src_reg = 0,					\
324
+		.off   = OFF,					\
325
+		.imm   = IMM })
326
+
309327 /* Unconditional jumps, goto pc + off16 */
310328 
311329 #define BPF_JMP_A(OFF)						\
....@@ -418,7 +436,7 @@
418436 
419437 #define BPF_FIELD_SIZEOF(type, field)				\
420438 	({							\
421
-		const int __size = bytes_to_bpf_size(FIELD_SIZEOF(type, field)); \
439
+		const int __size = bytes_to_bpf_size(sizeof_field(type, field)); \
422440 		BUILD_BUG_ON(__size < 0);			\
423441 		__size;						\
424442 	})
....@@ -465,10 +483,11 @@
465483 #define BPF_CALL_x(x, name, ...)					       \
466484 	static __always_inline						       \
467485 	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__));   \
486
+	typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \
468487 	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__));	       \
469488 	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__))	       \
470489 	{								       \
471
-		return ____##name(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\
490
+		return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\
472491 	}								       \
473492 	static __always_inline						       \
474493 	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__))
....@@ -484,50 +503,53 @@
484503 	offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1
485504 #define bpf_ctx_range_till(TYPE, MEMBER1, MEMBER2)				\
486505 	offsetof(TYPE, MEMBER1) ... offsetofend(TYPE, MEMBER2) - 1
506
+#if BITS_PER_LONG == 64
507
+# define bpf_ctx_range_ptr(TYPE, MEMBER)					\
508
+	offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1
509
+#else
510
+# define bpf_ctx_range_ptr(TYPE, MEMBER)					\
511
+	offsetof(TYPE, MEMBER) ... offsetof(TYPE, MEMBER) + 8 - 1
512
+#endif /* BITS_PER_LONG == 64 */
487513 
488514 #define bpf_target_off(TYPE, MEMBER, SIZE, PTR_SIZE)				\
489515 	({									\
490
-		BUILD_BUG_ON(FIELD_SIZEOF(TYPE, MEMBER) != (SIZE));		\
516
+		BUILD_BUG_ON(sizeof_field(TYPE, MEMBER) != (SIZE));		\
491517 		*(PTR_SIZE) = (SIZE);						\
492518 		offsetof(TYPE, MEMBER);						\
493519 	})
494520 
495
-#ifdef CONFIG_COMPAT
496521 /* A struct sock_filter is architecture independent. */
497522 struct compat_sock_fprog {
498523 	u16		len;
499524 	compat_uptr_t	filter;	/* struct sock_filter * */
500525 };
501
-#endif
502526 
503527 struct sock_fprog_kern {
504528 	u16			len;
505529 	struct sock_filter	*filter;
506530 };
507531 
508
-#define BPF_BINARY_HEADER_MAGIC	0x05de0e82
532
+/* Some arches need doubleword alignment for their instructions and/or data */
533
+#define BPF_IMAGE_ALIGNMENT 8
509534 
510535 struct bpf_binary_header {
511
-#ifdef CONFIG_CFI_CLANG
512
-	u32 magic;
513
-#endif
514536 	u32 pages;
515
-	/* Some arches need word alignment for their instructions */
516
-	u8 image[] __aligned(4);
537
+	u8 image[] __aligned(BPF_IMAGE_ALIGNMENT);
517538 };
518539 
519540 struct bpf_prog {
520541 	u16			pages;		/* Number of allocated pages */
521542 	u16			jited:1,	/* Is our filter JIT'ed? */
522543 				jit_requested:1,/* archs need to JIT the prog */
523
-				undo_set_mem:1,	/* Passed set_memory_ro() checkpoint */
524544 				gpl_compatible:1, /* Is filter GPL compatible? */
525545 				cb_access:1,	/* Is control block accessed? */
526546 				dst_needed:1,	/* Do we need dst entry? */
527547 				blinded:1,	/* Was blinded */
528548 				is_func:1,	/* program is a bpf function */
529549 				kprobe_override:1, /* Do we override a kprobe? */
530
-				has_callchain_buf:1; /* callchain buffer allocated? */
550
+				has_callchain_buf:1, /* callchain buffer allocated? */
551
+				enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
552
+				call_get_stack:1; /* Do we call bpf_get_stack() or bpf_get_stackid() */
531553 	enum bpf_prog_type	type;		/* Type of BPF program */
532554 	enum bpf_attach_type	expected_attach_type; /* For some prog types */
533555 	u32			len;		/* Number of filter blocks */
....@@ -538,10 +560,8 @@
538560 	unsigned int		(*bpf_func)(const void *ctx,
539561 					    const struct bpf_insn *insn);
540562 	/* Instructions for interpreter */
541
-	union {
542
-		struct sock_filter	insns[0];
543
-		struct bpf_insn		insnsi[0];
544
-	};
563
+	struct sock_filter	insns[0];
564
+	struct bpf_insn		insnsi[];
545565 };
546566 
547567 struct sk_filter {
....@@ -550,62 +570,49 @@
550570 	struct bpf_prog	*prog;
551571 };
552572 
553
-#if IS_ENABLED(CONFIG_BPF_JIT) && IS_ENABLED(CONFIG_CFI_CLANG)
573
+DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
574
+
575
+#define __BPF_PROG_RUN(prog, ctx, dfunc)	({			\
576
+	u32 __ret;							\
577
+	cant_migrate();							\
578
+	if (static_branch_unlikely(&bpf_stats_enabled_key)) {		\
579
+		struct bpf_prog_stats *__stats;				\
580
+		u64 __start = sched_clock();				\
581
+		__ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func);	\
582
+		__stats = this_cpu_ptr(prog->aux->stats);		\
583
+		u64_stats_update_begin(&__stats->syncp);		\
584
+		__stats->cnt++;						\
585
+		__stats->nsecs += sched_clock() - __start;		\
586
+		u64_stats_update_end(&__stats->syncp);			\
587
+	} else {							\
588
+		__ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func);	\
589
+	}								\
590
+	__ret; })
591
+
592
+#define BPF_PROG_RUN(prog, ctx)						\
593
+	__BPF_PROG_RUN(prog, ctx, bpf_dispatcher_nop_func)
594
+
554595 /*
555
- * With JIT, the kernel makes an indirect call to dynamically generated
556
- * code. Use bpf_call_func to perform additional validation of the call
557
- * target to narrow down attack surface. Architectures implementing BPF
558
- * JIT can override arch_bpf_jit_check_func for arch-specific checking.
596
+ * Use in preemptible and therefore migratable context to make sure that
597
+ * the execution of the BPF program runs on one CPU.
598
+ *
599
+ * This uses migrate_disable/enable() explicitly to document that the
600
+ * invocation of a BPF program does not require reentrancy protection
601
+ * against a BPF program which is invoked from a preempting task.
602
+ *
603
+ * For non RT enabled kernels migrate_disable/enable() maps to
604
+ * preempt_disable/enable(), i.e. it disables also preemption.
559605  */
560
-extern bool arch_bpf_jit_check_func(const struct bpf_prog *prog);
561
-
562
-static inline unsigned int __bpf_call_func(const struct bpf_prog *prog,
563
-					   const void *ctx)
606
+static inline u32 bpf_prog_run_pin_on_cpu(const struct bpf_prog *prog,
607
+					  const void *ctx)
564608 {
565
-	/* Call interpreter with CFI checking. */
566
-	return prog->bpf_func(ctx, prog->insnsi);
609
+	u32 ret;
610
+
611
+	migrate_disable();
612
+	ret = __BPF_PROG_RUN(prog, ctx, bpf_dispatcher_nop_func);
613
+	migrate_enable();
614
+	return ret;
567615 }
568
-
569
-static inline struct bpf_binary_header *
570
-bpf_jit_binary_hdr(const struct bpf_prog *fp);
571
-
572
-static inline unsigned int __nocfi bpf_call_func(const struct bpf_prog *prog,
573
-						 const void *ctx)
574
-{
575
-	const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog);
576
-
577
-	if (!IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) && !prog->jited)
578
-		return __bpf_call_func(prog, ctx);
579
-
580
-	/*
581
-	 * We are about to call dynamically generated code. Check that the
582
-	 * page has bpf_binary_header with a valid magic to limit possible
583
-	 * call targets.
584
-	 */
585
-	BUG_ON(hdr->magic != BPF_BINARY_HEADER_MAGIC ||
586
-		!arch_bpf_jit_check_func(prog));
587
-
588
-	/* Call jited function without CFI checking. */
589
-	return prog->bpf_func(ctx, prog->insnsi);
590
-}
591
-
592
-static inline void bpf_jit_set_header_magic(struct bpf_binary_header *hdr)
593
-{
594
-	hdr->magic = BPF_BINARY_HEADER_MAGIC;
595
-}
596
-#else
597
-static inline unsigned int bpf_call_func(const struct bpf_prog *prog,
598
-					 const void *ctx)
599
-{
600
-	return prog->bpf_func(ctx, prog->insnsi);
601
-}
602
-
603
-static inline void bpf_jit_set_header_magic(struct bpf_binary_header *hdr)
604
-{
605
-}
606
-#endif
607
-
608
-#define BPF_PROG_RUN(filter, ctx)  bpf_call_func(filter, ctx)
609616 
610617 #define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
611618 
....@@ -615,30 +622,21 @@
615622 	void *data_end;
616623 };
617624 
618
-struct sk_msg_buff {
619
-	void *data;
620
-	void *data_end;
621
-	__u32 apply_bytes;
622
-	__u32 cork_bytes;
623
-	int sg_copybreak;
624
-	int sg_start;
625
-	int sg_curr;
626
-	int sg_end;
627
-	struct scatterlist sg_data[MAX_SKB_FRAGS];
628
-	bool sg_copy[MAX_SKB_FRAGS];
629
-	__u32 flags;
630
-	struct sock *sk_redir;
631
-	struct sock *sk;
632
-	struct sk_buff *skb;
633
-	struct list_head list;
625
+struct bpf_nh_params {
626
+	u32 nh_family;
627
+	union {
628
+		u32 ipv4_nh;
629
+		struct in6_addr ipv6_nh;
630
+	};
634631 };
635632 
636633 struct bpf_redirect_info {
637
-	u32 ifindex;
638634 	u32 flags;
635
+	u32 tgt_index;
636
+	void *tgt_value;
639637 	struct bpf_map *map;
640
-	struct bpf_map *map_to_flush;
641638 	u32 kern_flags;
639
+	struct bpf_nh_params nh;
642640 };
643641 
644642 DECLARE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);
....@@ -656,9 +654,30 @@
656654 {
657655 	struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;
658656 
659
-	BUILD_BUG_ON(sizeof(*cb) > FIELD_SIZEOF(struct sk_buff, cb));
657
+	BUILD_BUG_ON(sizeof(*cb) > sizeof_field(struct sk_buff, cb));
660658 	cb->data_meta = skb->data - skb_metadata_len(skb);
661659 	cb->data_end  = skb->data + skb_headlen(skb);
660
+}
661
+
662
+/* Similar to bpf_compute_data_pointers(), except that save orginal
663
+ * data in cb->data and cb->meta_data for restore.
664
+ */
665
+static inline void bpf_compute_and_save_data_end(
666
+	struct sk_buff *skb, void **saved_data_end)
667
+{
668
+	struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;
669
+
670
+	*saved_data_end = cb->data_end;
671
+	cb->data_end  = skb->data + skb_headlen(skb);
672
+}
673
+
674
+/* Restore data saved by bpf_compute_data_pointers(). */
675
+static inline void bpf_restore_data_end(
676
+	struct sk_buff *skb, void *saved_data_end)
677
+{
678
+	struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;
679
+
680
+	cb->data_end = saved_data_end;
662681 }
663682 
664683 static inline u8 *bpf_skb_cb(struct sk_buff *skb)
....@@ -673,15 +692,16 @@
673692 	 * attached to sockets, we need to clear the bpf_skb_cb() area
674693 	 * to not leak previous contents to user space.
675694 	 */
676
-	BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) != BPF_SKB_CB_LEN);
677
-	BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) !=
678
-		     FIELD_SIZEOF(struct qdisc_skb_cb, data));
695
+	BUILD_BUG_ON(sizeof_field(struct __sk_buff, cb) != BPF_SKB_CB_LEN);
696
+	BUILD_BUG_ON(sizeof_field(struct __sk_buff, cb) !=
697
+		     sizeof_field(struct qdisc_skb_cb, data));
679698 
680699 	return qdisc_skb_cb(skb)->data;
681700 }
682701 
683
-static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog,
684
-				       struct sk_buff *skb)
702
+/* Must be invoked with migration disabled */
703
+static inline u32 __bpf_prog_run_save_cb(const struct bpf_prog *prog,
704
+					 struct sk_buff *skb)
685705 {
686706 	u8 *cb_data = bpf_skb_cb(skb);
687707 	u8 cb_saved[BPF_SKB_CB_LEN];
....@@ -700,16 +720,31 @@
700720 	return res;
701721 }
702722 
723
+static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog,
724
+				       struct sk_buff *skb)
725
+{
726
+	u32 res;
727
+
728
+	migrate_disable();
729
+	res = __bpf_prog_run_save_cb(prog, skb);
730
+	migrate_enable();
731
+	return res;
732
+}
733
+
703734 static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog,
704735 					struct sk_buff *skb)
705736 {
706737 	u8 *cb_data = bpf_skb_cb(skb);
738
+	u32 res;
707739 
708740 	if (unlikely(prog->cb_access))
709741 		memset(cb_data, 0, BPF_SKB_CB_LEN);
710742 
711
-	return BPF_PROG_RUN(prog, skb);
743
+	res = bpf_prog_run_pin_on_cpu(prog, skb);
744
+	return res;
712745 }
746
+
747
+DECLARE_BPF_DISPATCHER(xdp)
713748 
714749 static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
715750 					    struct xdp_buff *xdp)
....@@ -720,8 +755,10 @@
720755 	 * already takes rcu_read_lock() when fetching the program, so
721756 	 * it's not necessary here anymore.
722757 	 */
723
-	return BPF_PROG_RUN(prog, xdp);
758
+	return __BPF_PROG_RUN(prog, xdp, BPF_DISPATCHER_FUNC(xdp));
724759 }
760
+
761
+void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog);
725762 
726763 static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)
727764 {
....@@ -731,7 +768,7 @@
731768 static inline u32 bpf_prog_tag_scratch_size(const struct bpf_prog *prog)
732769 {
733770 	return round_up(bpf_prog_insn_size(prog) +
734
-			sizeof(__be64) + 1, SHA_MESSAGE_BYTES);
771
+			sizeof(__be64) + 1, SHA1_BLOCK_SIZE);
735772 }
736773 
737774 static inline unsigned int bpf_prog_size(unsigned int proglen)
....@@ -766,29 +803,41 @@
766803 	return size <= size_default && (size & (size - 1)) == 0;
767804 }
768805 
806
+static inline u8
807
+bpf_ctx_narrow_access_offset(u32 off, u32 size, u32 size_default)
808
+{
809
+	u8 access_off = off & (size_default - 1);
810
+
811
+#ifdef __LITTLE_ENDIAN
812
+	return access_off;
813
+#else
814
+	return size_default - (access_off + size);
815
+#endif
816
+}
817
+
818
+#define bpf_ctx_wide_access_ok(off, size, type, field)			\
819
+	(size == sizeof(__u64) &&					\
820
+	off >= offsetof(type, field) &&					\
821
+	off + sizeof(__u64) <= offsetofend(type, field) &&		\
822
+	off % sizeof(__u64) == 0)
823
+
769824 #define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0]))
770825 
771826 static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
772827 {
773
-	fp->undo_set_mem = 1;
774
-	set_memory_ro((unsigned long)fp, fp->pages);
775
-}
776
-
777
-static inline void bpf_prog_unlock_ro(struct bpf_prog *fp)
778
-{
779
-	if (fp->undo_set_mem)
780
-		set_memory_rw((unsigned long)fp, fp->pages);
828
+#ifndef CONFIG_BPF_JIT_ALWAYS_ON
829
+	if (!fp->jited) {
830
+		set_vm_flush_reset_perms(fp);
831
+		set_memory_ro((unsigned long)fp, fp->pages);
832
+	}
833
+#endif
781834 }
782835 
783836 static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
784837 {
838
+	set_vm_flush_reset_perms(hdr);
785839 	set_memory_ro((unsigned long)hdr, hdr->pages);
786840 	set_memory_x((unsigned long)hdr, hdr->pages);
787
-}
788
-
789
-static inline void bpf_jit_binary_unlock_ro(struct bpf_binary_header *hdr)
790
-{
791
-	set_memory_rw((unsigned long)hdr, hdr->pages);
792841 }
793842 
794843 static inline struct bpf_binary_header *
....@@ -800,16 +849,7 @@
800849 	return (void *)addr;
801850 }
802851 
803
-#ifdef CONFIG_FILTER
804852 int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);
805
-#else
806
-static inline
807
-int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap)
808
-{
809
-	return 0;
810
-}
811
-#endif
812
-
813853 static inline int sk_filter(struct sock *sk, struct sk_buff *skb)
814854 {
815855 	return sk_filter_trim_cap(sk, skb, 1);
....@@ -820,21 +860,27 @@
820860 
821861 bool bpf_opcode_in_insntable(u8 code);
822862 
863
+void bpf_prog_free_linfo(struct bpf_prog *prog);
864
+void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
865
+			       const u32 *insn_to_jit_off);
866
+int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog);
867
+void bpf_prog_free_jited_linfo(struct bpf_prog *prog);
868
+void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);
869
+
823870 struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
871
+struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags);
824872 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
825873 				  gfp_t gfp_extra_flags);
826874 void __bpf_prog_free(struct bpf_prog *fp);
827875 
828876 static inline void bpf_prog_unlock_free(struct bpf_prog *fp)
829877 {
830
-	bpf_prog_unlock_ro(fp);
831878 	__bpf_prog_free(fp);
832879 }
833880 
834881 typedef int (*bpf_aux_classic_check_t)(struct sock_filter *filter,
835882 				       unsigned int flen);
836883 
837
-#ifdef CONFIG_FILTER
838884 int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog);
839885 int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
840886 			      bpf_aux_classic_check_t trans, bool save_orig);
....@@ -852,71 +898,6 @@
852898 bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
853899 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
854900 
855
-#else
856
-static inline
857
-int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog)
858
-{
859
-	return 0;
860
-}
861
-
862
-static inline
863
-int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
864
-			      bpf_aux_classic_check_t trans, bool save_orig)
865
-{
866
-	return 0;
867
-}
868
-
869
-static inline void bpf_prog_destroy(struct bpf_prog *fp)
870
-{
871
-}
872
-
873
-static inline int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
874
-{
875
-	return 0;
876
-}
877
-
878
-static inline int sk_attach_bpf(u32 ufd, struct sock *sk)
879
-{
880
-	return 0;
881
-}
882
-
883
-static inline
884
-int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk)
885
-{
886
-	return 0;
887
-}
888
-
889
-static inline int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk)
890
-{
891
-	return 0;
892
-}
893
-
894
-static inline void sk_reuseport_prog_free(struct bpf_prog *prog)
895
-{
896
-}
897
-
898
-static inline int sk_detach_filter(struct sock *sk)
899
-{
900
-	return 0;
901
-}
902
-
903
-static inline
904
-int sk_get_filter(struct sock *sk, struct sock_filter __user *filter,
905
-		  unsigned int len)
906
-{
907
-	return 0;
908
-}
909
-
910
-static inline bool sk_filter_charge(struct sock *sk, struct sk_filter *fp)
911
-{
912
-	return false;
913
-}
914
-
915
-static inline void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
916
-{
917
-}
918
-#endif
919
-
920901 u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
921902 #define __bpf_call_base_args \
922903 	((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
....@@ -924,6 +905,7 @@
924905 
925906 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
926907 void bpf_jit_compile(struct bpf_prog *prog);
908
+bool bpf_jit_needs_zext(void);
927909 bool bpf_helper_changes_pkt_data(void *func);
928910 
929911 static inline bool bpf_dump_raw_ok(const struct cred *cred)
....@@ -936,6 +918,7 @@
936918 
937919 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
938920 				       const struct bpf_insn *patch, u32 len);
921
+int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);
939922 
940923 void bpf_clear_redirect_map(struct bpf_map *map);
941924 
....@@ -975,8 +958,7 @@
975958 	return 0;
976959 }
977960 
978
-#ifdef CONFIG_FILTER
979
-/* The pair of xdp_do_redirect and xdp_do_flush_map MUST be called in the
961
+/* The pair of xdp_do_redirect and xdp_do_flush MUST be called in the
980962  * same cpu context. Further for best results no more than a single map
981963  * for the do_redirect/do_flush pair should be used. This limitation is
982964  * because we only track one map and force a flush when the map changes.
....@@ -987,38 +969,17 @@
987969 int xdp_do_redirect(struct net_device *dev,
988970 		    struct xdp_buff *xdp,
989971 		    struct bpf_prog *prog);
990
-void xdp_do_flush_map(void);
972
+void xdp_do_flush(void);
973
+
974
+/* The xdp_do_flush_map() helper has been renamed to drop the _map suffix, as
975
+ * it is no longer only flushing maps. Keep this define for compatibility
976
+ * until all drivers are updated - do not use xdp_do_flush_map() in new code!
977
+ */
978
+#define xdp_do_flush_map xdp_do_flush
991979 
992980 void bpf_warn_invalid_xdp_action(u32 act);
993
-#else
994
-static inline
995
-int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
996
-			    struct xdp_buff *xdp, struct bpf_prog *prog)
997
-{
998
-	return 0;
999
-}
1000981 
1001
-static inline
1002
-int xdp_do_redirect(struct net_device *dev,
1003
-		    struct xdp_buff *xdp,
1004
-		    struct bpf_prog *prog)
1005
-{
1006
-	return 0;
1007
-}
1008
-
1009
-static inline void xdp_do_flush_map(void)
1010
-{
1011
-}
1012
-
1013
-static inline void bpf_warn_invalid_xdp_action(u32 act)
1014
-{
1015
-}
1016
-#endif
1017
-
1018
-struct sock *do_sk_redirect_map(struct sk_buff *skb);
1019
-struct sock *do_msg_redirect_map(struct sk_msg_buff *md);
1020
-
1021
-#if (IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_FILTER))
982
+#ifdef CONFIG_INET
1022983 struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
1023984 				  struct bpf_prog *prog, struct sk_buff *skb,
1024985 				  u32 hash);
....@@ -1050,6 +1011,13 @@
10501011 void *bpf_jit_alloc_exec(unsigned long size);
10511012 void bpf_jit_free_exec(void *addr);
10521013 void bpf_jit_free(struct bpf_prog *fp);
1014
+
1015
+int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,
1016
+				struct bpf_jit_poke_descriptor *poke);
1017
+
1018
+int bpf_jit_get_func_addr(const struct bpf_prog *prog,
1019
+			  const struct bpf_insn *insn, bool extra_pass,
1020
+			  u64 *func_addr, bool *func_addr_fixed);
10531021 
10541022 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
10551023 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);
....@@ -1144,9 +1112,21 @@
11441112 	return false;
11451113 }
11461114 
1115
+static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog)
1116
+{
1117
+	return false;
1118
+}
1119
+
11471120 static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp)
11481121 {
11491122 	return false;
1123
+}
1124
+
1125
+static inline int
1126
+bpf_jit_add_poke_descriptor(struct bpf_prog *prog,
1127
+			    struct bpf_jit_poke_descriptor *poke)
1128
+{
1129
+	return -ENOTSUPP;
11501130 }
11511131 
11521132 static inline void bpf_jit_free(struct bpf_prog *fp)
....@@ -1191,9 +1171,9 @@
11911171 static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp)
11921172 {
11931173 }
1174
+
11941175 #endif /* CONFIG_BPF_JIT */
11951176 
1196
-void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp);
11971177 void bpf_prog_kallsyms_del_all(struct bpf_prog *fp);
11981178 
11991179 #define BPF_ANC		BIT(15)
....@@ -1245,7 +1225,7 @@
12451225 		BPF_ANCILLARY(RANDOM);
12461226 		BPF_ANCILLARY(VLAN_TPID);
12471227 		}
1248
-		/* Fallthrough. */
1228
+		fallthrough;
12491229 	default:
12501230 		return ftest->code;
12511231 	}
....@@ -1281,13 +1261,17 @@
12811261 
12821262 struct bpf_sock_ops_kern {
12831263 	struct	sock *sk;
1284
-	u32	op;
12851264 	union {
12861265 		u32 args[4];
12871266 		u32 reply;
12881267 		u32 replylong[4];
12891268 	};
1290
-	u32	is_fullsock;
1269
+	struct sk_buff	*syn_skb;
1270
+	struct sk_buff	*skb;
1271
+	void	*skb_data_end;
1272
+	u8	op;
1273
+	u8	is_fullsock;
1274
+	u8	remaining_opt_len;
12911275 	u64	temp;			/* temp and everything after is not
12921276 					 * initialized to 0 before calling
12931277 					 * the BPF program. New fields that
....@@ -1299,4 +1283,177 @@
12991283 					 */
13001284 };
13011285 
1286
+struct bpf_sysctl_kern {
1287
+	struct ctl_table_header *head;
1288
+	struct ctl_table *table;
1289
+	void *cur_val;
1290
+	size_t cur_len;
1291
+	void *new_val;
1292
+	size_t new_len;
1293
+	int new_updated;
1294
+	int write;
1295
+	loff_t *ppos;
1296
+	/* Temporary "register" for indirect stores to ppos. */
1297
+	u64 tmp_reg;
1298
+};
1299
+
1300
+struct bpf_sockopt_kern {
1301
+	struct sock	*sk;
1302
+	u8		*optval;
1303
+	u8		*optval_end;
1304
+	s32		level;
1305
+	s32		optname;
1306
+	s32		optlen;
1307
+	s32		retval;
1308
+};
1309
+
1310
+int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len);
1311
+
1312
+struct bpf_sk_lookup_kern {
1313
+	u16		family;
1314
+	u16		protocol;
1315
+	__be16		sport;
1316
+	u16		dport;
1317
+	struct {
1318
+		__be32 saddr;
1319
+		__be32 daddr;
1320
+	} v4;
1321
+	struct {
1322
+		const struct in6_addr *saddr;
1323
+		const struct in6_addr *daddr;
1324
+	} v6;
1325
+	struct sock	*selected_sk;
1326
+	bool		no_reuseport;
1327
+};
1328
+
1329
+extern struct static_key_false bpf_sk_lookup_enabled;
1330
+
1331
+/* Runners for BPF_SK_LOOKUP programs to invoke on socket lookup.
1332
+ *
1333
+ * Allowed return values for a BPF SK_LOOKUP program are SK_PASS and
1334
+ * SK_DROP. Their meaning is as follows:
1335
+ *
1336
+ *  SK_PASS && ctx.selected_sk != NULL: use selected_sk as lookup result
1337
+ *  SK_PASS && ctx.selected_sk == NULL: continue to htable-based socket lookup
1338
+ *  SK_DROP                           : terminate lookup with -ECONNREFUSED
1339
+ *
1340
+ * This macro aggregates return values and selected sockets from
1341
+ * multiple BPF programs according to following rules in order:
1342
+ *
1343
+ *  1. If any program returned SK_PASS and a non-NULL ctx.selected_sk,
1344
+ *     macro result is SK_PASS and last ctx.selected_sk is used.
1345
+ *  2. If any program returned SK_DROP return value,
1346
+ *     macro result is SK_DROP.
1347
+ *  3. Otherwise result is SK_PASS and ctx.selected_sk is NULL.
1348
+ *
1349
+ * Caller must ensure that the prog array is non-NULL, and that the
1350
+ * array as well as the programs it contains remain valid.
1351
+ */
1352
+#define BPF_PROG_SK_LOOKUP_RUN_ARRAY(array, ctx, func)			\
1353
+	({								\
1354
+		struct bpf_sk_lookup_kern *_ctx = &(ctx);		\
1355
+		struct bpf_prog_array_item *_item;			\
1356
+		struct sock *_selected_sk = NULL;			\
1357
+		bool _no_reuseport = false;				\
1358
+		struct bpf_prog *_prog;					\
1359
+		bool _all_pass = true;					\
1360
+		u32 _ret;						\
1361
+									\
1362
+		migrate_disable();					\
1363
+		_item = &(array)->items[0];				\
1364
+		while ((_prog = READ_ONCE(_item->prog))) {		\
1365
+			/* restore most recent selection */		\
1366
+			_ctx->selected_sk = _selected_sk;		\
1367
+			_ctx->no_reuseport = _no_reuseport;		\
1368
+									\
1369
+			_ret = func(_prog, _ctx);			\
1370
+			if (_ret == SK_PASS && _ctx->selected_sk) {	\
1371
+				/* remember last non-NULL socket */	\
1372
+				_selected_sk = _ctx->selected_sk;	\
1373
+				_no_reuseport = _ctx->no_reuseport;	\
1374
+			} else if (_ret == SK_DROP && _all_pass) {	\
1375
+				_all_pass = false;			\
1376
+			}						\
1377
+			_item++;					\
1378
+		}							\
1379
+		_ctx->selected_sk = _selected_sk;			\
1380
+		_ctx->no_reuseport = _no_reuseport;			\
1381
+		migrate_enable();					\
1382
+		_all_pass || _selected_sk ? SK_PASS : SK_DROP;		\
1383
+	 })
1384
+
1385
+static inline bool bpf_sk_lookup_run_v4(struct net *net, int protocol,
1386
+					const __be32 saddr, const __be16 sport,
1387
+					const __be32 daddr, const u16 dport,
1388
+					struct sock **psk)
1389
+{
1390
+	struct bpf_prog_array *run_array;
1391
+	struct sock *selected_sk = NULL;
1392
+	bool no_reuseport = false;
1393
+
1394
+	rcu_read_lock();
1395
+	run_array = rcu_dereference(net->bpf.run_array[NETNS_BPF_SK_LOOKUP]);
1396
+	if (run_array) {
1397
+		struct bpf_sk_lookup_kern ctx = {
1398
+			.family		= AF_INET,
1399
+			.protocol	= protocol,
1400
+			.v4.saddr	= saddr,
1401
+			.v4.daddr	= daddr,
1402
+			.sport		= sport,
1403
+			.dport		= dport,
1404
+		};
1405
+		u32 act;
1406
+
1407
+		act = BPF_PROG_SK_LOOKUP_RUN_ARRAY(run_array, ctx, BPF_PROG_RUN);
1408
+		if (act == SK_PASS) {
1409
+			selected_sk = ctx.selected_sk;
1410
+			no_reuseport = ctx.no_reuseport;
1411
+		} else {
1412
+			selected_sk = ERR_PTR(-ECONNREFUSED);
1413
+		}
1414
+	}
1415
+	rcu_read_unlock();
1416
+	*psk = selected_sk;
1417
+	return no_reuseport;
1418
+}
1419
+
1420
+#if IS_ENABLED(CONFIG_IPV6)
1421
+static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol,
1422
+					const struct in6_addr *saddr,
1423
+					const __be16 sport,
1424
+					const struct in6_addr *daddr,
1425
+					const u16 dport,
1426
+					struct sock **psk)
1427
+{
1428
+	struct bpf_prog_array *run_array;
1429
+	struct sock *selected_sk = NULL;
1430
+	bool no_reuseport = false;
1431
+
1432
+	rcu_read_lock();
1433
+	run_array = rcu_dereference(net->bpf.run_array[NETNS_BPF_SK_LOOKUP]);
1434
+	if (run_array) {
1435
+		struct bpf_sk_lookup_kern ctx = {
1436
+			.family		= AF_INET6,
1437
+			.protocol	= protocol,
1438
+			.v6.saddr	= saddr,
1439
+			.v6.daddr	= daddr,
1440
+			.sport		= sport,
1441
+			.dport		= dport,
1442
+		};
1443
+		u32 act;
1444
+
1445
+		act = BPF_PROG_SK_LOOKUP_RUN_ARRAY(run_array, ctx, BPF_PROG_RUN);
1446
+		if (act == SK_PASS) {
1447
+			selected_sk = ctx.selected_sk;
1448
+			no_reuseport = ctx.no_reuseport;
1449
+		} else {
1450
+			selected_sk = ERR_PTR(-ECONNREFUSED);
1451
+		}
1452
+	}
1453
+	rcu_read_unlock();
1454
+	*psk = selected_sk;
1455
+	return no_reuseport;
1456
+}
1457
+#endif /* IS_ENABLED(CONFIG_IPV6) */
1458
+
13021459 #endif /* __LINUX_FILTER_H__ */