~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* INET An implementation of the TCP/IP protocol suite for the LINUX
3	4	* operating system. INET is implemented using the BSD Socket
..	..	@@ -69,19 +70,13 @@
69	70	* a single port at the same time.
70	71	* Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
71	72	* James Chapman : Add L2TP encapsulation type.
72		- *
73		- *
74		- * This program is free software; you can redistribute it and/or
75		- * modify it under the terms of the GNU General Public License
76		- * as published by the Free Software Foundation; either version
77		- * 2 of the License, or (at your option) any later version.
78	73	*/
79	74
80	75	#define pr_fmt(fmt) "UDP: " fmt
81	76
82	77	#include <linux/uaccess.h>
83	78	#include <asm/ioctls.h>
84		-#include <linux/bootmem.h>
	79	+#include <linux/memblock.h>
85	80	#include <linux/highmem.h>
86	81	#include <linux/swap.h>
87	82	#include <linux/types.h>
..	..	@@ -105,16 +100,23 @@
105	100	#include <net/net_namespace.h>
106	101	#include <net/icmp.h>
107	102	#include <net/inet_hashtables.h>
	103	+#include <net/ip_tunnels.h>
108	104	#include <net/route.h>
109	105	#include <net/checksum.h>
110	106	#include <net/xfrm.h>
111	107	#include <trace/events/udp.h>
112	108	#include <linux/static_key.h>
	109	+#include <linux/btf_ids.h>
113	110	#include <trace/events/skb.h>
114	111	#include <net/busy_poll.h>
115	112	#include "udp_impl.h"
116	113	#include <net/sock_reuseport.h>
117	114	#include <net/addrconf.h>
	115	+#include <net/udp_tunnel.h>
	116	+#if IS_ENABLED(CONFIG_IPV6)
	117	+#include <net/ipv6_stubs.h>
	118	+#endif
	119	+#include <trace/hooks/ipv4.h>
118	120
119	121	struct udp_table udp_table __read_mostly;
120	122	EXPORT_SYMBOL(udp_table);
..	..	@@ -127,17 +129,6 @@
127	129
128	130	#define MAX_UDP_PORTS 65536
129	131	#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
130		-
131		-/* IPCB reference means this can not be used from early demux */
132		-static bool udp_lib_exact_dif_match(struct net net, struct sk_buff skb)
133		-{
134		-#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
135		- if (!net->ipv4.sysctl_udp_l3mdev_accept &&
136		- skb && ipv4_l3mdev_skb(IPCB(skb)->flags))
137		- return true;
138		-#endif
139		- return false;
140		-}
141	132
142	133	static int udp_lib_lport_inuse(struct net *net, __u16 num,
143	134	const struct udp_hslot *hslot,
..	..	@@ -367,25 +358,23 @@
367	358	static int compute_score(struct sock sk, struct net net,
368	359	__be32 saddr, __be16 sport,
369	360	__be32 daddr, unsigned short hnum,
370		- int dif, int sdif, bool exact_dif)
	361	+ int dif, int sdif)
371	362	{
372	363	int score;
373	364	struct inet_sock *inet;
	365	+ bool dev_match;
374	366
375	367	if (!net_eq(sock_net(sk), net) \|\|
376	368	udp_sk(sk)->udp_port_hash != hnum \|\|
377	369	ipv6_only_sock(sk))
378	370	return -1;
379	371
	372	+ if (sk->sk_rcv_saddr != daddr)
	373	+ return -1;
	374	+
380	375	score = (sk->sk_family == PF_INET) ? 2 : 1;
	376	+
381	377	inet = inet_sk(sk);
382		-
383		- if (inet->inet_rcv_saddr) {
384		- if (inet->inet_rcv_saddr != daddr)
385		- return -1;
386		- score += 4;
387		- }
388		-
389	378	if (inet->inet_daddr) {
390	379	if (inet->inet_daddr != saddr)
391	380	return -1;
..	..	@@ -398,15 +387,12 @@
398	387	score += 4;
399	388	}
400	389
401		- if (sk->sk_bound_dev_if \|\| exact_dif) {
402		- bool dev_match = (sk->sk_bound_dev_if == dif \|\|
403		- sk->sk_bound_dev_if == sdif);
404		-
405		- if (!dev_match)
406		- return -1;
407		- if (sk->sk_bound_dev_if)
408		- score += 4;
409		- }
	390	+ dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
	391	+ dif, sdif);
	392	+ if (!dev_match)
	393	+ return -1;
	394	+ if (sk->sk_bound_dev_if)
	395	+ score += 4;
410	396
411	397	if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
412	398	score++;
..	..	@@ -425,41 +411,73 @@
425	411	udp_ehash_secret + net_hash_mix(net));
426	412	}
427	413
	414	+static struct sock lookup_reuseport(struct net net, struct sock *sk,
	415	+ struct sk_buff *skb,
	416	+ __be32 saddr, __be16 sport,
	417	+ __be32 daddr, unsigned short hnum)
	418	+{
	419	+ struct sock *reuse_sk = NULL;
	420	+ u32 hash;
	421	+
	422	+ if (sk->sk_reuseport && sk->sk_state != TCP_ESTABLISHED) {
	423	+ hash = udp_ehashfn(net, daddr, hnum, saddr, sport);
	424	+ reuse_sk = reuseport_select_sock(sk, hash, skb,
	425	+ sizeof(struct udphdr));
	426	+ }
	427	+ return reuse_sk;
	428	+}
	429	+
428	430	/* called with rcu_read_lock() */
429	431	static struct sock udp4_lib_lookup2(struct net net,
430	432	__be32 saddr, __be16 sport,
431	433	__be32 daddr, unsigned int hnum,
432		- int dif, int sdif, bool exact_dif,
	434	+ int dif, int sdif,
433	435	struct udp_hslot *hslot2,
434	436	struct sk_buff *skb)
435	437	{
436		- struct sock sk, result, *reuseport_result;
	438	+ struct sock sk, result;
437	439	int score, badness;
438		- u32 hash = 0;
439	440
440	441	result = NULL;
441	442	badness = 0;
442	443	udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
443	444	score = compute_score(sk, net, saddr, sport,
444		- daddr, hnum, dif, sdif, exact_dif);
	445	+ daddr, hnum, dif, sdif);
445	446	if (score > badness) {
446		- reuseport_result = NULL;
	447	+ result = lookup_reuseport(net, sk, skb,
	448	+ saddr, sport, daddr, hnum);
	449	+ /* Fall back to scoring if group has connections */
	450	+ if (result && !reuseport_has_conns(sk))
	451	+ return result;
447	452
448		- if (sk->sk_reuseport &&
449		- sk->sk_state != TCP_ESTABLISHED) {
450		- hash = udp_ehashfn(net, daddr, hnum,
451		- saddr, sport);
452		- reuseport_result = reuseport_select_sock(sk, hash, skb,
453		- sizeof(struct udphdr));
454		- if (reuseport_result && !reuseport_has_conns(sk, false))
455		- return reuseport_result;
456		- }
457		-
458		- result = reuseport_result ? : sk;
	453	+ result = result ? : sk;
459	454	badness = score;
460	455	}
461	456	}
462	457	return result;
	458	+}
	459	+
	460	+static struct sock udp4_lookup_run_bpf(struct net net,
	461	+ struct udp_table *udptable,
	462	+ struct sk_buff *skb,
	463	+ __be32 saddr, __be16 sport,
	464	+ __be32 daddr, u16 hnum)
	465	+{
	466	+ struct sock sk, reuse_sk;
	467	+ bool no_reuseport;
	468	+
	469	+ if (udptable != &udp_table)
	470	+ return NULL; /* only UDP is supported */
	471	+
	472	+ no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_UDP,
	473	+ saddr, sport, daddr, hnum, &sk);
	474	+ if (no_reuseport \|\| IS_ERR_OR_NULL(sk))
	475	+ return sk;
	476	+
	477	+ reuse_sk = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum);
	478	+ if (reuse_sk)
	479	+ sk = reuse_sk;
	480	+ return sk;
463	481	}
464	482
465	483	/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
..	..	@@ -469,65 +487,47 @@
469	487	__be16 sport, __be32 daddr, __be16 dport, int dif,
470	488	int sdif, struct udp_table udptable, struct sk_buff skb)
471	489	{
472		- struct sock sk, result;
473	490	unsigned short hnum = ntohs(dport);
474		- unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
475		- struct udp_hslot hslot2, hslot = &udptable->hash[slot];
476		- bool exact_dif = udp_lib_exact_dif_match(net, skb);
477		- int score, badness;
478		- u32 hash = 0;
	491	+ unsigned int hash2, slot2;
	492	+ struct udp_hslot *hslot2;
	493	+ struct sock result, sk;
479	494
480		- if (hslot->count > 10) {
481		- hash2 = ipv4_portaddr_hash(net, daddr, hnum);
482		- slot2 = hash2 & udptable->mask;
483		- hslot2 = &udptable->hash2[slot2];
484		- if (hslot->count < hslot2->count)
485		- goto begin;
	495	+ hash2 = ipv4_portaddr_hash(net, daddr, hnum);
	496	+ slot2 = hash2 & udptable->mask;
	497	+ hslot2 = &udptable->hash2[slot2];
486	498
487		- result = udp4_lib_lookup2(net, saddr, sport,
488		- daddr, hnum, dif, sdif,
489		- exact_dif, hslot2, skb);
490		- if (!result) {
491		- unsigned int old_slot2 = slot2;
492		- hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
493		- slot2 = hash2 & udptable->mask;
494		- /* avoid searching the same slot again. */
495		- if (unlikely(slot2 == old_slot2))
496		- return result;
	499	+ /* Lookup connected or non-wildcard socket */
	500	+ result = udp4_lib_lookup2(net, saddr, sport,
	501	+ daddr, hnum, dif, sdif,
	502	+ hslot2, skb);
	503	+ if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
	504	+ goto done;
497	505
498		- hslot2 = &udptable->hash2[slot2];
499		- if (hslot->count < hslot2->count)
500		- goto begin;
501		-
502		- result = udp4_lib_lookup2(net, saddr, sport,
503		- daddr, hnum, dif, sdif,
504		- exact_dif, hslot2, skb);
505		- }
506		- if (unlikely(IS_ERR(result)))
507		- return NULL;
508		- return result;
509		- }
510		-begin:
511		- result = NULL;
512		- badness = 0;
513		- sk_for_each_rcu(sk, &hslot->head) {
514		- score = compute_score(sk, net, saddr, sport,
515		- daddr, hnum, dif, sdif, exact_dif);
516		- if (score > badness) {
517		- if (sk->sk_reuseport) {
518		- hash = udp_ehashfn(net, daddr, hnum,
519		- saddr, sport);
520		- result = reuseport_select_sock(sk, hash, skb,
521		- sizeof(struct udphdr));
522		- if (unlikely(IS_ERR(result)))
523		- return NULL;
524		- if (result)
525		- return result;
526		- }
	506	+ /* Lookup redirect from BPF */
	507	+ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
	508	+ sk = udp4_lookup_run_bpf(net, udptable, skb,
	509	+ saddr, sport, daddr, hnum);
	510	+ if (sk) {
527	511	result = sk;
528		- badness = score;
	512	+ goto done;
529	513	}
530	514	}
	515	+
	516	+ /* Got non-wildcard socket or error on first lookup */
	517	+ if (result)
	518	+ goto done;
	519	+
	520	+ /* Lookup wildcard sockets */
	521	+ hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
	522	+ slot2 = hash2 & udptable->mask;
	523	+ hslot2 = &udptable->hash2[slot2];
	524	+
	525	+ result = udp4_lib_lookup2(net, saddr, sport,
	526	+ htonl(INADDR_ANY), hnum, dif, sdif,
	527	+ hslot2, skb);
	528	+done:
	529	+ if (IS_ERR(result))
	530	+ return NULL;
531	531	return result;
532	532	}
533	533	EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
..	..	@@ -585,12 +585,102 @@
585	585	(inet->inet_dport != rmt_port && inet->inet_dport) \|\|
586	586	(inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) \|\|
587	587	ipv6_only_sock(sk) \|\|
588		- (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif &&
589		- sk->sk_bound_dev_if != sdif))
	588	+ !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
590	589	return false;
591	590	if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
592	591	return false;
593	592	return true;
	593	+}
	594	+
	595	+DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
	596	+void udp_encap_enable(void)
	597	+{
	598	+ static_branch_inc(&udp_encap_needed_key);
	599	+}
	600	+EXPORT_SYMBOL(udp_encap_enable);
	601	+
	602	+void udp_encap_disable(void)
	603	+{
	604	+ static_branch_dec(&udp_encap_needed_key);
	605	+}
	606	+EXPORT_SYMBOL(udp_encap_disable);
	607	+
	608	+/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
	609	+ * through error handlers in encapsulations looking for a match.
	610	+ */
	611	+static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
	612	+{
	613	+ int i;
	614	+
	615	+ for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
	616	+ int (handler)(struct sk_buff skb, u32 info);
	617	+ const struct ip_tunnel_encap_ops *encap;
	618	+
	619	+ encap = rcu_dereference(iptun_encaps[i]);
	620	+ if (!encap)
	621	+ continue;
	622	+ handler = encap->err_handler;
	623	+ if (handler && !handler(skb, info))
	624	+ return 0;
	625	+ }
	626	+
	627	+ return -ENOENT;
	628	+}
	629	+
	630	+/* Try to match ICMP errors to UDP tunnels by looking up a socket without
	631	+ * reversing source and destination port: this will match tunnels that force the
	632	+ * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
	633	+ * lwtunnels might actually break this assumption by being configured with
	634	+ * different destination ports on endpoints, in this case we won't be able to
	635	+ * trace ICMP messages back to them.
	636	+ *
	637	+ * If this doesn't match any socket, probe tunnels with arbitrary destination
	638	+ * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
	639	+ * we've sent packets to won't necessarily match the local destination port.
	640	+ *
	641	+ * Then ask the tunnel implementation to match the error against a valid
	642	+ * association.
	643	+ *
	644	+ * Return an error if we can't find a match, the socket if we need further
	645	+ * processing, zero otherwise.
	646	+ */
	647	+static struct sock __udp4_lib_err_encap(struct net net,
	648	+ const struct iphdr *iph,
	649	+ struct udphdr *uh,
	650	+ struct udp_table *udptable,
	651	+ struct sk_buff *skb, u32 info)
	652	+{
	653	+ int network_offset, transport_offset;
	654	+ struct sock *sk;
	655	+
	656	+ network_offset = skb_network_offset(skb);
	657	+ transport_offset = skb_transport_offset(skb);
	658	+
	659	+ /* Network header needs to point to the outer IPv4 header inside ICMP */
	660	+ skb_reset_network_header(skb);
	661	+
	662	+ /* Transport header needs to point to the UDP header */
	663	+ skb_set_transport_header(skb, iph->ihl << 2);
	664	+
	665	+ sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
	666	+ iph->saddr, uh->dest, skb->dev->ifindex, 0,
	667	+ udptable, NULL);
	668	+ if (sk) {
	669	+ int (lookup)(struct sock sk, struct sk_buff *skb);
	670	+ struct udp_sock *up = udp_sk(sk);
	671	+
	672	+ lookup = READ_ONCE(up->encap_err_lookup);
	673	+ if (!lookup \|\| lookup(sk, skb))
	674	+ sk = NULL;
	675	+ }
	676	+
	677	+ if (!sk)
	678	+ sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
	679	+
	680	+ skb_set_transport_header(skb, transport_offset);
	681	+ skb_set_network_header(skb, network_offset);
	682	+
	683	+ return sk;
594	684	}
595	685
596	686	/*
..	..	@@ -604,24 +694,38 @@
604	694	* to find the appropriate port.
605	695	*/
606	696
607		-void __udp4_lib_err(struct sk_buff skb, u32 info, struct udp_table udptable)
	697	+int __udp4_lib_err(struct sk_buff skb, u32 info, struct udp_table udptable)
608	698	{
609	699	struct inet_sock *inet;
610	700	const struct iphdr iph = (const struct iphdr )skb->data;
611	701	struct udphdr uh = (struct udphdr )(skb->data+(iph->ihl<<2));
612	702	const int type = icmp_hdr(skb)->type;
613	703	const int code = icmp_hdr(skb)->code;
	704	+ bool tunnel = false;
614	705	struct sock *sk;
615	706	int harderr;
616	707	int err;
617	708	struct net *net = dev_net(skb->dev);
618	709
619	710	sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
620		- iph->saddr, uh->source, skb->dev->ifindex, 0,
621		- udptable, NULL);
	711	+ iph->saddr, uh->source, skb->dev->ifindex,
	712	+ inet_sdif(skb), udptable, NULL);
622	713	if (!sk) {
623		- __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
624		- return; /* No socket for error */
	714	+ /* No socket for error: try tunnels before discarding */
	715	+ sk = ERR_PTR(-ENOENT);
	716	+ if (static_branch_unlikely(&udp_encap_needed_key)) {
	717	+ sk = __udp4_lib_err_encap(net, iph, uh, udptable, skb,
	718	+ info);
	719	+ if (!sk)
	720	+ return 0;
	721	+ }
	722	+
	723	+ if (IS_ERR(sk)) {
	724	+ __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
	725	+ return PTR_ERR(sk);
	726	+ }
	727	+
	728	+ tunnel = true;
625	729	}
626	730
627	731	err = 0;
..	..	@@ -664,6 +768,10 @@
664	768	* RFC1122: OK. Passes ICMP errors back to application, as per
665	769	* 4.1.3.3.
666	770	*/
	771	+ if (tunnel) {
	772	+ /* ...not for tunnels though: we don't have a sending socket */
	773	+ goto out;
	774	+ }
667	775	if (!inet->recverr) {
668	776	if (!harderr \|\| sk->sk_state != TCP_ESTABLISHED)
669	777	goto out;
..	..	@@ -673,12 +781,12 @@
673	781	sk->sk_err = err;
674	782	sk->sk_error_report(sk);
675	783	out:
676		- return;
	784	+ return 0;
677	785	}
678	786
679		-void udp_err(struct sk_buff *skb, u32 info)
	787	+int udp_err(struct sk_buff *skb, u32 info)
680	788	{
681		- __udp4_lib_err(skb, info, &udp_table);
	789	+ return __udp4_lib_err(skb, info, &udp_table);
682	790	}
683	791
684	792	/*
..	..	@@ -949,6 +1057,7 @@
949	1057
950	1058	if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
951	1059	return -EOPNOTSUPP;
	1060	+ trace_android_rvh_udp_sendmsg(sk);
952	1061
953	1062	getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
954	1063
..	..	@@ -1061,7 +1170,7 @@
1061	1170	}
1062	1171
1063	1172	if (ipv4_is_multicast(daddr)) {
1064		- if (!ipc.oif)
	1173	+ if (!ipc.oif \|\| netif_index_is_l3_master(sock_net(sk), ipc.oif))
1065	1174	ipc.oif = inet->mc_index;
1066	1175	if (!saddr)
1067	1176	saddr = inet->mc_addr;
..	..	@@ -1070,7 +1179,7 @@
1070	1179	ipc.oif = inet->uc_index;
1071	1180	} else if (ipv4_is_lbcast(daddr) && inet->uc_index) {
1072	1181	/* oif is set, packet is to local broadcast and
1073		- * and uc_index is set. oif is most likely set
	1182	+ * uc_index is set. oif is most likely set
1074	1183	* by sk_bound_dev_if. If uc_index != oif check if the
1075	1184	* oif is an L3 master and uc_index is an L3 slave.
1076	1185	* If so, we want to allow the send using the uc_index.
..	..	@@ -1091,13 +1200,13 @@
1091	1200
1092	1201	fl4 = &fl4_stack;
1093	1202
1094		- flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos,
	1203	+ flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos,
1095	1204	RT_SCOPE_UNIVERSE, sk->sk_protocol,
1096	1205	flow_flags,
1097	1206	faddr, saddr, dport, inet->inet_sport,
1098	1207	sk->sk_uid);
1099	1208
1100		- security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
	1209	+ security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
1101	1210	rt = ip_route_output_flow(net, fl4, sk);
1102	1211	if (IS_ERR(rt)) {
1103	1212	err = PTR_ERR(rt);
..	..	@@ -1254,6 +1363,27 @@
1254	1363
1255	1364	#define UDP_SKB_IS_STATELESS 0x80000000
1256	1365
	1366	+/* all head states (dst, sk, nf conntrack) except skb extensions are
	1367	+ * cleared by udp_rcv().
	1368	+ *
	1369	+ * We need to preserve secpath, if present, to eventually process
	1370	+ * IP_CMSG_PASSSEC at recvmsg() time.
	1371	+ *
	1372	+ * Other extensions can be cleared.
	1373	+ */
	1374	+static bool udp_try_make_stateless(struct sk_buff *skb)
	1375	+{
	1376	+ if (!skb_has_extensions(skb))
	1377	+ return true;
	1378	+
	1379	+ if (!secpath_exists(skb)) {
	1380	+ skb_ext_reset(skb);
	1381	+ return true;
	1382	+ }
	1383	+
	1384	+ return false;
	1385	+}
	1386	+
1257	1387	static void udp_set_dev_scratch(struct sk_buff *skb)
1258	1388	{
1259	1389	struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
..	..	@@ -1265,11 +1395,7 @@
1265	1395	scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1266	1396	scratch->is_linear = !skb_is_nonlinear(skb);
1267	1397	#endif
1268		- /* all head states execept sp (dst, sk, nf) are always cleared by
1269		- * udp_rcv() and we need to preserve secpath, if present, to eventually
1270		- * process IP_CMSG_PASSSEC at recvmsg() time
1271		- */
1272		- if (likely(!skb_sec_path(skb)))
	1398	+ if (udp_try_make_stateless(skb))
1273	1399	scratch->_tsize_state \|= UDP_SKB_IS_STATELESS;
1274	1400	}
1275	1401
..	..	@@ -1590,7 +1716,7 @@
1590	1716	EXPORT_SYMBOL(udp_ioctl);
1591	1717
1592	1718	struct sk_buff __skb_recv_udp(struct sock sk, unsigned int flags,
1593		- int noblock, int peeked, int off, int *err)
	1719	+ int noblock, int off, int err)
1594	1720	{
1595	1721	struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1596	1722	struct sk_buff_head *queue;
..	..	@@ -1609,14 +1735,13 @@
1609	1735	break;
1610	1736
1611	1737	error = -EAGAIN;
1612		- *peeked = 0;
1613	1738	do {
1614	1739	spin_lock_bh(&queue->lock);
1615		- skb = __skb_try_recv_from_queue(sk, queue, flags,
1616		- udp_skb_destructor,
1617		- peeked, off, err,
1618		- &last);
	1740	+ skb = __skb_try_recv_from_queue(sk, queue, flags, off,
	1741	+ err, &last);
1619	1742	if (skb) {
	1743	+ if (!(flags & MSG_PEEK))
	1744	+ udp_skb_destructor(sk, skb);
1620	1745	spin_unlock_bh(&queue->lock);
1621	1746	return skb;
1622	1747	}
..	..	@@ -1634,10 +1759,10 @@
1634	1759	spin_lock(&sk_queue->lock);
1635	1760	skb_queue_splice_tail_init(sk_queue, queue);
1636	1761
1637		- skb = __skb_try_recv_from_queue(sk, queue, flags,
1638		- udp_skb_dtor_locked,
1639		- peeked, off, err,
1640		- &last);
	1762	+ skb = __skb_try_recv_from_queue(sk, queue, flags, off,
	1763	+ err, &last);
	1764	+ if (skb && !(flags & MSG_PEEK))
	1765	+ udp_skb_dtor_locked(sk, skb);
1641	1766	spin_unlock(&sk_queue->lock);
1642	1767	spin_unlock_bh(&queue->lock);
1643	1768	if (skb)
..	..	@@ -1652,7 +1777,8 @@
1652	1777
1653	1778	/* sk_queue is empty, reader_queue may contain peeked packets */
1654	1779	} while (timeo &&
1655		- !__skb_wait_for_more_packets(sk, &error, &timeo,
	1780	+ !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue,
	1781	+ &error, &timeo,
1656	1782	(struct sk_buff *)sk_queue));
1657	1783
1658	1784	*err = error;
..	..	@@ -1672,8 +1798,7 @@
1672	1798	DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
1673	1799	struct sk_buff *skb;
1674	1800	unsigned int ulen, copied;
1675		- int peeked, peeking, off;
1676		- int err;
	1801	+ int off, err, peeking = flags & MSG_PEEK;
1677	1802	int is_udplite = IS_UDPLITE(sk);
1678	1803	bool checksum_valid = false;
1679	1804
..	..	@@ -1681,11 +1806,11 @@
1681	1806	return ip_recv_error(sk, msg, len, addr_len);
1682	1807
1683	1808	try_again:
1684		- peeking = flags & MSG_PEEK;
1685	1809	off = sk_peek_offset(sk, flags);
1686		- skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err);
	1810	+ skb = __skb_recv_udp(sk, flags, noblock, &off, &err);
1687	1811	if (!skb)
1688	1812	return err;
	1813	+ trace_android_rvh_udp_recvmsg(sk);
1689	1814
1690	1815	ulen = udp_skb_len(skb);
1691	1816	copied = len;
..	..	@@ -1721,7 +1846,7 @@
1721	1846	}
1722	1847
1723	1848	if (unlikely(err)) {
1724		- if (!peeked) {
	1849	+ if (!peeking) {
1725	1850	atomic_inc(&sk->sk_drops);
1726	1851	UDP_INC_STATS(sock_net(sk),
1727	1852	UDP_MIB_INERRORS, is_udplite);
..	..	@@ -1730,7 +1855,7 @@
1730	1855	return err;
1731	1856	}
1732	1857
1733		- if (!peeked)
	1858	+ if (!peeking)
1734	1859	UDP_INC_STATS(sock_net(sk),
1735	1860	UDP_MIB_INDATAGRAMS, is_udplite);
1736	1861
..	..	@@ -1748,6 +1873,10 @@
1748	1873	BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
1749	1874	(struct sockaddr *)sin);
1750	1875	}
	1876	+
	1877	+ if (udp_sk(sk)->gro_enabled)
	1878	+ udp_cmsg_recv(msg, sk, skb);
	1879	+
1751	1880	if (inet->cmsg_flags)
1752	1881	ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
1753	1882
..	..	@@ -1797,8 +1926,12 @@
1797	1926	inet->inet_dport = 0;
1798	1927	sock_rps_reset_rxhash(sk);
1799	1928	sk->sk_bound_dev_if = 0;
1800		- if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
	1929	+ if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) {
1801	1930	inet_reset_saddr(sk);
	1931	+ if (sk->sk_prot->rehash &&
	1932	+ (sk->sk_userlocks & SOCK_BINDPORT_LOCK))
	1933	+ sk->sk_prot->rehash(sk);
	1934	+ }
1802	1935
1803	1936	if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1804	1937	sk->sk_prot->unhash(sk);
..	..	@@ -1887,7 +2020,7 @@
1887	2020	}
1888	2021	EXPORT_SYMBOL(udp_lib_rehash);
1889	2022
1890		-static void udp_v4_rehash(struct sock *sk)
	2023	+void udp_v4_rehash(struct sock *sk)
1891	2024	{
1892	2025	u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
1893	2026	inet_sk(sk)->inet_rcv_saddr,
..	..	@@ -1924,13 +2057,6 @@
1924	2057	return 0;
1925	2058	}
1926	2059
1927		-static DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
1928		-void udp_encap_enable(void)
1929		-{
1930		- static_branch_enable(&udp_encap_needed_key);
1931		-}
1932		-EXPORT_SYMBOL(udp_encap_enable);
1933		-
1934	2060	/* returns:
1935	2061	* -1: error
1936	2062	* 0: success
..	..	@@ -1939,7 +2065,7 @@
1939	2065	* Note that in the success and error cases, the skb is assumed to
1940	2066	* have either been requeued or freed.
1941	2067	*/
1942		-static int udp_queue_rcv_skb(struct sock sk, struct sk_buff skb)
	2068	+static int udp_queue_rcv_one_skb(struct sock sk, struct sk_buff skb)
1943	2069	{
1944	2070	struct udp_sock *up = udp_sk(sk);
1945	2071	int is_udplite = IS_UDPLITE(sk);
..	..	@@ -1949,7 +2075,7 @@
1949	2075	*/
1950	2076	if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1951	2077	goto drop;
1952		- nf_reset(skb);
	2078	+ nf_reset_ct(skb);
1953	2079
1954	2080	if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
1955	2081	int (encap_rcv)(struct sock sk, struct sk_buff *skb);
..	..	@@ -2042,6 +2168,26 @@
2042	2168	return -1;
2043	2169	}
2044	2170
	2171	+static int udp_queue_rcv_skb(struct sock sk, struct sk_buff skb)
	2172	+{
	2173	+ struct sk_buff next, segs;
	2174	+ int ret;
	2175	+
	2176	+ if (likely(!udp_unexpected_gso(sk, skb)))
	2177	+ return udp_queue_rcv_one_skb(sk, skb);
	2178	+
	2179	+ BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
	2180	+ __skb_push(skb, -skb_mac_offset(skb));
	2181	+ segs = udp_rcv_segment(sk, skb, true);
	2182	+ skb_list_walk_safe(segs, skb, next) {
	2183	+ __skb_pull(skb, skb_transport_offset(skb));
	2184	+ ret = udp_queue_rcv_one_skb(sk, skb);
	2185	+ if (ret > 0)
	2186	+ ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret);
	2187	+ }
	2188	+ return 0;
	2189	+}
	2190	+
2045	2191	/* For TCP sockets, sk_rx_dst is protected by socket lock
2046	2192	* For UDP, we use xchg() to guard against concurrent changes.
2047	2193	*/
..	..	@@ -2050,7 +2196,7 @@
2050	2196	struct dst_entry *old;
2051	2197
2052	2198	if (dst_hold_safe(dst)) {
2053		- old = xchg(&sk->sk_rx_dst, dst);
	2199	+ old = xchg((__force struct dst_entry **)&sk->sk_rx_dst, dst);
2054	2200	dst_release(old);
2055	2201	return old != dst;
2056	2202	}
..	..	@@ -2130,7 +2276,7 @@
2130	2276
2131	2277	/* Initialize UDP checksum. If exited with zero value (success),
2132	2278	* CHECKSUM_UNNECESSARY means, that no more checks are required.
2133		- * Otherwise, csum completion requires chacksumming packet body,
	2279	+ * Otherwise, csum completion requires checksumming packet body,
2134	2280	* including udp header and folding it to skb->csum.
2135	2281	*/
2136	2282	static inline int udp4_csum_init(struct sk_buff skb, struct udphdr uh,
..	..	@@ -2184,8 +2330,7 @@
2184	2330	int ret;
2185	2331
2186	2332	if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
2187		- skb_checksum_try_convert(skb, IPPROTO_UDP, uh->check,
2188		- inet_compute_pseudo);
	2333	+ skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
2189	2334
2190	2335	ret = udp_queue_rcv_skb(sk, skb);
2191	2336
..	..	@@ -2210,6 +2355,7 @@
2210	2355	struct rtable *rt = skb_rtable(skb);
2211	2356	__be32 saddr, daddr;
2212	2357	struct net *net = dev_net(skb->dev);
	2358	+ bool refcounted;
2213	2359
2214	2360	/*
2215	2361	* Validate the packet.
..	..	@@ -2235,16 +2381,17 @@
2235	2381	if (udp4_csum_init(skb, uh, proto))
2236	2382	goto csum_error;
2237	2383
2238		- sk = skb_steal_sock(skb);
	2384	+ sk = skb_steal_sock(skb, &refcounted);
2239	2385	if (sk) {
2240	2386	struct dst_entry *dst = skb_dst(skb);
2241	2387	int ret;
2242	2388
2243		- if (unlikely(sk->sk_rx_dst != dst))
	2389	+ if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
2244	2390	udp_sk_rx_dst_set(sk, dst);
2245	2391
2246	2392	ret = udp_unicast_rcv_skb(sk, skb, uh);
2247		- sock_put(sk);
	2393	+ if (refcounted)
	2394	+ sock_put(sk);
2248	2395	return ret;
2249	2396	}
2250	2397
..	..	@@ -2258,7 +2405,7 @@
2258	2405
2259	2406	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2260	2407	goto drop;
2261		- nf_reset(skb);
	2408	+ nf_reset_ct(skb);
2262	2409
2263	2410	/* No socket. Drop packet silently, if checksum is wrong */
2264	2411	if (udp_lib_checksum_complete(skb))
..	..	@@ -2346,8 +2493,7 @@
2346	2493	struct sock *sk;
2347	2494
2348	2495	udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
2349		- if (INET_MATCH(sk, net, acookie, rmt_addr,
2350		- loc_addr, ports, dif, sdif))
	2496	+ if (INET_MATCH(net, sk, acookie, ports, dif, sdif))
2351	2497	return sk;
2352	2498	/* Only check first socket in chain */
2353	2499	break;
..	..	@@ -2398,7 +2544,7 @@
2398	2544
2399	2545	skb->sk = sk;
2400	2546	skb->destructor = sock_efree;
2401		- dst = READ_ONCE(sk->sk_rx_dst);
	2547	+ dst = rcu_dereference(sk->sk_rx_dst);
2402	2548
2403	2549	if (dst)
2404	2550	dst = dst_check(dst, 0);
..	..	@@ -2437,11 +2583,15 @@
2437	2583	sock_set_flag(sk, SOCK_DEAD);
2438	2584	udp_flush_pending_frames(sk);
2439	2585	unlock_sock_fast(sk, slow);
2440		- if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
2441		- void (encap_destroy)(struct sock sk);
2442		- encap_destroy = READ_ONCE(up->encap_destroy);
2443		- if (encap_destroy)
2444		- encap_destroy(sk);
	2586	+ if (static_branch_unlikely(&udp_encap_needed_key)) {
	2587	+ if (up->encap_type) {
	2588	+ void (encap_destroy)(struct sock sk);
	2589	+ encap_destroy = READ_ONCE(up->encap_destroy);
	2590	+ if (encap_destroy)
	2591	+ encap_destroy(sk);
	2592	+ }
	2593	+ if (up->encap_enabled)
	2594	+ static_branch_dec(&udp_encap_needed_key);
2445	2595	}
2446	2596	}
2447	2597
..	..	@@ -2449,7 +2599,7 @@
2449	2599	* Socket option code for UDP
2450	2600	*/
2451	2601	int udp_lib_setsockopt(struct sock *sk, int level, int optname,
2452		- char __user *optval, unsigned int optlen,
	2602	+ sockptr_t optval, unsigned int optlen,
2453	2603	int (push_pending_frames)(struct sock ))
2454	2604	{
2455	2605	struct udp_sock *up = udp_sk(sk);
..	..	@@ -2460,7 +2610,7 @@
2460	2610	if (optlen < sizeof(int))
2461	2611	return -EINVAL;
2462	2612
2463		- if (get_user(val, (int __user *)optval))
	2613	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
2464	2614	return -EFAULT;
2465	2615
2466	2616	valbool = val ? 1 : 0;
..	..	@@ -2480,13 +2630,22 @@
2480	2630	case UDP_ENCAP:
2481	2631	switch (val) {
2482	2632	case 0:
	2633	+#ifdef CONFIG_XFRM
2483	2634	case UDP_ENCAP_ESPINUDP:
2484	2635	case UDP_ENCAP_ESPINUDP_NON_IKE:
2485		- up->encap_rcv = xfrm4_udp_encap_rcv;
2486		- /* FALLTHROUGH */
	2636	+#if IS_ENABLED(CONFIG_IPV6)
	2637	+ if (sk->sk_family == AF_INET6)
	2638	+ up->encap_rcv = ipv6_stub->xfrm6_udp_encap_rcv;
	2639	+ else
	2640	+#endif
	2641	+ up->encap_rcv = xfrm4_udp_encap_rcv;
	2642	+#endif
	2643	+ fallthrough;
2487	2644	case UDP_ENCAP_L2TPINUDP:
2488	2645	up->encap_type = val;
2489		- udp_encap_enable();
	2646	+ lock_sock(sk);
	2647	+ udp_tunnel_encap_enable(sk->sk_socket);
	2648	+ release_sock(sk);
2490	2649	break;
2491	2650	default:
2492	2651	err = -ENOPROTOOPT;
..	..	@@ -2506,6 +2665,17 @@
2506	2665	if (val < 0 \|\| val > USHRT_MAX)
2507	2666	return -EINVAL;
2508	2667	WRITE_ONCE(up->gso_size, val);
	2668	+ break;
	2669	+
	2670	+ case UDP_GRO:
	2671	+ lock_sock(sk);
	2672	+
	2673	+ /* when enabling GRO, accept the related GSO packet type */
	2674	+ if (valbool)
	2675	+ udp_tunnel_encap_enable(sk->sk_socket);
	2676	+ up->gro_enabled = valbool;
	2677	+ up->accept_udp_l4 = valbool;
	2678	+ release_sock(sk);
2509	2679	break;
2510	2680
2511	2681	/*
..	..	@@ -2547,25 +2717,15 @@
2547	2717	}
2548	2718	EXPORT_SYMBOL(udp_lib_setsockopt);
2549	2719
2550		-int udp_setsockopt(struct sock *sk, int level, int optname,
2551		- char __user *optval, unsigned int optlen)
	2720	+int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
	2721	+ unsigned int optlen)
2552	2722	{
2553	2723	if (level == SOL_UDP \|\| level == SOL_UDPLITE)
2554		- return udp_lib_setsockopt(sk, level, optname, optval, optlen,
	2724	+ return udp_lib_setsockopt(sk, level, optname,
	2725	+ optval, optlen,
2555	2726	udp_push_pending_frames);
2556	2727	return ip_setsockopt(sk, level, optname, optval, optlen);
2557	2728	}
2558		-
2559		-#ifdef CONFIG_COMPAT
2560		-int compat_udp_setsockopt(struct sock *sk, int level, int optname,
2561		- char __user *optval, unsigned int optlen)
2562		-{
2563		- if (level == SOL_UDP \|\| level == SOL_UDPLITE)
2564		- return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2565		- udp_push_pending_frames);
2566		- return compat_ip_setsockopt(sk, level, optname, optval, optlen);
2567		-}
2568		-#endif
2569	2729
2570	2730	int udp_lib_getsockopt(struct sock *sk, int level, int optname,
2571	2731	char __user optval, int __user optlen)
..	..	@@ -2602,6 +2762,10 @@
2602	2762	val = READ_ONCE(up->gso_size);
2603	2763	break;
2604	2764
	2765	+ case UDP_GRO:
	2766	+ val = up->gro_enabled;
	2767	+ break;
	2768	+
2605	2769	/* The following two cannot be changed on UDP sockets, the return is
2606	2770	* always 0 (which corresponds to the full checksum coverage of UDP). */
2607	2771	case UDPLITE_SEND_CSCOV:
..	..	@@ -2632,20 +2796,11 @@
2632	2796	return ip_getsockopt(sk, level, optname, optval, optlen);
2633	2797	}
2634	2798
2635		-#ifdef CONFIG_COMPAT
2636		-int compat_udp_getsockopt(struct sock *sk, int level, int optname,
2637		- char __user optval, int __user optlen)
2638		-{
2639		- if (level == SOL_UDP \|\| level == SOL_UDPLITE)
2640		- return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2641		- return compat_ip_getsockopt(sk, level, optname, optval, optlen);
2642		-}
2643		-#endif
2644	2799	/**
2645	2800	* udp_poll - wait for a UDP event.
2646		- * @file - file struct
2647		- * @sock - socket
2648		- * @wait - poll table
	2801	+ * @file: - file struct
	2802	+ * @sock: - socket
	2803	+ * @wait: - poll table
2649	2804	*
2650	2805	* This is same as datagram poll, except for the special case of
2651	2806	* blocking sockets. If application is using a blocking fd
..	..	@@ -2719,10 +2874,6 @@
2719	2874	.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
2720	2875	.obj_size = sizeof(struct udp_sock),
2721	2876	.h.udp_table = &udp_table,
2722		-#ifdef CONFIG_COMPAT
2723		- .compat_setsockopt = compat_udp_setsockopt,
2724		- .compat_getsockopt = compat_udp_getsockopt,
2725		-#endif
2726	2877	.diag_destroy = udp_abort,
2727	2878	};
2728	2879	EXPORT_SYMBOL(udp_prot);
..	..	@@ -2733,9 +2884,14 @@
2733	2884	static struct sock udp_get_first(struct seq_file seq, int start)
2734	2885	{
2735	2886	struct sock *sk;
2736		- struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
	2887	+ struct udp_seq_afinfo *afinfo;
2737	2888	struct udp_iter_state *state = seq->private;
2738	2889	struct net *net = seq_file_net(seq);
	2890	+
	2891	+ if (state->bpf_seq_afinfo)
	2892	+ afinfo = state->bpf_seq_afinfo;
	2893	+ else
	2894	+ afinfo = PDE_DATA(file_inode(seq->file));
2739	2895
2740	2896	for (state->bucket = start; state->bucket <= afinfo->udp_table->mask;
2741	2897	++state->bucket) {
..	..	@@ -2748,7 +2904,8 @@
2748	2904	sk_for_each(sk, &hslot->head) {
2749	2905	if (!net_eq(sock_net(sk), net))
2750	2906	continue;
2751		- if (sk->sk_family == afinfo->family)
	2907	+ if (afinfo->family == AF_UNSPEC \|\|
	2908	+ sk->sk_family == afinfo->family)
2752	2909	goto found;
2753	2910	}
2754	2911	spin_unlock_bh(&hslot->lock);
..	..	@@ -2760,13 +2917,20 @@
2760	2917
2761	2918	static struct sock udp_get_next(struct seq_file seq, struct sock *sk)
2762	2919	{
2763		- struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
	2920	+ struct udp_seq_afinfo *afinfo;
2764	2921	struct udp_iter_state *state = seq->private;
2765	2922	struct net *net = seq_file_net(seq);
2766	2923
	2924	+ if (state->bpf_seq_afinfo)
	2925	+ afinfo = state->bpf_seq_afinfo;
	2926	+ else
	2927	+ afinfo = PDE_DATA(file_inode(seq->file));
	2928	+
2767	2929	do {
2768	2930	sk = sk_next(sk);
2769		- } while (sk && (!net_eq(sock_net(sk), net) \|\| sk->sk_family != afinfo->family));
	2931	+ } while (sk && (!net_eq(sock_net(sk), net) \|\|
	2932	+ (afinfo->family != AF_UNSPEC &&
	2933	+ sk->sk_family != afinfo->family)));
2770	2934
2771	2935	if (!sk) {
2772	2936	if (state->bucket <= afinfo->udp_table->mask)
..	..	@@ -2811,8 +2975,13 @@
2811	2975
2812	2976	void udp_seq_stop(struct seq_file seq, void v)
2813	2977	{
2814		- struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
	2978	+ struct udp_seq_afinfo *afinfo;
2815	2979	struct udp_iter_state *state = seq->private;
	2980	+
	2981	+ if (state->bpf_seq_afinfo)
	2982	+ afinfo = state->bpf_seq_afinfo;
	2983	+ else
	2984	+ afinfo = PDE_DATA(file_inode(seq->file));
2816	2985
2817	2986	if (state->bucket <= afinfo->udp_table->mask)
2818	2987	spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
..	..	@@ -2830,7 +2999,7 @@
2830	2999	__u16 srcp = ntohs(inet->inet_sport);
2831	3000
2832	3001	seq_printf(f, "%5d: %08X:%04X %08X:%04X"
2833		- " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d",
	3002	+ " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u",
2834	3003	bucket, src, srcp, dest, destp, sp->sk_state,
2835	3004	sk_wmem_alloc_get(sp),
2836	3005	udp_rqueue_get(sp),
..	..	@@ -2856,6 +3025,67 @@
2856	3025	seq_pad(seq, '\n');
2857	3026	return 0;
2858	3027	}
	3028	+
	3029	+#ifdef CONFIG_BPF_SYSCALL
	3030	+struct bpf_iter__udp {
	3031	+ __bpf_md_ptr(struct bpf_iter_meta *, meta);
	3032	+ __bpf_md_ptr(struct udp_sock *, udp_sk);
	3033	+ uid_t uid __aligned(8);
	3034	+ int bucket __aligned(8);
	3035	+};
	3036	+
	3037	+static int udp_prog_seq_show(struct bpf_prog prog, struct bpf_iter_meta meta,
	3038	+ struct udp_sock *udp_sk, uid_t uid, int bucket)
	3039	+{
	3040	+ struct bpf_iter__udp ctx;
	3041	+
	3042	+ meta->seq_num--; /* skip SEQ_START_TOKEN */
	3043	+ ctx.meta = meta;
	3044	+ ctx.udp_sk = udp_sk;
	3045	+ ctx.uid = uid;
	3046	+ ctx.bucket = bucket;
	3047	+ return bpf_iter_run_prog(prog, &ctx);
	3048	+}
	3049	+
	3050	+static int bpf_iter_udp_seq_show(struct seq_file seq, void v)
	3051	+{
	3052	+ struct udp_iter_state *state = seq->private;
	3053	+ struct bpf_iter_meta meta;
	3054	+ struct bpf_prog *prog;
	3055	+ struct sock *sk = v;
	3056	+ uid_t uid;
	3057	+
	3058	+ if (v == SEQ_START_TOKEN)
	3059	+ return 0;
	3060	+
	3061	+ uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
	3062	+ meta.seq = seq;
	3063	+ prog = bpf_iter_get_info(&meta, false);
	3064	+ return udp_prog_seq_show(prog, &meta, v, uid, state->bucket);
	3065	+}
	3066	+
	3067	+static void bpf_iter_udp_seq_stop(struct seq_file seq, void v)
	3068	+{
	3069	+ struct bpf_iter_meta meta;
	3070	+ struct bpf_prog *prog;
	3071	+
	3072	+ if (!v) {
	3073	+ meta.seq = seq;
	3074	+ prog = bpf_iter_get_info(&meta, true);
	3075	+ if (prog)
	3076	+ (void)udp_prog_seq_show(prog, &meta, v, 0, 0);
	3077	+ }
	3078	+
	3079	+ udp_seq_stop(seq, v);
	3080	+}
	3081	+
	3082	+static const struct seq_operations bpf_iter_udp_seq_ops = {
	3083	+ .start = udp_seq_start,
	3084	+ .next = udp_seq_next,
	3085	+ .stop = bpf_iter_udp_seq_stop,
	3086	+ .show = bpf_iter_udp_seq_show,
	3087	+};
	3088	+#endif
2859	3089
2860	3090	const struct seq_operations udp_seq_ops = {
2861	3091	.start = udp_seq_start,
..	..	@@ -2974,6 +3204,62 @@
2974	3204	.init = udp_sysctl_init,
2975	3205	};
2976	3206
	3207	+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
	3208	+DEFINE_BPF_ITER_FUNC(udp, struct bpf_iter_meta *meta,
	3209	+ struct udp_sock *udp_sk, uid_t uid, int bucket)
	3210	+
	3211	+static int bpf_iter_init_udp(void priv_data, struct bpf_iter_aux_info aux)
	3212	+{
	3213	+ struct udp_iter_state *st = priv_data;
	3214	+ struct udp_seq_afinfo *afinfo;
	3215	+ int ret;
	3216	+
	3217	+ afinfo = kmalloc(sizeof(*afinfo), GFP_USER \| __GFP_NOWARN);
	3218	+ if (!afinfo)
	3219	+ return -ENOMEM;
	3220	+
	3221	+ afinfo->family = AF_UNSPEC;
	3222	+ afinfo->udp_table = &udp_table;
	3223	+ st->bpf_seq_afinfo = afinfo;
	3224	+ ret = bpf_iter_init_seq_net(priv_data, aux);
	3225	+ if (ret)
	3226	+ kfree(afinfo);
	3227	+ return ret;
	3228	+}
	3229	+
	3230	+static void bpf_iter_fini_udp(void *priv_data)
	3231	+{
	3232	+ struct udp_iter_state *st = priv_data;
	3233	+
	3234	+ kfree(st->bpf_seq_afinfo);
	3235	+ bpf_iter_fini_seq_net(priv_data);
	3236	+}
	3237	+
	3238	+static const struct bpf_iter_seq_info udp_seq_info = {
	3239	+ .seq_ops = &bpf_iter_udp_seq_ops,
	3240	+ .init_seq_private = bpf_iter_init_udp,
	3241	+ .fini_seq_private = bpf_iter_fini_udp,
	3242	+ .seq_priv_size = sizeof(struct udp_iter_state),
	3243	+};
	3244	+
	3245	+static struct bpf_iter_reg udp_reg_info = {
	3246	+ .target = "udp",
	3247	+ .ctx_arg_info_size = 1,
	3248	+ .ctx_arg_info = {
	3249	+ { offsetof(struct bpf_iter__udp, udp_sk),
	3250	+ PTR_TO_BTF_ID_OR_NULL },
	3251	+ },
	3252	+ .seq_info = &udp_seq_info,
	3253	+};
	3254	+
	3255	+static void __init bpf_iter_register(void)
	3256	+{
	3257	+ udp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UDP];
	3258	+ if (bpf_iter_reg_target(&udp_reg_info))
	3259	+ pr_warn("Warning: could not register bpf iterator udp\n");
	3260	+}
	3261	+#endif
	3262	+
2977	3263	void __init udp_init(void)
2978	3264	{
2979	3265	unsigned long limit;
..	..	@@ -2999,4 +3285,8 @@
2999	3285
3000	3286	if (register_pernet_subsys(&udp_sysctl_ops))
3001	3287	panic("UDP: failed to init sysctl parameters.\n");
	3288	+
	3289	+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
	3290	+ bpf_iter_register();
	3291	+#endif
3002	3292	}