~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* INET An implementation of the TCP/IP protocol suite for the LINUX
3	4	* operating system. INET is implemented using the BSD Socket
..	..	@@ -7,18 +8,12 @@
7	8	*
8	9	* IPv4 specific functions
9	10	*
10		- *
11	11	* code split from:
12	12	* linux/ipv4/tcp.c
13	13	* linux/ipv4/tcp_input.c
14	14	* linux/ipv4/tcp_output.c
15	15	*
16	16	* See tcp.c for author information
17		- *
18		- * This program is free software; you can redistribute it and/or
19		- * modify it under the terms of the GNU General Public License
20		- * as published by the Free Software Foundation; either version
21		- * 2 of the License, or (at your option) any later version.
22	17	*/
23	18
24	19	/*
..	..	@@ -62,7 +57,6 @@
62	57	#include <linux/init.h>
63	58	#include <linux/times.h>
64	59	#include <linux/slab.h>
65		-#include <linux/locallock.h>
66	60
67	61	#include <net/net_namespace.h>
68	62	#include <net/icmp.h>
..	..	@@ -82,6 +76,7 @@
82	76	#include <linux/proc_fs.h>
83	77	#include <linux/seq_file.h>
84	78	#include <linux/inetdevice.h>
	79	+#include <linux/btf_ids.h>
85	80
86	81	#include <crypto/hash.h>
87	82	#include <linux/scatterlist.h>
..	..	@@ -111,10 +106,10 @@
111	106
112	107	int tcp_twsk_unique(struct sock sk, struct sock sktw, void *twp)
113	108	{
	109	+ int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
114	110	const struct inet_timewait_sock *tw = inet_twsk(sktw);
115	111	const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
116	112	struct tcp_sock *tp = tcp_sk(sk);
117		- int reuse = sock_net(sk)->ipv4.sysctl_tcp_tw_reuse;
118	113
119	114	if (reuse == 2) {
120	115	/* Still does not detect everything that goes through
..	..	@@ -127,11 +122,9 @@
127	122	#if IS_ENABLED(CONFIG_IPV6)
128	123	if (tw->tw_family == AF_INET6) {
129	124	if (ipv6_addr_loopback(&tw->tw_v6_daddr) \|\|
130		- (ipv6_addr_v4mapped(&tw->tw_v6_daddr) &&
131		- (tw->tw_v6_daddr.s6_addr[12] == 127)) \|\|
	125	+ ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) \|\|
132	126	ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) \|\|
133		- (ipv6_addr_v4mapped(&tw->tw_v6_rcv_saddr) &&
134		- (tw->tw_v6_rcv_saddr.s6_addr[12] == 127)))
	127	+ ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
135	128	loopback = true;
136	129	} else
137	130	#endif
..	..	@@ -329,6 +322,8 @@
329	322	* if necessary.
330	323	*/
331	324	tcp_set_state(sk, TCP_CLOSE);
	325	+ if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
	326	+ inet_reset_saddr(sk);
332	327	ip_rt_put(rt);
333	328	sk->sk_route_caps = 0;
334	329	inet->inet_dport = 0;
..	..	@@ -411,6 +406,46 @@
411	406	}
412	407	EXPORT_SYMBOL(tcp_req_err);
413	408
	409	+/* TCP-LD (RFC 6069) logic */
	410	+void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
	411	+{
	412	+ struct inet_connection_sock *icsk = inet_csk(sk);
	413	+ struct tcp_sock *tp = tcp_sk(sk);
	414	+ struct sk_buff *skb;
	415	+ s32 remaining;
	416	+ u32 delta_us;
	417	+
	418	+ if (sock_owned_by_user(sk))
	419	+ return;
	420	+
	421	+ if (seq != tp->snd_una \|\| !icsk->icsk_retransmits \|\|
	422	+ !icsk->icsk_backoff)
	423	+ return;
	424	+
	425	+ skb = tcp_rtx_queue_head(sk);
	426	+ if (WARN_ON_ONCE(!skb))
	427	+ return;
	428	+
	429	+ icsk->icsk_backoff--;
	430	+ icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
	431	+ icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
	432	+
	433	+ tcp_mstamp_refresh(tp);
	434	+ delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
	435	+ remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
	436	+
	437	+ if (remaining > 0) {
	438	+ inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
	439	+ remaining, TCP_RTO_MAX);
	440	+ } else {
	441	+ /* RTO revert clocked out retransmission.
	442	+ * Will retransmit now.
	443	+ */
	444	+ tcp_retransmit_timer(sk);
	445	+ }
	446	+}
	447	+EXPORT_SYMBOL(tcp_ld_RTO_revert);
	448	+
414	449	/*
415	450	* This routine is called by the ICMP module when it gets some
416	451	* sort of error condition. If err < 0 then the socket should
..	..	@@ -427,43 +462,40 @@
427	462	*
428	463	*/
429	464
430		-void tcp_v4_err(struct sk_buff *icmp_skb, u32 info)
	465	+int tcp_v4_err(struct sk_buff *skb, u32 info)
431	466	{
432		- const struct iphdr iph = (const struct iphdr )icmp_skb->data;
433		- struct tcphdr th = (struct tcphdr )(icmp_skb->data + (iph->ihl << 2));
434		- struct inet_connection_sock *icsk;
	467	+ const struct iphdr iph = (const struct iphdr )skb->data;
	468	+ struct tcphdr th = (struct tcphdr )(skb->data + (iph->ihl << 2));
435	469	struct tcp_sock *tp;
436	470	struct inet_sock *inet;
437		- const int type = icmp_hdr(icmp_skb)->type;
438		- const int code = icmp_hdr(icmp_skb)->code;
	471	+ const int type = icmp_hdr(skb)->type;
	472	+ const int code = icmp_hdr(skb)->code;
439	473	struct sock *sk;
440		- struct sk_buff *skb;
441	474	struct request_sock *fastopen;
442	475	u32 seq, snd_una;
443		- s32 remaining;
444		- u32 delta_us;
445	476	int err;
446		- struct net *net = dev_net(icmp_skb->dev);
	477	+ struct net *net = dev_net(skb->dev);
447	478
448	479	sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
449	480	th->dest, iph->saddr, ntohs(th->source),
450		- inet_iif(icmp_skb), 0);
	481	+ inet_iif(skb), 0);
451	482	if (!sk) {
452	483	__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
453		- return;
	484	+ return -ENOENT;
454	485	}
455	486	if (sk->sk_state == TCP_TIME_WAIT) {
456	487	inet_twsk_put(inet_twsk(sk));
457		- return;
	488	+ return 0;
458	489	}
459	490	seq = ntohl(th->seq);
460		- if (sk->sk_state == TCP_NEW_SYN_RECV)
461		- return tcp_req_err(sk, seq,
462		- type == ICMP_PARAMETERPROB \|\|
463		- type == ICMP_TIME_EXCEEDED \|\|
464		- (type == ICMP_DEST_UNREACH &&
465		- (code == ICMP_NET_UNREACH \|\|
466		- code == ICMP_HOST_UNREACH)));
	491	+ if (sk->sk_state == TCP_NEW_SYN_RECV) {
	492	+ tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB \|\|
	493	+ type == ICMP_TIME_EXCEEDED \|\|
	494	+ (type == ICMP_DEST_UNREACH &&
	495	+ (code == ICMP_NET_UNREACH \|\|
	496	+ code == ICMP_HOST_UNREACH)));
	497	+ return 0;
	498	+ }
467	499
468	500	bh_lock_sock(sk);
469	501	/* If too many ICMPs get dropped on busy
..	..	@@ -483,10 +515,9 @@
483	515	goto out;
484	516	}
485	517
486		- icsk = inet_csk(sk);
487	518	tp = tcp_sk(sk);
488	519	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
489		- fastopen = tp->fastopen_rsk;
	520	+ fastopen = rcu_dereference(tp->fastopen_rsk);
490	521	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
491	522	if (sk->sk_state != TCP_LISTEN &&
492	523	!between(seq, snd_una, tp->snd_nxt)) {
..	..	@@ -497,7 +528,7 @@
497	528	switch (type) {
498	529	case ICMP_REDIRECT:
499	530	if (!sock_owned_by_user(sk))
500		- do_redirect(icmp_skb, sk);
	531	+ do_redirect(skb, sk);
501	532	goto out;
502	533	case ICMP_SOURCE_QUENCH:
503	534	/* Just silently ignore these. */
..	..	@@ -528,40 +559,12 @@
528	559	}
529	560
530	561	err = icmp_err_convert[code].errno;
531		- /* check if icmp_skb allows revert of backoff
532		- * (see draft-zimmermann-tcp-lcd) */
533		- if (code != ICMP_NET_UNREACH && code != ICMP_HOST_UNREACH)
534		- break;
535		- if (seq != tp->snd_una \|\| !icsk->icsk_retransmits \|\|
536		- !icsk->icsk_backoff \|\| fastopen)
537		- break;
538		-
539		- if (sock_owned_by_user(sk))
540		- break;
541		-
542		- skb = tcp_rtx_queue_head(sk);
543		- if (WARN_ON_ONCE(!skb))
544		- break;
545		-
546		- icsk->icsk_backoff--;
547		- icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) :
548		- TCP_TIMEOUT_INIT;
549		- icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
550		-
551		- tcp_mstamp_refresh(tp);
552		- delta_us = (u32)(tp->tcp_mstamp - skb->skb_mstamp);
553		- remaining = icsk->icsk_rto -
554		- usecs_to_jiffies(delta_us);
555		-
556		- if (remaining > 0) {
557		- inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
558		- remaining, TCP_RTO_MAX);
559		- } else {
560		- /* RTO revert clocked out retransmission.
561		- * Will retransmit now */
562		- tcp_retransmit_timer(sk);
563		- }
564		-
	562	+ /* check if this ICMP message allows revert of backoff.
	563	+ * (see RFC 6069)
	564	+ */
	565	+ if (!fastopen &&
	566	+ (code == ICMP_NET_UNREACH \|\| code == ICMP_HOST_UNREACH))
	567	+ tcp_ld_RTO_revert(sk, seq);
565	568	break;
566	569	case ICMP_TIME_EXCEEDED:
567	570	err = EHOSTUNREACH;
..	..	@@ -574,10 +577,12 @@
574	577	case TCP_SYN_SENT:
575	578	case TCP_SYN_RECV:
576	579	/* Only in fast or simultaneous open. If a fast open socket is
577		- * is already accepted it is treated as a connected one below.
	580	+ * already accepted it is treated as a connected one below.
578	581	*/
579	582	if (fastopen && !fastopen->sk)
580	583	break;
	584	+
	585	+ ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
581	586
582	587	if (!sock_owned_by_user(sk)) {
583	588	sk->sk_err = err;
..	..	@@ -618,6 +623,7 @@
618	623	out:
619	624	bh_unlock_sock(sk);
620	625	sock_put(sk);
	626	+ return 0;
621	627	}
622	628
623	629	void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
..	..	@@ -638,7 +644,6 @@
638	644	}
639	645	EXPORT_SYMBOL(tcp_v4_send_check);
640	646
641		-static DEFINE_LOCAL_IRQ_LOCK(tcp_sk_lock);
642	647	/*
643	648	* This routine will send an RST to the other tcp.
644	649	*
..	..	@@ -669,8 +674,9 @@
669	674	int genhash;
670	675	struct sock *sk1 = NULL;
671	676	#endif
672		- struct net *net;
	677	+ u64 transmit_time = 0;
673	678	struct sock *ctl_sk;
	679	+ struct net *net;
674	680
675	681	/* Never send a reset in response to a reset. */
676	682	if (th->rst)
..	..	@@ -706,9 +712,21 @@
706	712	rcu_read_lock();
707	713	hash_location = tcp_parse_md5sig_option(th);
708	714	if (sk && sk_fullsock(sk)) {
709		- key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)
710		- &ip_hdr(skb)->saddr, AF_INET);
	715	+ const union tcp_md5_addr *addr;
	716	+ int l3index;
	717	+
	718	+ /* sdif set, means packet ingressed via a device
	719	+ * in an L3 domain and inet_iif is set to it.
	720	+ */
	721	+ l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
	722	+ addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
	723	+ key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
711	724	} else if (hash_location) {
	725	+ const union tcp_md5_addr *addr;
	726	+ int sdif = tcp_v4_sdif(skb);
	727	+ int dif = inet_iif(skb);
	728	+ int l3index;
	729	+
712	730	/*
713	731	* active side is lost. Try to find listening socket through
714	732	* source port, and then find md5 key through listening socket.
..	..	@@ -719,14 +737,17 @@
719	737	sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
720	738	ip_hdr(skb)->saddr,
721	739	th->source, ip_hdr(skb)->daddr,
722		- ntohs(th->source), inet_iif(skb),
723		- tcp_v4_sdif(skb));
	740	+ ntohs(th->source), dif, sdif);
724	741	/* don't send rst if it can't find key */
725	742	if (!sk1)
726	743	goto out;
727	744
728		- key = tcp_md5_do_lookup(sk1, (union tcp_md5_addr *)
729		- &ip_hdr(skb)->saddr, AF_INET);
	745	+ /* sdif set, means packet ingressed via a device
	746	+ * in an L3 domain and dif is set to it.
	747	+ */
	748	+ l3index = sdif ? dif : 0;
	749	+ addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
	750	+ key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
730	751	if (!key)
731	752	goto out;
732	753
..	..	@@ -773,20 +794,23 @@
773	794	arg.tos = ip_hdr(skb)->tos;
774	795	arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
775	796	local_bh_disable();
776		- local_lock(tcp_sk_lock);
777		- ctl_sk = *this_cpu_ptr(net->ipv4.tcp_sk);
778		- if (sk)
	797	+ ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
	798	+ if (sk) {
779	799	ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
780	800	inet_twsk(sk)->tw_mark : sk->sk_mark;
	801	+ ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
	802	+ inet_twsk(sk)->tw_priority : sk->sk_priority;
	803	+ transmit_time = tcp_transmit_time(sk);
	804	+ }
781	805	ip_send_unicast_reply(ctl_sk,
782	806	skb, &TCP_SKB_CB(skb)->header.h4.opt,
783	807	ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
784		- &arg, arg.iov[0].iov_len);
	808	+ &arg, arg.iov[0].iov_len,
	809	+ transmit_time);
785	810
786	811	ctl_sk->sk_mark = 0;
787	812	__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
788	813	__TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
789		- local_unlock(tcp_sk_lock);
790	814	local_bh_enable();
791	815
792	816	#ifdef CONFIG_TCP_MD5SIG
..	..	@@ -817,6 +841,7 @@
817	841	struct net *net = sock_net(sk);
818	842	struct ip_reply_arg arg;
819	843	struct sock *ctl_sk;
	844	+ u64 transmit_time;
820	845
821	846	memset(&rep.th, 0, sizeof(struct tcphdr));
822	847	memset(&arg, 0, sizeof(arg));
..	..	@@ -867,19 +892,20 @@
867	892	arg.tos = tos;
868	893	arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
869	894	local_bh_disable();
870		- local_lock(tcp_sk_lock);
871		- ctl_sk = *this_cpu_ptr(net->ipv4.tcp_sk);
872		- if (sk)
873		- ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
874		- inet_twsk(sk)->tw_mark : sk->sk_mark;
	895	+ ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
	896	+ ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
	897	+ inet_twsk(sk)->tw_mark : sk->sk_mark;
	898	+ ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
	899	+ inet_twsk(sk)->tw_priority : sk->sk_priority;
	900	+ transmit_time = tcp_transmit_time(sk);
875	901	ip_send_unicast_reply(ctl_sk,
876	902	skb, &TCP_SKB_CB(skb)->header.h4.opt,
877	903	ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
878		- &arg, arg.iov[0].iov_len);
	904	+ &arg, arg.iov[0].iov_len,
	905	+ transmit_time);
879	906
880	907	ctl_sk->sk_mark = 0;
881	908	__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
882		- local_unlock(tcp_sk_lock);
883	909	local_bh_enable();
884	910	}
885	911
..	..	@@ -905,6 +931,9 @@
905	931	static void tcp_v4_reqsk_send_ack(const struct sock sk, struct sk_buff skb,
906	932	struct request_sock *req)
907	933	{
	934	+ const union tcp_md5_addr *addr;
	935	+ int l3index;
	936	+
908	937	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
909	938	* sk->sk_state == TCP_SYN_RECV -> for Fast Open.
910	939	*/
..	..	@@ -916,14 +945,15 @@
916	945	* exception of <SYN> segments, MUST be right-shifted by
917	946	* Rcv.Wind.Shift bits:
918	947	*/
	948	+ addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
	949	+ l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
919	950	tcp_v4_send_ack(sk, skb, seq,
920	951	tcp_rsk(req)->rcv_nxt,
921	952	req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
922	953	tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
923		- req->ts_recent,
	954	+ READ_ONCE(req->ts_recent),
924	955	0,
925		- tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&ip_hdr(skb)->saddr,
926		- AF_INET),
	956	+ tcp_md5_do_lookup(sk, l3index, addr, AF_INET),
927	957	inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
928	958	ip_hdr(skb)->tos);
929	959	}
..	..	@@ -937,26 +967,38 @@
937	967	struct flowi *fl,
938	968	struct request_sock *req,
939	969	struct tcp_fastopen_cookie *foc,
940		- enum tcp_synack_type synack_type)
	970	+ enum tcp_synack_type synack_type,
	971	+ struct sk_buff *syn_skb)
941	972	{
942	973	const struct inet_request_sock *ireq = inet_rsk(req);
943	974	struct flowi4 fl4;
944	975	int err = -1;
945	976	struct sk_buff *skb;
	977	+ u8 tos;
946	978
947	979	/* First, grab a route. */
948	980	if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
949	981	return -1;
950	982
951		- skb = tcp_make_synack(sk, dst, req, foc, synack_type);
	983	+ skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
952	984
953	985	if (skb) {
954	986	__tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
955	987
	988	+ tos = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
	989	+ (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) \|
	990	+ (inet_sk(sk)->tos & INET_ECN_MASK) :
	991	+ inet_sk(sk)->tos;
	992	+
	993	+ if (!INET_ECN_is_capable(tos) &&
	994	+ tcp_bpf_ca_needs_ecn((struct sock *)req))
	995	+ tos \|= INET_ECN_ECT_0;
	996	+
956	997	rcu_read_lock();
957	998	err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
958	999	ireq->ir_rmt_addr,
959		- rcu_dereference(ireq->ireq_opt));
	1000	+ rcu_dereference(ireq->ireq_opt),
	1001	+ tos);
960	1002	rcu_read_unlock();
961	1003	err = net_xmit_eval(err);
962	1004	}
..	..	@@ -979,10 +1021,27 @@
979	1021	* We need to maintain these in the sk structure.
980	1022	*/
981	1023
	1024	+DEFINE_STATIC_KEY_FALSE(tcp_md5_needed);
	1025	+EXPORT_SYMBOL(tcp_md5_needed);
	1026	+
	1027	+static bool better_md5_match(struct tcp_md5sig_key old, struct tcp_md5sig_key new)
	1028	+{
	1029	+ if (!old)
	1030	+ return true;
	1031	+
	1032	+ /* l3index always overrides non-l3index */
	1033	+ if (old->l3index && new->l3index == 0)
	1034	+ return false;
	1035	+ if (old->l3index == 0 && new->l3index)
	1036	+ return true;
	1037	+
	1038	+ return old->prefixlen < new->prefixlen;
	1039	+}
	1040	+
982	1041	/* Find the Key structure for an address. */
983		-struct tcp_md5sig_key tcp_md5_do_lookup(const struct sock sk,
984		- const union tcp_md5_addr *addr,
985		- int family)
	1042	+struct tcp_md5sig_key __tcp_md5_do_lookup(const struct sock sk, int l3index,
	1043	+ const union tcp_md5_addr *addr,
	1044	+ int family)
986	1045	{
987	1046	const struct tcp_sock *tp = tcp_sk(sk);
988	1047	struct tcp_md5sig_key *key;
..	..	@@ -997,10 +1056,12 @@
997	1056	if (!md5sig)
998	1057	return NULL;
999	1058
1000		- hlist_for_each_entry_rcu(key, &md5sig->head, node) {
	1059	+ hlist_for_each_entry_rcu(key, &md5sig->head, node,
	1060	+ lockdep_sock_is_held(sk)) {
1001	1061	if (key->family != family)
1002	1062	continue;
1003		-
	1063	+ if (key->l3index && key->l3index != l3index)
	1064	+ continue;
1004	1065	if (family == AF_INET) {
1005	1066	mask = inet_make_mask(key->prefixlen);
1006	1067	match = (key->addr.a4.s_addr & mask) ==
..	..	@@ -1014,17 +1075,17 @@
1014	1075	match = false;
1015	1076	}
1016	1077
1017		- if (match && (!best_match \|\|
1018		- key->prefixlen > best_match->prefixlen))
	1078	+ if (match && better_md5_match(best_match, key))
1019	1079	best_match = key;
1020	1080	}
1021	1081	return best_match;
1022	1082	}
1023		-EXPORT_SYMBOL(tcp_md5_do_lookup);
	1083	+EXPORT_SYMBOL(__tcp_md5_do_lookup);
1024	1084
1025	1085	static struct tcp_md5sig_key tcp_md5_do_lookup_exact(const struct sock sk,
1026	1086	const union tcp_md5_addr *addr,
1027		- int family, u8 prefixlen)
	1087	+ int family, u8 prefixlen,
	1088	+ int l3index)
1028	1089	{
1029	1090	const struct tcp_sock *tp = tcp_sk(sk);
1030	1091	struct tcp_md5sig_key *key;
..	..	@@ -1040,8 +1101,11 @@
1040	1101	if (family == AF_INET6)
1041	1102	size = sizeof(struct in6_addr);
1042	1103	#endif
1043		- hlist_for_each_entry_rcu(key, &md5sig->head, node) {
	1104	+ hlist_for_each_entry_rcu(key, &md5sig->head, node,
	1105	+ lockdep_sock_is_held(sk)) {
1044	1106	if (key->family != family)
	1107	+ continue;
	1108	+ if (key->l3index != l3index)
1045	1109	continue;
1046	1110	if (!memcmp(&key->addr, addr, size) &&
1047	1111	key->prefixlen == prefixlen)
..	..	@@ -1054,28 +1118,34 @@
1054	1118	const struct sock *addr_sk)
1055	1119	{
1056	1120	const union tcp_md5_addr *addr;
	1121	+ int l3index;
1057	1122
	1123	+ l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
	1124	+ addr_sk->sk_bound_dev_if);
1058	1125	addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1059		- return tcp_md5_do_lookup(sk, addr, AF_INET);
	1126	+ return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1060	1127	}
1061	1128	EXPORT_SYMBOL(tcp_v4_md5_lookup);
1062	1129
1063	1130	/* This can be called on a newly created socket, from other files */
1064	1131	int tcp_md5_do_add(struct sock sk, const union tcp_md5_addr addr,
1065		- int family, u8 prefixlen, const u8 *newkey, u8 newkeylen,
1066		- gfp_t gfp)
	1132	+ int family, u8 prefixlen, int l3index,
	1133	+ const u8 *newkey, u8 newkeylen, gfp_t gfp)
1067	1134	{
1068	1135	/* Add Key to the list */
1069	1136	struct tcp_md5sig_key *key;
1070	1137	struct tcp_sock *tp = tcp_sk(sk);
1071	1138	struct tcp_md5sig_info *md5sig;
1072	1139
1073		- key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen);
	1140	+ key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index);
1074	1141	if (key) {
1075	1142	/* Pre-existing entry - just update that one.
1076	1143	* Note that the key might be used concurrently.
	1144	+ * data_race() is telling kcsan that we do not care of
	1145	+ * key mismatches, since changing MD5 key on live flows
	1146	+ * can lead to packet drops.
1077	1147	*/
1078		- memcpy(key->key, newkey, newkeylen);
	1148	+ data_race(memcpy(key->key, newkey, newkeylen));
1079	1149
1080	1150	/* Pairs with READ_ONCE() in tcp_md5_hash_key().
1081	1151	* Also note that a reader could catch new key->keylen value
..	..	@@ -1111,6 +1181,7 @@
1111	1181	key->keylen = newkeylen;
1112	1182	key->family = family;
1113	1183	key->prefixlen = prefixlen;
	1184	+ key->l3index = l3index;
1114	1185	memcpy(&key->addr, addr,
1115	1186	(family == AF_INET6) ? sizeof(struct in6_addr) :
1116	1187	sizeof(struct in_addr));
..	..	@@ -1120,11 +1191,11 @@
1120	1191	EXPORT_SYMBOL(tcp_md5_do_add);
1121	1192
1122	1193	int tcp_md5_do_del(struct sock sk, const union tcp_md5_addr addr, int family,
1123		- u8 prefixlen)
	1194	+ u8 prefixlen, int l3index)
1124	1195	{
1125	1196	struct tcp_md5sig_key *key;
1126	1197
1127		- key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen);
	1198	+ key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index);
1128	1199	if (!key)
1129	1200	return -ENOENT;
1130	1201	hlist_del_rcu(&key->node);
..	..	@@ -1151,16 +1222,18 @@
1151	1222	}
1152	1223
1153	1224	static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1154		- char __user *optval, int optlen)
	1225	+ sockptr_t optval, int optlen)
1155	1226	{
1156	1227	struct tcp_md5sig cmd;
1157	1228	struct sockaddr_in sin = (struct sockaddr_in )&cmd.tcpm_addr;
	1229	+ const union tcp_md5_addr *addr;
1158	1230	u8 prefixlen = 32;
	1231	+ int l3index = 0;
1159	1232
1160	1233	if (optlen < sizeof(cmd))
1161	1234	return -EINVAL;
1162	1235
1163		- if (copy_from_user(&cmd, optval, sizeof(cmd)))
	1236	+ if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
1164	1237	return -EFAULT;
1165	1238
1166	1239	if (sin->sin_family != AF_INET)
..	..	@@ -1173,16 +1246,34 @@
1173	1246	return -EINVAL;
1174	1247	}
1175	1248
	1249	+ if (optname == TCP_MD5SIG_EXT &&
	1250	+ cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
	1251	+ struct net_device *dev;
	1252	+
	1253	+ rcu_read_lock();
	1254	+ dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
	1255	+ if (dev && netif_is_l3_master(dev))
	1256	+ l3index = dev->ifindex;
	1257	+
	1258	+ rcu_read_unlock();
	1259	+
	1260	+ /* ok to reference set/not set outside of rcu;
	1261	+ * right now device MUST be an L3 master
	1262	+ */
	1263	+ if (!dev \|\| !l3index)
	1264	+ return -EINVAL;
	1265	+ }
	1266	+
	1267	+ addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
	1268	+
1176	1269	if (!cmd.tcpm_keylen)
1177		- return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
1178		- AF_INET, prefixlen);
	1270	+ return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index);
1179	1271
1180	1272	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1181	1273	return -EINVAL;
1182	1274
1183		- return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
1184		- AF_INET, prefixlen, cmd.tcpm_key, cmd.tcpm_keylen,
1185		- GFP_KERNEL);
	1275	+ return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index,
	1276	+ cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL);
1186	1277	}
1187	1278
1188	1279	static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
..	..	@@ -1292,7 +1383,8 @@
1292	1383
1293	1384	/* Called with rcu_read_lock() */
1294	1385	static bool tcp_v4_inbound_md5_hash(const struct sock *sk,
1295		- const struct sk_buff *skb)
	1386	+ const struct sk_buff *skb,
	1387	+ int dif, int sdif)
1296	1388	{
1297	1389	#ifdef CONFIG_TCP_MD5SIG
1298	1390	/*
..	..	@@ -1307,11 +1399,17 @@
1307	1399	struct tcp_md5sig_key *hash_expected;
1308	1400	const struct iphdr *iph = ip_hdr(skb);
1309	1401	const struct tcphdr *th = tcp_hdr(skb);
1310		- int genhash;
	1402	+ const union tcp_md5_addr *addr;
1311	1403	unsigned char newhash[16];
	1404	+ int genhash, l3index;
1312	1405
1313		- hash_expected = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&iph->saddr,
1314		- AF_INET);
	1406	+ /* sdif set, means packet ingressed via a device
	1407	+ * in an L3 domain and dif is set to the l3mdev
	1408	+ */
	1409	+ l3index = sdif ? dif : 0;
	1410	+
	1411	+ addr = (union tcp_md5_addr *)&iph->saddr;
	1412	+ hash_expected = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1315	1413	hash_location = tcp_parse_md5sig_option(th);
1316	1414
1317	1415	/* We've parsed the options - do we have a hash? */
..	..	@@ -1337,11 +1435,11 @@
1337	1435
1338	1436	if (genhash \|\| memcmp(hash_location, newhash, 16) != 0) {
1339	1437	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
1340		- net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s\n",
	1438	+ net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s L3 index %d\n",
1341	1439	&iph->saddr, ntohs(th->source),
1342	1440	&iph->daddr, ntohs(th->dest),
1343	1441	genhash ? " tcp_v4_calc_md5_hash failed"
1344		- : "");
	1442	+ : "", l3index);
1345	1443	return true;
1346	1444	}
1347	1445	return false;
..	..	@@ -1378,7 +1476,7 @@
1378	1476	.syn_ack_timeout = tcp_syn_ack_timeout,
1379	1477	};
1380	1478
1381		-static const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
	1479	+const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1382	1480	.mss_clamp = TCP_MSS_DEFAULT,
1383	1481	#ifdef CONFIG_TCP_MD5SIG
1384	1482	.req_md5_lookup = tcp_v4_md5_lookup,
..	..	@@ -1421,11 +1519,14 @@
1421	1519	bool *own_req)
1422	1520	{
1423	1521	struct inet_request_sock *ireq;
	1522	+ bool found_dup_sk = false;
1424	1523	struct inet_sock *newinet;
1425	1524	struct tcp_sock *newtp;
1426	1525	struct sock *newsk;
1427	1526	#ifdef CONFIG_TCP_MD5SIG
	1527	+ const union tcp_md5_addr *addr;
1428	1528	struct tcp_md5sig_key *key;
	1529	+ int l3index;
1429	1530	#endif
1430	1531	struct ip_options_rcu *inet_opt;
1431	1532
..	..	@@ -1456,6 +1557,12 @@
1456	1557	inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1457	1558	newinet->inet_id = prandom_u32();
1458	1559
	1560	+ /* Set ToS of the new socket based upon the value of incoming SYN.
	1561	+ * ECT bits are set later in tcp_init_transfer().
	1562	+ */
	1563	+ if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
	1564	+ newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
	1565	+
1459	1566	if (!dst) {
1460	1567	dst = inet_csk_route_child_sock(sk, newsk, req);
1461	1568	if (!dst)
..	..	@@ -1473,9 +1580,10 @@
1473	1580	tcp_initialize_rcv_mss(newsk);
1474	1581
1475	1582	#ifdef CONFIG_TCP_MD5SIG
	1583	+ l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1476	1584	/* Copy over the MD5 key from the original socket */
1477		- key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&newinet->inet_daddr,
1478		- AF_INET);
	1585	+ addr = (union tcp_md5_addr *)&newinet->inet_daddr;
	1586	+ key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1479	1587	if (key) {
1480	1588	/*
1481	1589	* We're using one, so create a matching key
..	..	@@ -1483,20 +1591,30 @@
1483	1591	* memory, then we end up not copying the key
1484	1592	* across. Shucks.
1485	1593	*/
1486		- tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newinet->inet_daddr,
1487		- AF_INET, 32, key->key, key->keylen, GFP_ATOMIC);
	1594	+ tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index,
	1595	+ key->key, key->keylen, GFP_ATOMIC);
1488	1596	sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
1489	1597	}
1490	1598	#endif
1491	1599
1492	1600	if (__inet_inherit_port(sk, newsk) < 0)
1493	1601	goto put_and_exit;
1494		- *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash));
	1602	+ *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
	1603	+ &found_dup_sk);
1495	1604	if (likely(*own_req)) {
1496	1605	tcp_move_syn(newtp, req);
1497	1606	ireq->ireq_opt = NULL;
1498	1607	} else {
1499	1608	newinet->inet_opt = NULL;
	1609	+
	1610	+ if (!req_unhash && found_dup_sk) {
	1611	+ /* This code path should only be executed in the
	1612	+ * syncookie case only
	1613	+ */
	1614	+ bh_unlock_sock(newsk);
	1615	+ sock_put(newsk);
	1616	+ newsk = NULL;
	1617	+ }
1500	1618	}
1501	1619	return newsk;
1502	1620
..	..	@@ -1526,6 +1644,21 @@
1526	1644	return sk;
1527	1645	}
1528	1646
	1647	+u16 tcp_v4_get_syncookie(struct sock sk, struct iphdr iph,
	1648	+ struct tcphdr th, u32 cookie)
	1649	+{
	1650	+ u16 mss = 0;
	1651	+#ifdef CONFIG_SYN_COOKIES
	1652	+ mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
	1653	+ &tcp_request_sock_ipv4_ops, sk, th);
	1654	+ if (mss) {
	1655	+ *cookie = __cookie_v4_init_sequence(iph, th, &mss);
	1656	+ tcp_synq_overflow(sk);
	1657	+ }
	1658	+#endif
	1659	+ return mss;
	1660	+}
	1661	+
1529	1662	/* The socket must have it's spinlock held when we get
1530	1663	* here, unless it is a TCP_LISTEN socket.
1531	1664	*
..	..	@@ -1539,15 +1672,18 @@
1539	1672	struct sock *rsk;
1540	1673
1541	1674	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1542		- struct dst_entry *dst = sk->sk_rx_dst;
	1675	+ struct dst_entry *dst;
	1676	+
	1677	+ dst = rcu_dereference_protected(sk->sk_rx_dst,
	1678	+ lockdep_sock_is_held(sk));
1543	1679
1544	1680	sock_rps_save_rxhash(sk, skb);
1545	1681	sk_mark_napi_id(sk, skb);
1546	1682	if (dst) {
1547	1683	if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif \|\|
1548	1684	!dst->ops->check(dst, 0)) {
	1685	+ RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1549	1686	dst_release(dst);
1550		- sk->sk_rx_dst = NULL;
1551	1687	}
1552	1688	}
1553	1689	tcp_rcv_established(sk, skb);
..	..	@@ -1622,7 +1758,7 @@
1622	1758	skb->sk = sk;
1623	1759	skb->destructor = sock_edemux;
1624	1760	if (sk_fullsock(sk)) {
1625		- struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
	1761	+ struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
1626	1762
1627	1763	if (dst)
1628	1764	dst = dst_check(dst, 0);
..	..	@@ -1636,13 +1772,16 @@
1636	1772
1637	1773	bool tcp_add_backlog(struct sock sk, struct sk_buff skb)
1638	1774	{
1639		- u32 limit = sk->sk_rcvbuf + sk->sk_sndbuf;
1640		-
1641		- /* Only socket owner can try to collapse/prune rx queues
1642		- * to reduce memory overhead, so add a little headroom here.
1643		- * Few sockets backlog are possibly concurrently non empty.
1644		- */
1645		- limit += 64*1024;
	1775	+ u32 limit, tail_gso_size, tail_gso_segs;
	1776	+ struct skb_shared_info *shinfo;
	1777	+ const struct tcphdr *th;
	1778	+ struct tcphdr *thtail;
	1779	+ struct sk_buff *tail;
	1780	+ unsigned int hdrlen;
	1781	+ bool fragstolen;
	1782	+ u32 gso_segs;
	1783	+ u32 gso_size;
	1784	+ int delta;
1646	1785
1647	1786	/* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1648	1787	* we can fix skb->truesize to its real value to avoid future drops.
..	..	@@ -1651,6 +1790,98 @@
1651	1790	* (if cooked by drivers without copybreak feature).
1652	1791	*/
1653	1792	skb_condense(skb);
	1793	+
	1794	+ skb_dst_drop(skb);
	1795	+
	1796	+ if (unlikely(tcp_checksum_complete(skb))) {
	1797	+ bh_unlock_sock(sk);
	1798	+ __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
	1799	+ __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
	1800	+ return true;
	1801	+ }
	1802	+
	1803	+ /* Attempt coalescing to last skb in backlog, even if we are
	1804	+ * above the limits.
	1805	+ * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
	1806	+ */
	1807	+ th = (const struct tcphdr *)skb->data;
	1808	+ hdrlen = th->doff * 4;
	1809	+
	1810	+ tail = sk->sk_backlog.tail;
	1811	+ if (!tail)
	1812	+ goto no_coalesce;
	1813	+ thtail = (struct tcphdr *)tail->data;
	1814	+
	1815	+ if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq \|\|
	1816	+ TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield \|\|
	1817	+ ((TCP_SKB_CB(tail)->tcp_flags \|
	1818	+ TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN \| TCPHDR_RST \| TCPHDR_URG)) \|\|
	1819	+ !((TCP_SKB_CB(tail)->tcp_flags &
	1820	+ TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) \|\|
	1821	+ ((TCP_SKB_CB(tail)->tcp_flags ^
	1822	+ TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE \| TCPHDR_CWR)) \|\|
	1823	+#ifdef CONFIG_TLS_DEVICE
	1824	+ tail->decrypted != skb->decrypted \|\|
	1825	+#endif
	1826	+ thtail->doff != th->doff \|\|
	1827	+ memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
	1828	+ goto no_coalesce;
	1829	+
	1830	+ __skb_pull(skb, hdrlen);
	1831	+
	1832	+ shinfo = skb_shinfo(skb);
	1833	+ gso_size = shinfo->gso_size ?: skb->len;
	1834	+ gso_segs = shinfo->gso_segs ?: 1;
	1835	+
	1836	+ shinfo = skb_shinfo(tail);
	1837	+ tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
	1838	+ tail_gso_segs = shinfo->gso_segs ?: 1;
	1839	+
	1840	+ if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
	1841	+ TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
	1842	+
	1843	+ if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
	1844	+ TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
	1845	+ thtail->window = th->window;
	1846	+ }
	1847	+
	1848	+ /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
	1849	+ * thtail->fin, so that the fast path in tcp_rcv_established()
	1850	+ * is not entered if we append a packet with a FIN.
	1851	+ * SYN, RST, URG are not present.
	1852	+ * ACK is set on both packets.
	1853	+ * PSH : we do not really care in TCP stack,
	1854	+ * at least for 'GRO' packets.
	1855	+ */
	1856	+ thtail->fin \|= th->fin;
	1857	+ TCP_SKB_CB(tail)->tcp_flags \|= TCP_SKB_CB(skb)->tcp_flags;
	1858	+
	1859	+ if (TCP_SKB_CB(skb)->has_rxtstamp) {
	1860	+ TCP_SKB_CB(tail)->has_rxtstamp = true;
	1861	+ tail->tstamp = skb->tstamp;
	1862	+ skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
	1863	+ }
	1864	+
	1865	+ /* Not as strict as GRO. We only need to carry mss max value */
	1866	+ shinfo->gso_size = max(gso_size, tail_gso_size);
	1867	+ shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
	1868	+
	1869	+ sk->sk_backlog.len += delta;
	1870	+ __NET_INC_STATS(sock_net(sk),
	1871	+ LINUX_MIB_TCPBACKLOGCOALESCE);
	1872	+ kfree_skb_partial(skb, fragstolen);
	1873	+ return false;
	1874	+ }
	1875	+ __skb_push(skb, hdrlen);
	1876	+
	1877	+no_coalesce:
	1878	+ limit = (u32)READ_ONCE(sk->sk_rcvbuf) + (u32)(READ_ONCE(sk->sk_sndbuf) >> 1);
	1879	+
	1880	+ /* Only socket owner can try to collapse/prune rx queues
	1881	+ * to reduce memory overhead, so add a little headroom here.
	1882	+ * Few sockets backlog are possibly concurrently non empty.
	1883	+ */
	1884	+ limit += 64 * 1024;
1654	1885
1655	1886	if (unlikely(sk_add_backlog(sk, skb, limit))) {
1656	1887	bh_unlock_sock(sk);
..	..	@@ -1704,7 +1935,9 @@
1704	1935	int tcp_v4_rcv(struct sk_buff *skb)
1705	1936	{
1706	1937	struct net *net = dev_net(skb->dev);
	1938	+ struct sk_buff *skb_to_free;
1707	1939	int sdif = inet_sdif(skb);
	1940	+ int dif = inet_iif(skb);
1708	1941	const struct iphdr *iph;
1709	1942	const struct tcphdr *th;
1710	1943	bool refcounted;
..	..	@@ -1753,7 +1986,8 @@
1753	1986	struct sock *nsk;
1754	1987
1755	1988	sk = req->rsk_listener;
1756		- if (unlikely(tcp_v4_inbound_md5_hash(sk, skb))) {
	1989	+ if (unlikely(!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb) \|\|
	1990	+ tcp_v4_inbound_md5_hash(sk, skb, dif, sdif))) {
1757	1991	sk_drops_add(sk, skb);
1758	1992	reqsk_put(req);
1759	1993	goto discard_it;
..	..	@@ -1792,6 +2026,7 @@
1792	2026	}
1793	2027	goto discard_and_relse;
1794	2028	}
	2029	+ nf_reset_ct(skb);
1795	2030	if (nsk == sk) {
1796	2031	reqsk_put(req);
1797	2032	tcp_v4_restore_cb(skb);
..	..	@@ -1811,10 +2046,10 @@
1811	2046	if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1812	2047	goto discard_and_relse;
1813	2048
1814		- if (tcp_v4_inbound_md5_hash(sk, skb))
	2049	+ if (tcp_v4_inbound_md5_hash(sk, skb, dif, sdif))
1815	2050	goto discard_and_relse;
1816	2051
1817		- nf_reset(skb);
	2052	+ nf_reset_ct(skb);
1818	2053
1819	2054	if (tcp_filter(sk, skb))
1820	2055	goto discard_and_relse;
..	..	@@ -1835,11 +2070,17 @@
1835	2070	tcp_segs_in(tcp_sk(sk), skb);
1836	2071	ret = 0;
1837	2072	if (!sock_owned_by_user(sk)) {
	2073	+ skb_to_free = sk->sk_rx_skb_cache;
	2074	+ sk->sk_rx_skb_cache = NULL;
1838	2075	ret = tcp_v4_do_rcv(sk, skb);
1839		- } else if (tcp_add_backlog(sk, skb)) {
1840		- goto discard_and_relse;
	2076	+ } else {
	2077	+ if (tcp_add_backlog(sk, skb))
	2078	+ goto discard_and_relse;
	2079	+ skb_to_free = NULL;
1841	2080	}
1842	2081	bh_unlock_sock(sk);
	2082	+ if (skb_to_free)
	2083	+ __kfree_skb(skb_to_free);
1843	2084
1844	2085	put_and_return:
1845	2086	if (refcounted)
..	..	@@ -1903,7 +2144,7 @@
1903	2144	}
1904	2145	}
1905	2146	/* to ACK */
1906		- /* fall through */
	2147	+ fallthrough;
1907	2148	case TCP_TW_ACK:
1908	2149	tcp_v4_timewait_ack(sk, skb);
1909	2150	break;
..	..	@@ -1927,7 +2168,7 @@
1927	2168	struct dst_entry *dst = skb_dst(skb);
1928	2169
1929	2170	if (dst && dst_hold_safe(dst)) {
1930		- sk->sk_rx_dst = dst;
	2171	+ rcu_assign_pointer(sk->sk_rx_dst, dst);
1931	2172	inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
1932	2173	}
1933	2174	}
..	..	@@ -1945,10 +2186,6 @@
1945	2186	.getsockopt = ip_getsockopt,
1946	2187	.addr2sockaddr = inet_csk_addr2sockaddr,
1947	2188	.sockaddr_len = sizeof(struct sockaddr_in),
1948		-#ifdef CONFIG_COMPAT
1949		- .compat_setsockopt = compat_ip_setsockopt,
1950		- .compat_getsockopt = compat_ip_getsockopt,
1951		-#endif
1952	2189	.mtu_reduced = tcp_v4_mtu_reduced,
1953	2190	};
1954	2191	EXPORT_SYMBOL(ipv4_specific);
..	..	@@ -2013,7 +2250,7 @@
2013	2250	if (inet_csk(sk)->icsk_bind_hash)
2014	2251	inet_put_port(sk);
2015	2252
2016		- BUG_ON(tp->fastopen_rsk);
	2253	+ BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2017	2254
2018	2255	/* If socket is aborted during connect operation */
2019	2256	tcp_free_fastopen_req(tp);
..	..	@@ -2034,12 +2271,17 @@
2034	2271	*/
2035	2272	static void listening_get_next(struct seq_file seq, void *cur)
2036	2273	{
2037		- struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
	2274	+ struct tcp_seq_afinfo *afinfo;
2038	2275	struct tcp_iter_state *st = seq->private;
2039	2276	struct net *net = seq_file_net(seq);
2040	2277	struct inet_listen_hashbucket *ilb;
2041	2278	struct hlist_nulls_node *node;
2042	2279	struct sock *sk = cur;
	2280	+
	2281	+ if (st->bpf_seq_afinfo)
	2282	+ afinfo = st->bpf_seq_afinfo;
	2283	+ else
	2284	+ afinfo = PDE_DATA(file_inode(seq->file));
2043	2285
2044	2286	if (!sk) {
2045	2287	get_head:
..	..	@@ -2058,7 +2300,8 @@
2058	2300	sk_nulls_for_each_from(sk, node) {
2059	2301	if (!net_eq(sock_net(sk), net))
2060	2302	continue;
2061		- if (sk->sk_family == afinfo->family)
	2303	+ if (afinfo->family == AF_UNSPEC \|\|
	2304	+ sk->sk_family == afinfo->family)
2062	2305	return sk;
2063	2306	}
2064	2307	spin_unlock(&ilb->lock);
..	..	@@ -2095,10 +2338,15 @@
2095	2338	*/
2096	2339	static void established_get_first(struct seq_file seq)
2097	2340	{
2098		- struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
	2341	+ struct tcp_seq_afinfo *afinfo;
2099	2342	struct tcp_iter_state *st = seq->private;
2100	2343	struct net *net = seq_file_net(seq);
2101	2344	void *rc = NULL;
	2345	+
	2346	+ if (st->bpf_seq_afinfo)
	2347	+ afinfo = st->bpf_seq_afinfo;
	2348	+ else
	2349	+ afinfo = PDE_DATA(file_inode(seq->file));
2102	2350
2103	2351	st->offset = 0;
2104	2352	for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
..	..	@@ -2112,7 +2360,8 @@
2112	2360
2113	2361	spin_lock_bh(lock);
2114	2362	sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
2115		- if (sk->sk_family != afinfo->family \|\|
	2363	+ if ((afinfo->family != AF_UNSPEC &&
	2364	+ sk->sk_family != afinfo->family) \|\|
2116	2365	!net_eq(sock_net(sk), net)) {
2117	2366	continue;
2118	2367	}
..	..	@@ -2127,11 +2376,16 @@
2127	2376
2128	2377	static void established_get_next(struct seq_file seq, void *cur)
2129	2378	{
2130		- struct tcp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
	2379	+ struct tcp_seq_afinfo *afinfo;
2131	2380	struct sock *sk = cur;
2132	2381	struct hlist_nulls_node *node;
2133	2382	struct tcp_iter_state *st = seq->private;
2134	2383	struct net *net = seq_file_net(seq);
	2384	+
	2385	+ if (st->bpf_seq_afinfo)
	2386	+ afinfo = st->bpf_seq_afinfo;
	2387	+ else
	2388	+ afinfo = PDE_DATA(file_inode(seq->file));
2135	2389
2136	2390	++st->num;
2137	2391	++st->offset;
..	..	@@ -2139,7 +2393,8 @@
2139	2393	sk = sk_nulls_next(sk);
2140	2394
2141	2395	sk_nulls_for_each_from(sk, node) {
2142		- if (sk->sk_family == afinfo->family &&
	2396	+ if ((afinfo->family == AF_UNSPEC \|\|
	2397	+ sk->sk_family == afinfo->family) &&
2143	2398	net_eq(sock_net(sk), net))
2144	2399	return sk;
2145	2400	}
..	..	@@ -2200,7 +2455,7 @@
2200	2455	break;
2201	2456	st->bucket = 0;
2202	2457	st->state = TCP_SEQ_STATE_ESTABLISHED;
2203		- /* Fallthrough */
	2458	+ fallthrough;
2204	2459	case TCP_SEQ_STATE_ESTABLISHED:
2205	2460	if (st->bucket > tcp_hashinfo.ehash_mask)
2206	2461	break;
..	..	@@ -2344,7 +2599,7 @@
2344	2599
2345	2600	state = inet_sk_state_load(sk);
2346	2601	if (state == TCP_LISTEN)
2347		- rx_queue = sk->sk_ack_backlog;
	2602	+ rx_queue = READ_ONCE(sk->sk_ack_backlog);
2348	2603	else
2349	2604	/* Because we don't lock the socket,
2350	2605	* we might find a transient negative value.
..	..	@@ -2366,7 +2621,7 @@
2366	2621	refcount_read(&sk->sk_refcnt), sk,
2367	2622	jiffies_to_clock_t(icsk->icsk_rto),
2368	2623	jiffies_to_clock_t(icsk->icsk_ack.ato),
2369		- (icsk->icsk_ack.quick << 1) \| icsk->icsk_ack.pingpong,
	2624	+ (icsk->icsk_ack.quick << 1) \| inet_csk_in_pingpong_mode(sk),
2370	2625	tp->snd_cwnd,
2371	2626	state == TCP_LISTEN ?
2372	2627	fastopenq->max_qlen :
..	..	@@ -2418,6 +2673,74 @@
2418	2673	seq_pad(seq, '\n');
2419	2674	return 0;
2420	2675	}
	2676	+
	2677	+#ifdef CONFIG_BPF_SYSCALL
	2678	+struct bpf_iter__tcp {
	2679	+ __bpf_md_ptr(struct bpf_iter_meta *, meta);
	2680	+ __bpf_md_ptr(struct sock_common *, sk_common);
	2681	+ uid_t uid __aligned(8);
	2682	+};
	2683	+
	2684	+static int tcp_prog_seq_show(struct bpf_prog prog, struct bpf_iter_meta meta,
	2685	+ struct sock_common *sk_common, uid_t uid)
	2686	+{
	2687	+ struct bpf_iter__tcp ctx;
	2688	+
	2689	+ meta->seq_num--; /* skip SEQ_START_TOKEN */
	2690	+ ctx.meta = meta;
	2691	+ ctx.sk_common = sk_common;
	2692	+ ctx.uid = uid;
	2693	+ return bpf_iter_run_prog(prog, &ctx);
	2694	+}
	2695	+
	2696	+static int bpf_iter_tcp_seq_show(struct seq_file seq, void v)
	2697	+{
	2698	+ struct bpf_iter_meta meta;
	2699	+ struct bpf_prog *prog;
	2700	+ struct sock *sk = v;
	2701	+ uid_t uid;
	2702	+
	2703	+ if (v == SEQ_START_TOKEN)
	2704	+ return 0;
	2705	+
	2706	+ if (sk->sk_state == TCP_TIME_WAIT) {
	2707	+ uid = 0;
	2708	+ } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
	2709	+ const struct request_sock *req = v;
	2710	+
	2711	+ uid = from_kuid_munged(seq_user_ns(seq),
	2712	+ sock_i_uid(req->rsk_listener));
	2713	+ } else {
	2714	+ uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
	2715	+ }
	2716	+
	2717	+ meta.seq = seq;
	2718	+ prog = bpf_iter_get_info(&meta, false);
	2719	+ return tcp_prog_seq_show(prog, &meta, v, uid);
	2720	+}
	2721	+
	2722	+static void bpf_iter_tcp_seq_stop(struct seq_file seq, void v)
	2723	+{
	2724	+ struct bpf_iter_meta meta;
	2725	+ struct bpf_prog *prog;
	2726	+
	2727	+ if (!v) {
	2728	+ meta.seq = seq;
	2729	+ prog = bpf_iter_get_info(&meta, true);
	2730	+ if (prog)
	2731	+ (void)tcp_prog_seq_show(prog, &meta, v, 0);
	2732	+ }
	2733	+
	2734	+ tcp_seq_stop(seq, v);
	2735	+}
	2736	+
	2737	+static const struct seq_operations bpf_iter_tcp_seq_ops = {
	2738	+ .show = bpf_iter_tcp_seq_show,
	2739	+ .start = tcp_seq_start,
	2740	+ .next = tcp_seq_next,
	2741	+ .stop = bpf_iter_tcp_seq_stop,
	2742	+};
	2743	+#endif
2421	2744
2422	2745	static const struct seq_operations tcp4_seq_ops = {
2423	2746	.show = tcp4_seq_show,
..	..	@@ -2499,10 +2822,6 @@
2499	2822	.rsk_prot = &tcp_request_sock_ops,
2500	2823	.h.hashinfo = &tcp_hashinfo,
2501	2824	.no_autobind = true,
2502		-#ifdef CONFIG_COMPAT
2503		- .compat_setsockopt = compat_tcp_setsockopt,
2504		- .compat_getsockopt = compat_tcp_getsockopt,
2505		-#endif
2506	2825	.diag_destroy = tcp_abort,
2507	2826	};
2508	2827	EXPORT_SYMBOL(tcp_prot);
..	..	@@ -2512,7 +2831,8 @@
2512	2831	int cpu;
2513	2832
2514	2833	if (net->ipv4.tcp_congestion_control)
2515		- module_put(net->ipv4.tcp_congestion_control->owner);
	2834	+ bpf_module_put(net->ipv4.tcp_congestion_control,
	2835	+ net->ipv4.tcp_congestion_control->owner);
2516	2836
2517	2837	for_each_possible_cpu(cpu)
2518	2838	inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
..	..	@@ -2551,6 +2871,7 @@
2551	2871	net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
2552	2872	net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
2553	2873	net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
	2874	+ net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
2554	2875
2555	2876	net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
2556	2877	net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
..	..	@@ -2566,12 +2887,13 @@
2566	2887	net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
2567	2888	net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
2568	2889	net->ipv4.sysctl_tcp_tw_reuse = 2;
	2890	+ net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
2569	2891
2570	2892	cnt = tcp_hashinfo.ehash_mask + 1;
2571		- net->ipv4.tcp_death_row.sysctl_max_tw_buckets = (cnt + 1) / 2;
	2893	+ net->ipv4.tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
2572	2894	net->ipv4.tcp_death_row.hashinfo = &tcp_hashinfo;
2573	2895
2574		- net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 256);
	2896	+ net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 128);
2575	2897	net->ipv4.sysctl_tcp_sack = 1;
2576	2898	net->ipv4.sysctl_tcp_window_scaling = 1;
2577	2899	net->ipv4.sysctl_tcp_timestamps = 1;
..	..	@@ -2590,8 +2912,8 @@
2590	2912	* which are too large can cause TCP streams to be bursty.
2591	2913	*/
2592	2914	net->ipv4.sysctl_tcp_tso_win_divisor = 3;
2593		- /* Default TSQ limit of four TSO segments */
2594		- net->ipv4.sysctl_tcp_limit_output_bytes = 262144;
	2915	+ /* Default TSQ limit of 16 TSO segments */
	2916	+ net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
2595	2917	/* rfc5961 challenge ack rate limiting */
2596	2918	net->ipv4.sysctl_tcp_challenge_ack_limit = 1000;
2597	2919	net->ipv4.sysctl_tcp_min_tso_segs = 2;
..	..	@@ -2609,15 +2931,17 @@
2609	2931	sizeof(init_net.ipv4.sysctl_tcp_wmem));
2610	2932	}
2611	2933	net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
	2934	+ net->ipv4.sysctl_tcp_comp_sack_slack_ns = 100 * NSEC_PER_USEC;
2612	2935	net->ipv4.sysctl_tcp_comp_sack_nr = 44;
2613	2936	net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
2614	2937	spin_lock_init(&net->ipv4.tcp_fastopen_ctx_lock);
2615		- net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 60 * 60;
	2938	+ net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
2616	2939	atomic_set(&net->ipv4.tfo_active_disable_times, 0);
2617	2940
2618	2941	/* Reno is always built in */
2619	2942	if (!net_eq(net, &init_net) &&
2620		- try_module_get(init_net.ipv4.tcp_congestion_control->owner))
	2943	+ bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
	2944	+ init_net.ipv4.tcp_congestion_control->owner))
2621	2945	net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
2622	2946	else
2623	2947	net->ipv4.tcp_congestion_control = &tcp_reno;
..	..	@@ -2645,8 +2969,68 @@
2645	2969	.exit_batch = tcp_sk_exit_batch,
2646	2970	};
2647	2971
	2972	+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
	2973	+DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
	2974	+ struct sock_common *sk_common, uid_t uid)
	2975	+
	2976	+static int bpf_iter_init_tcp(void priv_data, struct bpf_iter_aux_info aux)
	2977	+{
	2978	+ struct tcp_iter_state *st = priv_data;
	2979	+ struct tcp_seq_afinfo *afinfo;
	2980	+ int ret;
	2981	+
	2982	+ afinfo = kmalloc(sizeof(*afinfo), GFP_USER \| __GFP_NOWARN);
	2983	+ if (!afinfo)
	2984	+ return -ENOMEM;
	2985	+
	2986	+ afinfo->family = AF_UNSPEC;
	2987	+ st->bpf_seq_afinfo = afinfo;
	2988	+ ret = bpf_iter_init_seq_net(priv_data, aux);
	2989	+ if (ret)
	2990	+ kfree(afinfo);
	2991	+ return ret;
	2992	+}
	2993	+
	2994	+static void bpf_iter_fini_tcp(void *priv_data)
	2995	+{
	2996	+ struct tcp_iter_state *st = priv_data;
	2997	+
	2998	+ kfree(st->bpf_seq_afinfo);
	2999	+ bpf_iter_fini_seq_net(priv_data);
	3000	+}
	3001	+
	3002	+static const struct bpf_iter_seq_info tcp_seq_info = {
	3003	+ .seq_ops = &bpf_iter_tcp_seq_ops,
	3004	+ .init_seq_private = bpf_iter_init_tcp,
	3005	+ .fini_seq_private = bpf_iter_fini_tcp,
	3006	+ .seq_priv_size = sizeof(struct tcp_iter_state),
	3007	+};
	3008	+
	3009	+static struct bpf_iter_reg tcp_reg_info = {
	3010	+ .target = "tcp",
	3011	+ .ctx_arg_info_size = 1,
	3012	+ .ctx_arg_info = {
	3013	+ { offsetof(struct bpf_iter__tcp, sk_common),
	3014	+ PTR_TO_BTF_ID_OR_NULL },
	3015	+ },
	3016	+ .seq_info = &tcp_seq_info,
	3017	+};
	3018	+
	3019	+static void __init bpf_iter_register(void)
	3020	+{
	3021	+ tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
	3022	+ if (bpf_iter_reg_target(&tcp_reg_info))
	3023	+ pr_warn("Warning: could not register bpf iterator tcp\n");
	3024	+}
	3025	+
	3026	+#endif
	3027	+
2648	3028	void __init tcp_v4_init(void)
2649	3029	{
2650	3030	if (register_pernet_subsys(&tcp_sk_ops))
2651	3031	panic("Failed to create the TCP control socket.\n");
	3032	+
	3033	+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
	3034	+ bpf_iter_register();
	3035	+#endif
2652	3036	}