~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* INET An implementation of the TCP/IP protocol suite for the LINUX
3	4	* operating system. INET is implemented using the BSD Socket
..	..	@@ -205,11 +206,6 @@
205	206	* Hirokazu Takahashi : Use copy_from_user() instead of
206	207	* csum_and_copy_from_user() if possible.
207	208	*
208		- * This program is free software; you can redistribute it and/or
209		- * modify it under the terms of the GNU General Public License
210		- * as published by the Free Software Foundation; either version
211		- * 2 of the License, or(at your option) any later version.
212		- *
213	209	* Description of States:
214	210	*
215	211	* TCP_SYN_SENT sent a connection request, waiting for ack
..	..	@@ -262,7 +258,7 @@
262	258	#include <linux/net.h>
263	259	#include <linux/socket.h>
264	260	#include <linux/random.h>
265		-#include <linux/bootmem.h>
	261	+#include <linux/memblock.h>
266	262	#include <linux/highmem.h>
267	263	#include <linux/swap.h>
268	264	#include <linux/cache.h>
..	..	@@ -275,6 +271,7 @@
275	271	#include <net/icmp.h>
276	272	#include <net/inet_common.h>
277	273	#include <net/tcp.h>
	274	+#include <net/mptcp.h>
278	275	#include <net/xfrm.h>
279	276	#include <net/ip.h>
280	277	#include <net/sock.h>
..	..	@@ -282,6 +279,8 @@
282	279	#include <linux/uaccess.h>
283	280	#include <asm/ioctls.h>
284	281	#include <net/busy_poll.h>
	282	+
	283	+#include <trace/hooks/ipv4.h>
285	284
286	285	struct percpu_counter tcp_orphan_count;
287	286	EXPORT_SYMBOL_GPL(tcp_orphan_count);
..	..	@@ -320,6 +319,11 @@
320	319	*/
321	320	unsigned long tcp_memory_pressure __read_mostly;
322	321	EXPORT_SYMBOL_GPL(tcp_memory_pressure);
	322	+
	323	+DEFINE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key);
	324	+EXPORT_SYMBOL(tcp_rx_skb_cache_key);
	325	+
	326	+DEFINE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key);
323	327
324	328	void tcp_enter_memory_pressure(struct sock *sk)
325	329	{
..	..	@@ -416,6 +420,8 @@
416	420	INIT_LIST_HEAD(&tp->tsorted_sent_queue);
417	421
418	422	icsk->icsk_rto = TCP_TIMEOUT_INIT;
	423	+ icsk->icsk_rto_min = TCP_RTO_MIN;
	424	+ icsk->icsk_delack_max = TCP_DELACK_MAX;
419	425	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
420	426	minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
421	427
..	..	@@ -436,38 +442,24 @@
436	442	tp->snd_cwnd_clamp = ~0;
437	443	tp->mss_cache = TCP_MSS_DEFAULT;
438	444
439		- tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering;
	445	+ tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering);
440	446	tcp_assign_congestion_control(sk);
441	447
442	448	tp->tsoffset = 0;
443	449	tp->rack.reo_wnd_steps = 1;
444		-
445		- sk->sk_state = TCP_CLOSE;
446	450
447	451	sk->sk_write_space = sk_stream_write_space;
448	452	sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
449	453
450	454	icsk->icsk_sync_mss = tcp_sync_mss;
451	455
452		- sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1];
453		- sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1];
	456	+ WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1]));
	457	+ WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1]));
454	458
455	459	sk_sockets_allocated_inc(sk);
456	460	sk->sk_route_forced_caps = NETIF_F_GSO;
457	461	}
458	462	EXPORT_SYMBOL(tcp_init_sock);
459		-
460		-void tcp_init_transfer(struct sock *sk, int bpf_op)
461		-{
462		- struct inet_connection_sock *icsk = inet_csk(sk);
463		-
464		- tcp_mtup_init(sk);
465		- icsk->icsk_af_ops->rebuild_header(sk);
466		- tcp_init_metrics(sk);
467		- tcp_call_bpf(sk, bpf_op, 0, NULL);
468		- tcp_init_congestion_control(sk);
469		- tcp_init_buffer_space(sk);
470		-}
471	463
472	464	static void tcp_tx_timestamp(struct sock *sk, u16 tsflags)
473	465	{
..	..	@@ -564,10 +556,10 @@
564	556
565	557	/* Connected or passive Fast Open socket? */
566	558	if (state != TCP_SYN_SENT &&
567		- (state != TCP_SYN_RECV \|\| tp->fastopen_rsk)) {
	559	+ (state != TCP_SYN_RECV \|\| rcu_access_pointer(tp->fastopen_rsk))) {
568	560	int target = sock_rcvlowat(sk, 0, INT_MAX);
569	561
570		- if (tp->urg_seq == READ_ONCE(tp->copied_seq) &&
	562	+ if (READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
571	563	!sock_flag(sk, SOCK_URGINLINE) &&
572	564	tp->urg_data)
573	565	target++;
..	..	@@ -576,7 +568,7 @@
576	568	mask \|= EPOLLIN \| EPOLLRDNORM;
577	569
578	570	if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
579		- if (sk_stream_is_writeable(sk)) {
	571	+ if (__sk_stream_is_writeable(sk, 1)) {
580	572	mask \|= EPOLLOUT \| EPOLLWRNORM;
581	573	} else { /* send SIGIO later */
582	574	sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
..	..	@@ -588,7 +580,7 @@
588	580	* pairs with the input side.
589	581	*/
590	582	smp_mb__after_atomic();
591		- if (sk_stream_is_writeable(sk))
	583	+ if (__sk_stream_is_writeable(sk, 1))
592	584	mask \|= EPOLLOUT \| EPOLLWRNORM;
593	585	}
594	586	} else
..	..	@@ -628,7 +620,8 @@
628	620	unlock_sock_fast(sk, slow);
629	621	break;
630	622	case SIOCATMARK:
631		- answ = tp->urg_data && tp->urg_seq == READ_ONCE(tp->copied_seq);
	623	+ answ = tp->urg_data &&
	624	+ READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);
632	625	break;
633	626	case SIOCOUTQ:
634	627	if (sk->sk_state == TCP_LISTEN)
..	..	@@ -646,7 +639,8 @@
646	639	if ((1 << sk->sk_state) & (TCPF_SYN_SENT \| TCPF_SYN_RECV))
647	640	answ = 0;
648	641	else
649		- answ = READ_ONCE(tp->write_seq) - tp->snd_nxt;
	642	+ answ = READ_ONCE(tp->write_seq) -
	643	+ READ_ONCE(tp->snd_nxt);
650	644	break;
651	645	default:
652	646	return -ENOIOCTLCMD;
..	..	@@ -678,7 +672,7 @@
678	672	tcb->sacked = 0;
679	673	__skb_header_release(skb);
680	674	tcp_add_write_queue_tail(sk, skb);
681		- sk->sk_wmem_queued += skb->truesize;
	675	+ sk_wmem_queued_add(sk, skb->truesize);
682	676	sk_mem_charge(sk, skb->truesize);
683	677	if (tp->nonagle & TCP_NAGLE_PUSH)
684	678	tp->nonagle &= ~TCP_NAGLE_PUSH;
..	..	@@ -706,13 +700,13 @@
706	700	int size_goal)
707	701	{
708	702	return skb->len < size_goal &&
709		- sock_net(sk)->ipv4.sysctl_tcp_autocorking &&
	703	+ READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) &&
710	704	!tcp_rtx_queue_empty(sk) &&
711	705	refcount_read(&sk->sk_wmem_alloc) > skb->truesize;
712	706	}
713	707
714		-static void tcp_push(struct sock *sk, int flags, int mss_now,
715		- int nonagle, int size_goal)
	708	+void tcp_push(struct sock *sk, int flags, int mss_now,
	709	+ int nonagle, int size_goal)
716	710	{
717	711	struct tcp_sock *tp = tcp_sk(sk);
718	712	struct sk_buff *skb;
..	..	@@ -875,6 +869,18 @@
875	869	{
876	870	struct sk_buff *skb;
877	871
	872	+ if (likely(!size)) {
	873	+ skb = sk->sk_tx_skb_cache;
	874	+ if (skb) {
	875	+ skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
	876	+ sk->sk_tx_skb_cache = NULL;
	877	+ pskb_trim(skb, 0);
	878	+ INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
	879	+ skb_shinfo(skb)->tx_flags = 0;
	880	+ memset(TCP_SKB_CB(skb), 0, sizeof(struct tcp_skb_cb));
	881	+ return skb;
	882	+ }
	883	+ }
878	884	/* The TCP header must be at least 32-bit aligned. */
879	885	size = ALIGN(size, 4);
880	886
..	..	@@ -934,7 +940,7 @@
934	940	return max(size_goal, mss_now);
935	941	}
936	942
937		-static int tcp_send_mss(struct sock sk, int size_goal, int flags)
	943	+int tcp_send_mss(struct sock sk, int size_goal, int flags)
938	944	{
939	945	int mss_now;
940	946
..	..	@@ -969,6 +975,11 @@
969	975	ssize_t copied;
970	976	long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
971	977
	978	+ if (IS_ENABLED(CONFIG_DEBUG_VM) &&
	979	+ WARN_ONCE(!sendpage_ok(page),
	980	+ "page must not be a Slab one and have page_count > 0"))
	981	+ return -EINVAL;
	982	+
972	983	/* Wait for a connection to finish. One exception is TCP Fast Open
973	984	* (passive side) where data is allowed to be sent before a connection
974	985	* is fully established.
..	..	@@ -998,13 +1009,16 @@
998	1009	!tcp_skb_can_collapse_to(skb)) {
999	1010	new_segment:
1000	1011	if (!sk_stream_memory_free(sk))
1001		- goto wait_for_sndbuf;
	1012	+ goto wait_for_space;
1002	1013
1003	1014	skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
1004	1015	tcp_rtx_and_write_queues_empty(sk));
1005	1016	if (!skb)
1006		- goto wait_for_memory;
	1017	+ goto wait_for_space;
1007	1018
	1019	+#ifdef CONFIG_TLS_DEVICE
	1020	+ skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED);
	1021	+#endif
1008	1022	skb_entail(sk, skb);
1009	1023	copy = size_goal;
1010	1024	}
..	..	@@ -1019,7 +1033,7 @@
1019	1033	goto new_segment;
1020	1034	}
1021	1035	if (!sk_wmem_schedule(sk, copy))
1022		- goto wait_for_memory;
	1036	+ goto wait_for_space;
1023	1037
1024	1038	if (can_coalesce) {
1025	1039	skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
..	..	@@ -1034,7 +1048,7 @@
1034	1048	skb->len += copy;
1035	1049	skb->data_len += copy;
1036	1050	skb->truesize += copy;
1037		- sk->sk_wmem_queued += copy;
	1051	+ sk_wmem_queued_add(sk, copy);
1038	1052	sk_mem_charge(sk, copy);
1039	1053	skb->ip_summed = CHECKSUM_PARTIAL;
1040	1054	WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
..	..	@@ -1060,9 +1074,8 @@
1060	1074	tcp_push_one(sk, mss_now);
1061	1075	continue;
1062	1076
1063		-wait_for_sndbuf:
	1077	+wait_for_space:
1064	1078	set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1065		-wait_for_memory:
1066	1079	tcp_push(sk, flags & ~MSG_MORE, mss_now,
1067	1080	TCP_NAGLE_PUSH, size_goal);
1068	1081
..	..	@@ -1120,30 +1133,6 @@
1120	1133	}
1121	1134	EXPORT_SYMBOL(tcp_sendpage);
1122	1135
1123		-/* Do not bother using a page frag for very small frames.
1124		- * But use this heuristic only for the first skb in write queue.
1125		- *
1126		- * Having no payload in skb->head allows better SACK shifting
1127		- * in tcp_shift_skb_data(), reducing sack/rack overhead, because
1128		- * write queue has less skbs.
1129		- * Each skb can hold up to MAX_SKB_FRAGS * 32Kbytes, or ~0.5 MB.
1130		- * This also speeds up tso_fragment(), since it wont fallback
1131		- * to tcp_fragment().
1132		- */
1133		-static int linear_payload_sz(bool first_skb)
1134		-{
1135		- if (first_skb)
1136		- return SKB_WITH_OVERHEAD(2048 - MAX_TCP_HEADER);
1137		- return 0;
1138		-}
1139		-
1140		-static int select_size(bool first_skb, bool zc)
1141		-{
1142		- if (zc)
1143		- return 0;
1144		- return linear_payload_sz(first_skb);
1145		-}
1146		-
1147	1136	void tcp_free_fastopen_req(struct tcp_sock *tp)
1148	1137	{
1149	1138	if (tp->fastopen_req) {
..	..	@@ -1153,14 +1142,16 @@
1153	1142	}
1154	1143
1155	1144	static int tcp_sendmsg_fastopen(struct sock sk, struct msghdr msg,
1156		- int *copied, size_t size)
	1145	+ int *copied, size_t size,
	1146	+ struct ubuf_info *uarg)
1157	1147	{
1158	1148	struct tcp_sock *tp = tcp_sk(sk);
1159	1149	struct inet_sock *inet = inet_sk(sk);
1160	1150	struct sockaddr *uaddr = msg->msg_name;
1161	1151	int err, flags;
1162	1152
1163		- if (!(sock_net(sk)->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) \|\|
	1153	+ if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) &
	1154	+ TFO_CLIENT_ENABLE) \|\|
1164	1155	(uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1165	1156	uaddr->sa_family == AF_UNSPEC))
1166	1157	return -EOPNOTSUPP;
..	..	@@ -1173,6 +1164,7 @@
1173	1164	return -ENOBUFS;
1174	1165	tp->fastopen_req->data = msg;
1175	1166	tp->fastopen_req->size = size;
	1167	+ tp->fastopen_req->uarg = uarg;
1176	1168
1177	1169	if (inet->defer_connect) {
1178	1170	err = tcp_connect(sk);
..	..	@@ -1205,18 +1197,14 @@
1205	1197	struct sockcm_cookie sockc;
1206	1198	int flags, err, copied = 0;
1207	1199	int mss_now = 0, size_goal, copied_syn = 0;
1208		- bool process_backlog = false;
	1200	+ int process_backlog = 0;
1209	1201	bool zc = false;
1210	1202	long timeo;
1211	1203
	1204	+ trace_android_rvh_tcp_sendmsg_locked(sk, size);
1212	1205	flags = msg->msg_flags;
1213	1206
1214	1207	if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) {
1215		- if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED \| TCPF_CLOSE_WAIT)) {
1216		- err = -EINVAL;
1217		- goto out_err;
1218		- }
1219		-
1220	1208	skb = tcp_write_queue_tail(sk);
1221	1209	uarg = sock_zerocopy_realloc(sk, size, skb_zcopy(skb));
1222	1210	if (!uarg) {
..	..	@@ -1231,7 +1219,7 @@
1231	1219
1232	1220	if (unlikely(flags & MSG_FASTOPEN \|\| inet_sk(sk)->defer_connect) &&
1233	1221	!tp->repair) {
1234		- err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size);
	1222	+ err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
1235	1223	if (err == -EINPROGRESS && copied_syn > 0)
1236	1224	goto out;
1237	1225	else if (err)
..	..	@@ -1297,31 +1285,30 @@
1297	1285
1298	1286	if (copy <= 0 \|\| !tcp_skb_can_collapse_to(skb)) {
1299	1287	bool first_skb;
1300		- int linear;
1301	1288
1302	1289	new_segment:
1303	1290	if (!sk_stream_memory_free(sk))
1304		- goto wait_for_sndbuf;
	1291	+ goto wait_for_space;
1305	1292
1306		- if (process_backlog && sk_flush_backlog(sk)) {
1307		- process_backlog = false;
1308		- goto restart;
	1293	+ if (unlikely(process_backlog >= 16)) {
	1294	+ process_backlog = 0;
	1295	+ if (sk_flush_backlog(sk))
	1296	+ goto restart;
1309	1297	}
1310	1298	first_skb = tcp_rtx_and_write_queues_empty(sk);
1311		- linear = select_size(first_skb, zc);
1312		- skb = sk_stream_alloc_skb(sk, linear, sk->sk_allocation,
	1299	+ skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
1313	1300	first_skb);
1314	1301	if (!skb)
1315		- goto wait_for_memory;
	1302	+ goto wait_for_space;
1316	1303
1317		- process_backlog = true;
	1304	+ process_backlog++;
1318	1305	skb->ip_summed = CHECKSUM_PARTIAL;
1319	1306
1320	1307	skb_entail(sk, skb);
1321	1308	copy = size_goal;
1322	1309
1323	1310	/* All packets are restored as if they have
1324		- * already been sent. skb_mstamp isn't set to
	1311	+ * already been sent. skb_mstamp_ns isn't set to
1325	1312	* avoid wrong rtt estimation.
1326	1313	*/
1327	1314	if (tp->repair)
..	..	@@ -1345,7 +1332,7 @@
1345	1332	struct page_frag *pfrag = sk_page_frag(sk);
1346	1333
1347	1334	if (!sk_page_frag_refill(sk, pfrag))
1348		- goto wait_for_memory;
	1335	+ goto wait_for_space;
1349	1336
1350	1337	if (!skb_can_coalesce(skb, i, pfrag->page,
1351	1338	pfrag->offset)) {
..	..	@@ -1359,7 +1346,7 @@
1359	1346	copy = min_t(int, copy, pfrag->size - pfrag->offset);
1360	1347
1361	1348	if (!sk_wmem_schedule(sk, copy))
1362		- goto wait_for_memory;
	1349	+ goto wait_for_space;
1363	1350
1364	1351	err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1365	1352	pfrag->page,
..	..	@@ -1378,6 +1365,9 @@
1378	1365	}
1379	1366	pfrag->offset += copy;
1380	1367	} else {
	1368	+ if (!sk_wmem_schedule(sk, copy))
	1369	+ goto wait_for_space;
	1370	+
1381	1371	err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg);
1382	1372	if (err == -EMSGSIZE \|\| err == -EEXIST) {
1383	1373	tcp_mark_push(tp, skb);
..	..	@@ -1412,9 +1402,8 @@
1412	1402	tcp_push_one(sk, mss_now);
1413	1403	continue;
1414	1404
1415		-wait_for_sndbuf:
	1405	+wait_for_space:
1416	1406	set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1417		-wait_for_memory:
1418	1407	if (copied)
1419	1408	tcp_push(sk, flags & ~MSG_MORE, mss_now,
1420	1409	TCP_NAGLE_PUSH, size_goal);
..	..	@@ -1443,7 +1432,7 @@
1443	1432	if (copied + copied_syn)
1444	1433	goto out;
1445	1434	out_err:
1446		- sock_zerocopy_put_abort(uarg);
	1435	+ sock_zerocopy_put_abort(uarg, true);
1447	1436	err = sk_stream_error(sk, flags, err);
1448	1437	/* make sure we wake any epoll edge trigger waiter */
1449	1438	if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
..	..	@@ -1546,7 +1535,7 @@
1546	1535	* calculation of whether or not we must ACK for the sake of
1547	1536	* a window update.
1548	1537	*/
1549		-static void tcp_cleanup_rbuf(struct sock *sk, int copied)
	1538	+void tcp_cleanup_rbuf(struct sock *sk, int copied)
1550	1539	{
1551	1540	struct tcp_sock *tp = tcp_sk(sk);
1552	1541	bool time_to_ack = false;
..	..	@@ -1559,10 +1548,8 @@
1559	1548
1560	1549	if (inet_csk_ack_scheduled(sk)) {
1561	1550	const struct inet_connection_sock *icsk = inet_csk(sk);
1562		- /* Delayed ACKs frequently hit locked sockets during bulk
1563		- * receive. */
1564		- if (icsk->icsk_ack.blocked \|\|
1565		- /* Once-per-two-segments ACK was not sent by tcp_input.c */
	1551	+
	1552	+ if (/* Once-per-two-segments ACK was not sent by tcp_input.c */
1566	1553	tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss \|\|
1567	1554	/*
1568	1555	* If this read emptied read buffer, we send ACK, if
..	..	@@ -1573,7 +1560,7 @@
1573	1560	(copied > 0 &&
1574	1561	((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) \|\|
1575	1562	((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1576		- !icsk->icsk_ack.pingpong)) &&
	1563	+ !inet_csk_in_pingpong_mode(sk))) &&
1577	1564	!atomic_read(&sk->sk_rmem_alloc)))
1578	1565	time_to_ack = true;
1579	1566	}
..	..	@@ -1669,11 +1656,13 @@
1669	1656	if (!copied)
1670	1657	copied = used;
1671	1658	break;
1672		- } else if (used <= len) {
1673		- seq += used;
1674		- copied += used;
1675		- offset += used;
1676	1659	}
	1660	+ if (WARN_ON_ONCE(used > len))
	1661	+ used = len;
	1662	+ seq += used;
	1663	+ copied += used;
	1664	+ offset += used;
	1665	+
1677	1666	/* If recv_actor drops the lock (e.g. TCP splice
1678	1667	* receive) the skb pointer might be invalid when
1679	1668	* getting here: tcp_collapse might have deleted it
..	..	@@ -1725,9 +1714,9 @@
1725	1714	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1726	1715	cap = sk->sk_rcvbuf >> 1;
1727	1716	else
1728		- cap = sock_net(sk)->ipv4.sysctl_tcp_rmem[2] >> 1;
	1717	+ cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
1729	1718	val = min(val, cap);
1730		- sk->sk_rcvlowat = val ? : 1;
	1719	+ WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1731	1720
1732	1721	/* Check if we need to signal EPOLLIN right now */
1733	1722	tcp_data_ready(sk);
..	..	@@ -1737,7 +1726,7 @@
1737	1726
1738	1727	val <<= 1;
1739	1728	if (val > sk->sk_rcvbuf) {
1740		- sk->sk_rcvbuf = val;
	1729	+ WRITE_ONCE(sk->sk_rcvbuf, val);
1741	1730	tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val);
1742	1731	}
1743	1732	return 0;
..	..	@@ -1755,7 +1744,7 @@
1755	1744	return -EPERM;
1756	1745	vma->vm_flags &= ~(VM_MAYWRITE \| VM_MAYEXEC);
1757	1746
1758		- /* Instruct vm_insert_page() to not down_read(mmap_sem) */
	1747	+ /* Instruct vm_insert_page() to not mmap_read_lock(mm) */
1759	1748	vma->vm_flags \|= VM_MIXEDMAP;
1760	1749
1761	1750	vma->vm_ops = &tcp_vm_ops;
..	..	@@ -1763,16 +1752,126 @@
1763	1752	}
1764	1753	EXPORT_SYMBOL(tcp_mmap);
1765	1754
	1755	+static skb_frag_t skb_advance_to_frag(struct sk_buff skb, u32 offset_skb,
	1756	+ u32 *offset_frag)
	1757	+{
	1758	+ skb_frag_t *frag;
	1759	+
	1760	+ if (unlikely(offset_skb >= skb->len))
	1761	+ return NULL;
	1762	+
	1763	+ offset_skb -= skb_headlen(skb);
	1764	+ if ((int)offset_skb < 0 \|\| skb_has_frag_list(skb))
	1765	+ return NULL;
	1766	+
	1767	+ frag = skb_shinfo(skb)->frags;
	1768	+ while (offset_skb) {
	1769	+ if (skb_frag_size(frag) > offset_skb) {
	1770	+ *offset_frag = offset_skb;
	1771	+ return frag;
	1772	+ }
	1773	+ offset_skb -= skb_frag_size(frag);
	1774	+ ++frag;
	1775	+ }
	1776	+ *offset_frag = 0;
	1777	+ return frag;
	1778	+}
	1779	+
	1780	+static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,
	1781	+ struct sk_buff *skb, u32 copylen,
	1782	+ u32 offset, u32 seq)
	1783	+{
	1784	+ unsigned long copy_address = (unsigned long)zc->copybuf_address;
	1785	+ struct msghdr msg = {};
	1786	+ struct iovec iov;
	1787	+ int err;
	1788	+
	1789	+ if (copy_address != zc->copybuf_address)
	1790	+ return -EINVAL;
	1791	+
	1792	+ err = import_single_range(READ, (void __user *)copy_address,
	1793	+ copylen, &iov, &msg.msg_iter);
	1794	+ if (err)
	1795	+ return err;
	1796	+ err = skb_copy_datagram_msg(skb, *offset, &msg, copylen);
	1797	+ if (err)
	1798	+ return err;
	1799	+ zc->recv_skip_hint -= copylen;
	1800	+ *offset += copylen;
	1801	+ *seq += copylen;
	1802	+ return (__s32)copylen;
	1803	+}
	1804	+
	1805	+static int tcp_zerocopy_handle_leftover_data(struct tcp_zerocopy_receive *zc,
	1806	+ struct sock *sk,
	1807	+ struct sk_buff *skb,
	1808	+ u32 *seq,
	1809	+ s32 copybuf_len)
	1810	+{
	1811	+ u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint);
	1812	+
	1813	+ if (!copylen)
	1814	+ return 0;
	1815	+ /* skb is null if inq < PAGE_SIZE. */
	1816	+ if (skb)
	1817	+ offset = *seq - TCP_SKB_CB(skb)->seq;
	1818	+ else
	1819	+ skb = tcp_recv_skb(sk, *seq, &offset);
	1820	+
	1821	+ zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset,
	1822	+ seq);
	1823	+ return zc->copybuf_len < 0 ? 0 : copylen;
	1824	+}
	1825	+
	1826	+static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma,
	1827	+ struct page **pages,
	1828	+ unsigned long pages_to_map,
	1829	+ unsigned long *insert_addr,
	1830	+ u32 *length_with_pending,
	1831	+ u32 *seq,
	1832	+ struct tcp_zerocopy_receive *zc)
	1833	+{
	1834	+ unsigned long pages_remaining = pages_to_map;
	1835	+ int bytes_mapped;
	1836	+ int ret;
	1837	+
	1838	+ ret = vm_insert_pages(vma, *insert_addr, pages, &pages_remaining);
	1839	+ bytes_mapped = PAGE_SIZE * (pages_to_map - pages_remaining);
	1840	+ /* Even if vm_insert_pages fails, it may have partially succeeded in
	1841	+ * mapping (some but not all of the pages).
	1842	+ */
	1843	+ *seq += bytes_mapped;
	1844	+ *insert_addr += bytes_mapped;
	1845	+ if (ret) {
	1846	+ /* But if vm_insert_pages did fail, we have to unroll some state
	1847	+ * we speculatively touched before.
	1848	+ */
	1849	+ const int bytes_not_mapped = PAGE_SIZE * pages_remaining;
	1850	+ *length_with_pending -= bytes_not_mapped;
	1851	+ zc->recv_skip_hint += bytes_not_mapped;
	1852	+ }
	1853	+ return ret;
	1854	+}
	1855	+
1766	1856	static int tcp_zerocopy_receive(struct sock *sk,
1767	1857	struct tcp_zerocopy_receive *zc)
1768	1858	{
	1859	+ u32 length = 0, offset, vma_len, avail_len, aligned_len, copylen = 0;
1769	1860	unsigned long address = (unsigned long)zc->address;
	1861	+ s32 copybuf_len = zc->copybuf_len;
	1862	+ struct tcp_sock *tp = tcp_sk(sk);
	1863	+ #define PAGE_BATCH_SIZE 8
	1864	+ struct page *pages[PAGE_BATCH_SIZE];
1770	1865	const skb_frag_t *frags = NULL;
1771		- u32 length = 0, seq, offset;
1772	1866	struct vm_area_struct *vma;
1773	1867	struct sk_buff *skb = NULL;
1774		- struct tcp_sock *tp;
	1868	+ unsigned long pg_idx = 0;
	1869	+ unsigned long curr_addr;
	1870	+ u32 seq = tp->copied_seq;
	1871	+ int inq = tcp_inq(sk);
1775	1872	int ret;
	1873	+
	1874	+ zc->copybuf_len = 0;
1776	1875
1777	1876	if (address & (PAGE_SIZE - 1) \|\| address != zc->address)
1778	1877	return -EINVAL;
..	..	@@ -1782,65 +1881,98 @@
1782	1881
1783	1882	sock_rps_record_flow(sk);
1784	1883
1785		- down_read(&current->mm->mmap_sem);
	1884	+ mmap_read_lock(current->mm);
1786	1885
1787	1886	vma = find_vma(current->mm, address);
1788	1887	if (!vma \|\| vma->vm_start > address \|\| vma->vm_ops != &tcp_vm_ops) {
1789		- up_read(&current->mm->mmap_sem);
	1888	+ mmap_read_unlock(current->mm);
1790	1889	return -EINVAL;
1791	1890	}
1792		- zc->length = min_t(unsigned long, zc->length, vma->vm_end - address);
1793		-
1794		- tp = tcp_sk(sk);
1795		- seq = tp->copied_seq;
1796		- zc->length = min_t(u32, zc->length, tcp_inq(sk));
1797		- zc->length &= ~(PAGE_SIZE - 1);
1798		-
1799		- zap_page_range(vma, address, zc->length);
1800		-
1801		- zc->recv_skip_hint = 0;
	1891	+ vma_len = min_t(unsigned long, zc->length, vma->vm_end - address);
	1892	+ avail_len = min_t(u32, vma_len, inq);
	1893	+ aligned_len = avail_len & ~(PAGE_SIZE - 1);
	1894	+ if (aligned_len) {
	1895	+ zap_page_range(vma, address, aligned_len);
	1896	+ zc->length = aligned_len;
	1897	+ zc->recv_skip_hint = 0;
	1898	+ } else {
	1899	+ zc->length = avail_len;
	1900	+ zc->recv_skip_hint = avail_len;
	1901	+ }
1802	1902	ret = 0;
	1903	+ curr_addr = address;
1803	1904	while (length + PAGE_SIZE <= zc->length) {
1804	1905	if (zc->recv_skip_hint < PAGE_SIZE) {
	1906	+ u32 offset_frag;
	1907	+
	1908	+ /* If we're here, finish the current batch. */
	1909	+ if (pg_idx) {
	1910	+ ret = tcp_zerocopy_vm_insert_batch(vma, pages,
	1911	+ pg_idx,
	1912	+ &curr_addr,
	1913	+ &length,
	1914	+ &seq, zc);
	1915	+ if (ret)
	1916	+ goto out;
	1917	+ pg_idx = 0;
	1918	+ }
1805	1919	if (skb) {
	1920	+ if (zc->recv_skip_hint > 0)
	1921	+ break;
1806	1922	skb = skb->next;
1807	1923	offset = seq - TCP_SKB_CB(skb)->seq;
1808	1924	} else {
1809	1925	skb = tcp_recv_skb(sk, seq, &offset);
1810	1926	}
1811		-
1812	1927	zc->recv_skip_hint = skb->len - offset;
1813		- offset -= skb_headlen(skb);
1814		- if ((int)offset < 0 \|\| skb_has_frag_list(skb))
	1928	+ frags = skb_advance_to_frag(skb, offset, &offset_frag);
	1929	+ if (!frags \|\| offset_frag)
1815	1930	break;
1816		- frags = skb_shinfo(skb)->frags;
1817		- while (offset) {
1818		- if (frags->size > offset)
1819		- goto out;
1820		- offset -= frags->size;
	1931	+ }
	1932	+ if (skb_frag_size(frags) != PAGE_SIZE \|\| skb_frag_off(frags)) {
	1933	+ int remaining = zc->recv_skip_hint;
	1934	+
	1935	+ while (remaining && (skb_frag_size(frags) != PAGE_SIZE \|\|
	1936	+ skb_frag_off(frags))) {
	1937	+ remaining -= skb_frag_size(frags);
1821	1938	frags++;
1822	1939	}
	1940	+ zc->recv_skip_hint -= remaining;
	1941	+ break;
1823	1942	}
1824		- if (frags->size != PAGE_SIZE \|\| frags->page_offset)
1825		- break;
1826		- ret = vm_insert_page(vma, address + length,
1827		- skb_frag_page(frags));
1828		- if (ret)
1829		- break;
	1943	+ pages[pg_idx] = skb_frag_page(frags);
	1944	+ pg_idx++;
1830	1945	length += PAGE_SIZE;
1831		- seq += PAGE_SIZE;
1832	1946	zc->recv_skip_hint -= PAGE_SIZE;
1833	1947	frags++;
	1948	+ if (pg_idx == PAGE_BATCH_SIZE) {
	1949	+ ret = tcp_zerocopy_vm_insert_batch(vma, pages, pg_idx,
	1950	+ &curr_addr, &length,
	1951	+ &seq, zc);
	1952	+ if (ret)
	1953	+ goto out;
	1954	+ pg_idx = 0;
	1955	+ }
	1956	+ }
	1957	+ if (pg_idx) {
	1958	+ ret = tcp_zerocopy_vm_insert_batch(vma, pages, pg_idx,
	1959	+ &curr_addr, &length, &seq,
	1960	+ zc);
1834	1961	}
1835	1962	out:
1836		- up_read(&current->mm->mmap_sem);
1837		- if (length) {
	1963	+ mmap_read_unlock(current->mm);
	1964	+ /* Try to copy straggler data. */
	1965	+ if (!ret)
	1966	+ copylen = tcp_zerocopy_handle_leftover_data(zc, sk, skb, &seq,
	1967	+ copybuf_len);
	1968	+
	1969	+ if (length + copylen) {
1838	1970	WRITE_ONCE(tp->copied_seq, seq);
1839	1971	tcp_rcv_space_adjust(sk);
1840	1972
1841	1973	/* Clean up data we have read: This will do ACK frames. */
1842	1974	tcp_recv_skb(sk, seq, &offset);
1843		- tcp_cleanup_rbuf(sk, length);
	1975	+ tcp_cleanup_rbuf(sk, length + copylen);
1844	1976	ret = 0;
1845	1977	if (length == zc->length)
1846	1978	zc->recv_skip_hint = 0;
..	..	@@ -1854,57 +1986,82 @@
1854	1986	#endif
1855	1987
1856	1988	static void tcp_update_recv_tstamps(struct sk_buff *skb,
1857		- struct scm_timestamping *tss)
	1989	+ struct scm_timestamping_internal *tss)
1858	1990	{
1859	1991	if (skb->tstamp)
1860		- tss->ts[0] = ktime_to_timespec(skb->tstamp);
	1992	+ tss->ts[0] = ktime_to_timespec64(skb->tstamp);
1861	1993	else
1862		- tss->ts[0] = (struct timespec) {0};
	1994	+ tss->ts[0] = (struct timespec64) {0};
1863	1995
1864	1996	if (skb_hwtstamps(skb)->hwtstamp)
1865		- tss->ts[2] = ktime_to_timespec(skb_hwtstamps(skb)->hwtstamp);
	1997	+ tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
1866	1998	else
1867		- tss->ts[2] = (struct timespec) {0};
	1999	+ tss->ts[2] = (struct timespec64) {0};
1868	2000	}
1869	2001
1870	2002	/* Similar to __sock_recv_timestamp, but does not require an skb */
1871	2003	static void tcp_recv_timestamp(struct msghdr msg, const struct sock sk,
1872		- struct scm_timestamping *tss)
	2004	+ struct scm_timestamping_internal *tss)
1873	2005	{
1874		- struct timeval tv;
	2006	+ int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
1875	2007	bool has_timestamping = false;
1876	2008
1877	2009	if (tss->ts[0].tv_sec \|\| tss->ts[0].tv_nsec) {
1878	2010	if (sock_flag(sk, SOCK_RCVTSTAMP)) {
1879	2011	if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
1880		- put_cmsg(msg, SOL_SOCKET, SCM_TIMESTAMPNS,
1881		- sizeof(tss->ts[0]), &tss->ts[0]);
	2012	+ if (new_tstamp) {
	2013	+ struct __kernel_timespec kts = {
	2014	+ .tv_sec = tss->ts[0].tv_sec,
	2015	+ .tv_nsec = tss->ts[0].tv_nsec,
	2016	+ };
	2017	+ put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
	2018	+ sizeof(kts), &kts);
	2019	+ } else {
	2020	+ struct __kernel_old_timespec ts_old = {
	2021	+ .tv_sec = tss->ts[0].tv_sec,
	2022	+ .tv_nsec = tss->ts[0].tv_nsec,
	2023	+ };
	2024	+ put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
	2025	+ sizeof(ts_old), &ts_old);
	2026	+ }
1882	2027	} else {
1883		- tv.tv_sec = tss->ts[0].tv_sec;
1884		- tv.tv_usec = tss->ts[0].tv_nsec / 1000;
1885		-
1886		- put_cmsg(msg, SOL_SOCKET, SCM_TIMESTAMP,
1887		- sizeof(tv), &tv);
	2028	+ if (new_tstamp) {
	2029	+ struct __kernel_sock_timeval stv = {
	2030	+ .tv_sec = tss->ts[0].tv_sec,
	2031	+ .tv_usec = tss->ts[0].tv_nsec / 1000,
	2032	+ };
	2033	+ put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
	2034	+ sizeof(stv), &stv);
	2035	+ } else {
	2036	+ struct __kernel_old_timeval tv = {
	2037	+ .tv_sec = tss->ts[0].tv_sec,
	2038	+ .tv_usec = tss->ts[0].tv_nsec / 1000,
	2039	+ };
	2040	+ put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
	2041	+ sizeof(tv), &tv);
	2042	+ }
1888	2043	}
1889	2044	}
1890	2045
1891	2046	if (sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE)
1892	2047	has_timestamping = true;
1893	2048	else
1894		- tss->ts[0] = (struct timespec) {0};
	2049	+ tss->ts[0] = (struct timespec64) {0};
1895	2050	}
1896	2051
1897	2052	if (tss->ts[2].tv_sec \|\| tss->ts[2].tv_nsec) {
1898	2053	if (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)
1899	2054	has_timestamping = true;
1900	2055	else
1901		- tss->ts[2] = (struct timespec) {0};
	2056	+ tss->ts[2] = (struct timespec64) {0};
1902	2057	}
1903	2058
1904	2059	if (has_timestamping) {
1905		- tss->ts[1] = (struct timespec) {0};
1906		- put_cmsg(msg, SOL_SOCKET, SCM_TIMESTAMPING,
1907		- sizeof(*tss), tss);
	2060	+ tss->ts[1] = (struct timespec64) {0};
	2061	+ if (sock_flag(sk, SOCK_TSTAMP_NEW))
	2062	+ put_cmsg_scm_timestamping64(msg, tss);
	2063	+ else
	2064	+ put_cmsg_scm_timestamping(msg, tss);
1908	2065	}
1909	2066	}
1910	2067
..	..	@@ -1950,12 +2107,12 @@
1950	2107	long timeo;
1951	2108	struct sk_buff skb, last;
1952	2109	u32 urg_hole = 0;
1953		- struct scm_timestamping tss;
1954		- bool has_tss = false;
1955		- bool has_cmsg;
	2110	+ struct scm_timestamping_internal tss;
	2111	+ int cmsg_flags;
1956	2112
1957	2113	if (unlikely(flags & MSG_ERRQUEUE))
1958	2114	return inet_recv_error(sk, msg, len, addr_len);
	2115	+ trace_android_rvh_tcp_recvmsg(sk);
1959	2116
1960	2117	if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue) &&
1961	2118	(sk->sk_state == TCP_ESTABLISHED))
..	..	@@ -1967,7 +2124,7 @@
1967	2124	if (sk->sk_state == TCP_LISTEN)
1968	2125	goto out;
1969	2126
1970		- has_cmsg = tp->recvmsg_inq;
	2127	+ cmsg_flags = tp->recvmsg_inq ? 1 : 0;
1971	2128	timeo = sock_rcvtimeo(sk, nonblock);
1972	2129
1973	2130	/* Urgent data needs to be handled specially. */
..	..	@@ -2100,7 +2257,7 @@
2100	2257	}
2101	2258	continue;
2102	2259
2103		- found_ok_skb:
	2260	+found_ok_skb:
2104	2261	/* Ok so how much can we use? */
2105	2262	used = skb->len - offset;
2106	2263	if (len < used)
..	..	@@ -2148,8 +2305,7 @@
2148	2305
2149	2306	if (TCP_SKB_CB(skb)->has_rxtstamp) {
2150	2307	tcp_update_recv_tstamps(skb, &tss);
2151		- has_tss = true;
2152		- has_cmsg = true;
	2308	+ cmsg_flags \|= 2;
2153	2309	}
2154	2310
2155	2311	if (used + offset < skb->len)
..	..	@@ -2161,7 +2317,7 @@
2161	2317	sk_eat_skb(sk, skb);
2162	2318	continue;
2163	2319
2164		- found_fin_ok:
	2320	+found_fin_ok:
2165	2321	/* Process the FIN. */
2166	2322	WRITE_ONCE(seq, seq + 1);
2167	2323	if (!(flags & MSG_PEEK))
..	..	@@ -2169,6 +2325,7 @@
2169	2325	break;
2170	2326	} while (len > 0);
2171	2327
	2328	+ trace_android_rvh_tcp_recvmsg_stat(sk, copied);
2172	2329	/* According to UNIX98, msg_name/msg_namelen are ignored
2173	2330	* on connected socket. I was just happy when found this 8) --ANK
2174	2331	*/
..	..	@@ -2178,10 +2335,10 @@
2178	2335
2179	2336	release_sock(sk);
2180	2337
2181		- if (has_cmsg) {
2182		- if (has_tss)
	2338	+ if (cmsg_flags) {
	2339	+ if (cmsg_flags & 2)
2183	2340	tcp_recv_timestamp(msg, sk, &tss);
2184		- if (tp->recvmsg_inq) {
	2341	+ if (cmsg_flags & 1) {
2185	2342	inq = tcp_inq_hint(sk);
2186	2343	put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq);
2187	2344	}
..	..	@@ -2245,7 +2402,7 @@
2245	2402	if (inet_csk(sk)->icsk_bind_hash &&
2246	2403	!(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
2247	2404	inet_put_port(sk);
2248		- /* fall through */
	2405	+ fallthrough;
2249	2406	default:
2250	2407	if (oldstate == TCP_ESTABLISHED)
2251	2408	TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
..	..	@@ -2255,10 +2412,6 @@
2255	2412	* socket sitting in hash tables.
2256	2413	*/
2257	2414	inet_sk_state_store(sk, state);
2258		-
2259		-#ifdef STATE_TRACE
2260		- SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n", sk, statename[oldstate], statename[state]);
2261		-#endif
2262	2415	}
2263	2416	EXPORT_SYMBOL_GPL(tcp_set_state);
2264	2417
..	..	@@ -2488,7 +2641,10 @@
2488	2641	}
2489	2642
2490	2643	if (sk->sk_state == TCP_CLOSE) {
2491		- struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
	2644	+ struct request_sock *req;
	2645	+
	2646	+ req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
	2647	+ lockdep_sock_is_held(sk));
2492	2648	/* We could get here with a non-NULL req if the socket is
2493	2649	* aborted (e.g., closed with unread data) before 3WHS
2494	2650	* finishes.
..	..	@@ -2543,6 +2699,11 @@
2543	2699	sk_wmem_free_skb(sk, skb);
2544	2700	}
2545	2701	tcp_rtx_queue_purge(sk);
	2702	+ skb = sk->sk_tx_skb_cache;
	2703	+ if (skb) {
	2704	+ __kfree_skb(skb);
	2705	+ sk->sk_tx_skb_cache = NULL;
	2706	+ }
2546	2707	INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
2547	2708	sk_mem_reclaim(sk);
2548	2709	tcp_clear_all_retrans_hints(tcp_sk(sk));
..	..	@@ -2579,6 +2740,10 @@
2579	2740
2580	2741	tcp_clear_xmit_timers(sk);
2581	2742	__skb_queue_purge(&sk->sk_receive_queue);
	2743	+ if (sk->sk_rx_skb_cache) {
	2744	+ __kfree_skb(sk->sk_rx_skb_cache);
	2745	+ sk->sk_rx_skb_cache = NULL;
	2746	+ }
2582	2747	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
2583	2748	tp->urg_data = 0;
2584	2749	tcp_write_queue_purge(sk);
..	..	@@ -2593,6 +2758,7 @@
2593	2758	sk->sk_shutdown = 0;
2594	2759	sock_reset_flag(sk, SOCK_DONE);
2595	2760	tp->srtt_us = 0;
	2761	+ tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
2596	2762	tp->rcv_rtt_last_tsecr = 0;
2597	2763
2598	2764	seq = tp->write_seq + tp->max_window + 2;
..	..	@@ -2600,16 +2766,24 @@
2600	2766	seq = 1;
2601	2767	WRITE_ONCE(tp->write_seq, seq);
2602	2768
2603		- tp->snd_cwnd = 2;
	2769	+ icsk->icsk_backoff = 0;
2604	2770	icsk->icsk_probes_out = 0;
	2771	+ icsk->icsk_probes_tstamp = 0;
	2772	+ icsk->icsk_rto = TCP_TIMEOUT_INIT;
	2773	+ icsk->icsk_rto_min = TCP_RTO_MIN;
	2774	+ icsk->icsk_delack_max = TCP_DELACK_MAX;
2605	2775	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
	2776	+ tp->snd_cwnd = TCP_INIT_CWND;
2606	2777	tp->snd_cwnd_cnt = 0;
	2778	+ tp->is_cwnd_limited = 0;
	2779	+ tp->max_packets_out = 0;
2607	2780	tp->window_clamp = 0;
2608	2781	tp->delivered = 0;
2609	2782	tp->delivered_ce = 0;
2610	2783	if (icsk->icsk_ca_ops->release)
2611	2784	icsk->icsk_ca_ops->release(sk);
2612	2785	memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
	2786	+ icsk->icsk_ca_initialized = 0;
2613	2787	tcp_set_ca_state(sk, TCP_CA_Open);
2614	2788	tp->is_sack_reneg = 0;
2615	2789	tcp_clear_retrans(tp);
..	..	@@ -2621,8 +2795,7 @@
2621	2795	icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
2622	2796	memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2623	2797	__sk_dst_reset(sk);
2624		- dst_release(sk->sk_rx_dst);
2625		- sk->sk_rx_dst = NULL;
	2798	+ dst_release(xchg((__force struct dst_entry **)&sk->sk_rx_dst, NULL));
2626	2799	tcp_saved_syn_free(tp);
2627	2800	tp->compressed_ack = 0;
2628	2801	tp->segs_in = 0;
..	..	@@ -2633,12 +2806,33 @@
2633	2806	tp->bytes_retrans = 0;
2634	2807	tp->data_segs_in = 0;
2635	2808	tp->data_segs_out = 0;
	2809	+ tp->duplicate_sack[0].start_seq = 0;
	2810	+ tp->duplicate_sack[0].end_seq = 0;
2636	2811	tp->dsack_dups = 0;
2637	2812	tp->reord_seen = 0;
	2813	+ tp->retrans_out = 0;
	2814	+ tp->sacked_out = 0;
	2815	+ tp->tlp_high_seq = 0;
	2816	+ tp->last_oow_ack_time = 0;
	2817	+ /* There's a bubble in the pipe until at least the first ACK. */
	2818	+ tp->app_limited = ~0U;
	2819	+ tp->rack.mstamp = 0;
	2820	+ tp->rack.advanced = 0;
	2821	+ tp->rack.reo_wnd_steps = 1;
	2822	+ tp->rack.last_delivered = 0;
	2823	+ tp->rack.reo_wnd_persist = 0;
	2824	+ tp->rack.dsack_seen = 0;
	2825	+ tp->syn_data_acked = 0;
	2826	+ tp->rx_opt.saw_tstamp = 0;
	2827	+ tp->rx_opt.dsack = 0;
	2828	+ tp->rx_opt.num_sacks = 0;
	2829	+ tp->rcv_ooopack = 0;
	2830	+
2638	2831
2639	2832	/* Clean up fastopen related fields */
2640	2833	tcp_free_fastopen_req(tp);
2641	2834	inet->defer_connect = 0;
	2835	+ tp->fastopen_client_fail = 0;
2642	2836
2643	2837	WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2644	2838
..	..	@@ -2659,7 +2853,7 @@
2659	2853	(sk->sk_state != TCP_LISTEN);
2660	2854	}
2661	2855
2662		-static int tcp_repair_set_window(struct tcp_sock tp, char __user optbuf, int len)
	2856	+static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)
2663	2857	{
2664	2858	struct tcp_repair_window opt;
2665	2859
..	..	@@ -2669,7 +2863,7 @@
2669	2863	if (len != sizeof(opt))
2670	2864	return -EINVAL;
2671	2865
2672		- if (copy_from_user(&opt, optbuf, sizeof(opt)))
	2866	+ if (copy_from_sockptr(&opt, optbuf, sizeof(opt)))
2673	2867	return -EFAULT;
2674	2868
2675	2869	if (opt.max_window < opt.snd_wnd)
..	..	@@ -2691,17 +2885,18 @@
2691	2885	return 0;
2692	2886	}
2693	2887
2694		-static int tcp_repair_options_est(struct sock *sk,
2695		- struct tcp_repair_opt __user *optbuf, unsigned int len)
	2888	+static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf,
	2889	+ unsigned int len)
2696	2890	{
2697	2891	struct tcp_sock *tp = tcp_sk(sk);
2698	2892	struct tcp_repair_opt opt;
	2893	+ size_t offset = 0;
2699	2894
2700	2895	while (len >= sizeof(opt)) {
2701		- if (copy_from_user(&opt, optbuf, sizeof(opt)))
	2896	+ if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt)))
2702	2897	return -EFAULT;
2703	2898
2704		- optbuf++;
	2899	+ offset += sizeof(opt);
2705	2900	len -= sizeof(opt);
2706	2901
2707	2902	switch (opt.opt_code) {
..	..	@@ -2740,11 +2935,183 @@
2740	2935	return 0;
2741	2936	}
2742	2937
	2938	+DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
	2939	+EXPORT_SYMBOL(tcp_tx_delay_enabled);
	2940	+
	2941	+static void tcp_enable_tx_delay(void)
	2942	+{
	2943	+ if (!static_branch_unlikely(&tcp_tx_delay_enabled)) {
	2944	+ static int __tcp_tx_delay_enabled = 0;
	2945	+
	2946	+ if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
	2947	+ static_branch_enable(&tcp_tx_delay_enabled);
	2948	+ pr_info("TCP_TX_DELAY enabled\n");
	2949	+ }
	2950	+ }
	2951	+}
	2952	+
	2953	+/* When set indicates to always queue non-full frames. Later the user clears
	2954	+ * this option and we transmit any pending partial frames in the queue. This is
	2955	+ * meant to be used alongside sendfile() to get properly filled frames when the
	2956	+ * user (for example) must write out headers with a write() call first and then
	2957	+ * use sendfile to send out the data parts.
	2958	+ *
	2959	+ * TCP_CORK can be set together with TCP_NODELAY and it is stronger than
	2960	+ * TCP_NODELAY.
	2961	+ */
	2962	+static void __tcp_sock_set_cork(struct sock *sk, bool on)
	2963	+{
	2964	+ struct tcp_sock *tp = tcp_sk(sk);
	2965	+
	2966	+ if (on) {
	2967	+ tp->nonagle \|= TCP_NAGLE_CORK;
	2968	+ } else {
	2969	+ tp->nonagle &= ~TCP_NAGLE_CORK;
	2970	+ if (tp->nonagle & TCP_NAGLE_OFF)
	2971	+ tp->nonagle \|= TCP_NAGLE_PUSH;
	2972	+ tcp_push_pending_frames(sk);
	2973	+ }
	2974	+}
	2975	+
	2976	+void tcp_sock_set_cork(struct sock *sk, bool on)
	2977	+{
	2978	+ lock_sock(sk);
	2979	+ __tcp_sock_set_cork(sk, on);
	2980	+ release_sock(sk);
	2981	+}
	2982	+EXPORT_SYMBOL(tcp_sock_set_cork);
	2983	+
	2984	+/* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is
	2985	+ * remembered, but it is not activated until cork is cleared.
	2986	+ *
	2987	+ * However, when TCP_NODELAY is set we make an explicit push, which overrides
	2988	+ * even TCP_CORK for currently queued segments.
	2989	+ */
	2990	+static void __tcp_sock_set_nodelay(struct sock *sk, bool on)
	2991	+{
	2992	+ if (on) {
	2993	+ tcp_sk(sk)->nonagle \|= TCP_NAGLE_OFF\|TCP_NAGLE_PUSH;
	2994	+ tcp_push_pending_frames(sk);
	2995	+ } else {
	2996	+ tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF;
	2997	+ }
	2998	+}
	2999	+
	3000	+void tcp_sock_set_nodelay(struct sock *sk)
	3001	+{
	3002	+ lock_sock(sk);
	3003	+ __tcp_sock_set_nodelay(sk, true);
	3004	+ release_sock(sk);
	3005	+}
	3006	+EXPORT_SYMBOL(tcp_sock_set_nodelay);
	3007	+
	3008	+static void __tcp_sock_set_quickack(struct sock *sk, int val)
	3009	+{
	3010	+ if (!val) {
	3011	+ inet_csk_enter_pingpong_mode(sk);
	3012	+ return;
	3013	+ }
	3014	+
	3015	+ inet_csk_exit_pingpong_mode(sk);
	3016	+ if ((1 << sk->sk_state) & (TCPF_ESTABLISHED \| TCPF_CLOSE_WAIT) &&
	3017	+ inet_csk_ack_scheduled(sk)) {
	3018	+ inet_csk(sk)->icsk_ack.pending \|= ICSK_ACK_PUSHED;
	3019	+ tcp_cleanup_rbuf(sk, 1);
	3020	+ if (!(val & 1))
	3021	+ inet_csk_enter_pingpong_mode(sk);
	3022	+ }
	3023	+}
	3024	+
	3025	+void tcp_sock_set_quickack(struct sock *sk, int val)
	3026	+{
	3027	+ lock_sock(sk);
	3028	+ __tcp_sock_set_quickack(sk, val);
	3029	+ release_sock(sk);
	3030	+}
	3031	+EXPORT_SYMBOL(tcp_sock_set_quickack);
	3032	+
	3033	+int tcp_sock_set_syncnt(struct sock *sk, int val)
	3034	+{
	3035	+ if (val < 1 \|\| val > MAX_TCP_SYNCNT)
	3036	+ return -EINVAL;
	3037	+
	3038	+ lock_sock(sk);
	3039	+ inet_csk(sk)->icsk_syn_retries = val;
	3040	+ release_sock(sk);
	3041	+ return 0;
	3042	+}
	3043	+EXPORT_SYMBOL(tcp_sock_set_syncnt);
	3044	+
	3045	+void tcp_sock_set_user_timeout(struct sock *sk, u32 val)
	3046	+{
	3047	+ lock_sock(sk);
	3048	+ inet_csk(sk)->icsk_user_timeout = val;
	3049	+ release_sock(sk);
	3050	+}
	3051	+EXPORT_SYMBOL(tcp_sock_set_user_timeout);
	3052	+
	3053	+int tcp_sock_set_keepidle_locked(struct sock *sk, int val)
	3054	+{
	3055	+ struct tcp_sock *tp = tcp_sk(sk);
	3056	+
	3057	+ if (val < 1 \|\| val > MAX_TCP_KEEPIDLE)
	3058	+ return -EINVAL;
	3059	+
	3060	+ tp->keepalive_time = val * HZ;
	3061	+ if (sock_flag(sk, SOCK_KEEPOPEN) &&
	3062	+ !((1 << sk->sk_state) & (TCPF_CLOSE \| TCPF_LISTEN))) {
	3063	+ u32 elapsed = keepalive_time_elapsed(tp);
	3064	+
	3065	+ if (tp->keepalive_time > elapsed)
	3066	+ elapsed = tp->keepalive_time - elapsed;
	3067	+ else
	3068	+ elapsed = 0;
	3069	+ inet_csk_reset_keepalive_timer(sk, elapsed);
	3070	+ }
	3071	+
	3072	+ return 0;
	3073	+}
	3074	+
	3075	+int tcp_sock_set_keepidle(struct sock *sk, int val)
	3076	+{
	3077	+ int err;
	3078	+
	3079	+ lock_sock(sk);
	3080	+ err = tcp_sock_set_keepidle_locked(sk, val);
	3081	+ release_sock(sk);
	3082	+ return err;
	3083	+}
	3084	+EXPORT_SYMBOL(tcp_sock_set_keepidle);
	3085	+
	3086	+int tcp_sock_set_keepintvl(struct sock *sk, int val)
	3087	+{
	3088	+ if (val < 1 \|\| val > MAX_TCP_KEEPINTVL)
	3089	+ return -EINVAL;
	3090	+
	3091	+ lock_sock(sk);
	3092	+ tcp_sk(sk)->keepalive_intvl = val * HZ;
	3093	+ release_sock(sk);
	3094	+ return 0;
	3095	+}
	3096	+EXPORT_SYMBOL(tcp_sock_set_keepintvl);
	3097	+
	3098	+int tcp_sock_set_keepcnt(struct sock *sk, int val)
	3099	+{
	3100	+ if (val < 1 \|\| val > MAX_TCP_KEEPCNT)
	3101	+ return -EINVAL;
	3102	+
	3103	+ lock_sock(sk);
	3104	+ tcp_sk(sk)->keepalive_probes = val;
	3105	+ release_sock(sk);
	3106	+ return 0;
	3107	+}
	3108	+EXPORT_SYMBOL(tcp_sock_set_keepcnt);
	3109	+
2743	3110	/*
2744	3111	* Socket option code for TCP.
2745	3112	*/
2746		-static int do_tcp_setsockopt(struct sock *sk, int level,
2747		- int optname, char __user *optval, unsigned int optlen)
	3113	+static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
	3114	+ sockptr_t optval, unsigned int optlen)
2748	3115	{
2749	3116	struct tcp_sock *tp = tcp_sk(sk);
2750	3117	struct inet_connection_sock *icsk = inet_csk(sk);
..	..	@@ -2760,14 +3127,14 @@
2760	3127	if (optlen < 1)
2761	3128	return -EINVAL;
2762	3129
2763		- val = strncpy_from_user(name, optval,
	3130	+ val = strncpy_from_sockptr(name, optval,
2764	3131	min_t(long, TCP_CA_NAME_MAX-1, optlen));
2765	3132	if (val < 0)
2766	3133	return -EFAULT;
2767	3134	name[val] = 0;
2768	3135
2769	3136	lock_sock(sk);
2770		- err = tcp_set_congestion_control(sk, name, true, true,
	3137	+ err = tcp_set_congestion_control(sk, name, true,
2771	3138	ns_capable(sock_net(sk)->user_ns,
2772	3139	CAP_NET_ADMIN));
2773	3140	release_sock(sk);
..	..	@@ -2779,7 +3146,7 @@
2779	3146	if (optlen < 1)
2780	3147	return -EINVAL;
2781	3148
2782		- val = strncpy_from_user(name, optval,
	3149	+ val = strncpy_from_sockptr(name, optval,
2783	3150	min_t(long, TCP_ULP_NAME_MAX - 1,
2784	3151	optlen));
2785	3152	if (val < 0)
..	..	@@ -2792,15 +3159,23 @@
2792	3159	return err;
2793	3160	}
2794	3161	case TCP_FASTOPEN_KEY: {
2795		- __u8 key[TCP_FASTOPEN_KEY_LENGTH];
	3162	+ __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
	3163	+ __u8 *backup_key = NULL;
2796	3164
2797		- if (optlen != sizeof(key))
	3165	+ /* Allow a backup key as well to facilitate key rotation
	3166	+ * First key is the active one.
	3167	+ */
	3168	+ if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
	3169	+ optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
2798	3170	return -EINVAL;
2799	3171
2800		- if (copy_from_user(key, optval, optlen))
	3172	+ if (copy_from_sockptr(key, optval, optlen))
2801	3173	return -EFAULT;
2802	3174
2803		- return tcp_fastopen_reset_cipher(net, sk, key, sizeof(key));
	3175	+ if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
	3176	+ backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
	3177	+
	3178	+ return tcp_fastopen_reset_cipher(net, sk, key, backup_key);
2804	3179	}
2805	3180	default:
2806	3181	/* fallthru */
..	..	@@ -2810,7 +3185,7 @@
2810	3185	if (optlen < sizeof(int))
2811	3186	return -EINVAL;
2812	3187
2813		- if (get_user(val, (int __user *)optval))
	3188	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
2814	3189	return -EFAULT;
2815	3190
2816	3191	lock_sock(sk);
..	..	@@ -2829,20 +3204,7 @@
2829	3204	break;
2830	3205
2831	3206	case TCP_NODELAY:
2832		- if (val) {
2833		- /* TCP_NODELAY is weaker than TCP_CORK, so that
2834		- * this option on corked socket is remembered, but
2835		- * it is not activated until cork is cleared.
2836		- *
2837		- * However, when TCP_NODELAY is set we make
2838		- * an explicit push, which overrides even TCP_CORK
2839		- * for currently queued segments.
2840		- */
2841		- tp->nonagle \|= TCP_NAGLE_OFF\|TCP_NAGLE_PUSH;
2842		- tcp_push_pending_frames(sk);
2843		- } else {
2844		- tp->nonagle &= ~TCP_NAGLE_OFF;
2845		- }
	3207	+ __tcp_sock_set_nodelay(sk, val);
2846	3208	break;
2847	3209
2848	3210	case TCP_THIN_LINEAR_TIMEOUTS:
..	..	@@ -2908,52 +3270,18 @@
2908	3270	case TCP_REPAIR_OPTIONS:
2909	3271	if (!tp->repair)
2910	3272	err = -EINVAL;
2911		- else if (sk->sk_state == TCP_ESTABLISHED)
2912		- err = tcp_repair_options_est(sk,
2913		- (struct tcp_repair_opt __user *)optval,
2914		- optlen);
	3273	+ else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent)
	3274	+ err = tcp_repair_options_est(sk, optval, optlen);
2915	3275	else
2916	3276	err = -EPERM;
2917	3277	break;
2918	3278
2919	3279	case TCP_CORK:
2920		- /* When set indicates to always queue non-full frames.
2921		- * Later the user clears this option and we transmit
2922		- * any pending partial frames in the queue. This is
2923		- * meant to be used alongside sendfile() to get properly
2924		- * filled frames when the user (for example) must write
2925		- * out headers with a write() call first and then use
2926		- * sendfile to send out the data parts.
2927		- *
2928		- * TCP_CORK can be set together with TCP_NODELAY and it is
2929		- * stronger than TCP_NODELAY.
2930		- */
2931		- if (val) {
2932		- tp->nonagle \|= TCP_NAGLE_CORK;
2933		- } else {
2934		- tp->nonagle &= ~TCP_NAGLE_CORK;
2935		- if (tp->nonagle&TCP_NAGLE_OFF)
2936		- tp->nonagle \|= TCP_NAGLE_PUSH;
2937		- tcp_push_pending_frames(sk);
2938		- }
	3280	+ __tcp_sock_set_cork(sk, val);
2939	3281	break;
2940	3282
2941	3283	case TCP_KEEPIDLE:
2942		- if (val < 1 \|\| val > MAX_TCP_KEEPIDLE)
2943		- err = -EINVAL;
2944		- else {
2945		- tp->keepalive_time = val * HZ;
2946		- if (sock_flag(sk, SOCK_KEEPOPEN) &&
2947		- !((1 << sk->sk_state) &
2948		- (TCPF_CLOSE \| TCPF_LISTEN))) {
2949		- u32 elapsed = keepalive_time_elapsed(tp);
2950		- if (tp->keepalive_time > elapsed)
2951		- elapsed = tp->keepalive_time - elapsed;
2952		- else
2953		- elapsed = 0;
2954		- inet_csk_reset_keepalive_timer(sk, elapsed);
2955		- }
2956		- }
	3284	+ err = tcp_sock_set_keepidle_locked(sk, val);
2957	3285	break;
2958	3286	case TCP_KEEPINTVL:
2959	3287	if (val < 1 \|\| val > MAX_TCP_KEEPINTVL)
..	..	@@ -2975,7 +3303,8 @@
2975	3303	break;
2976	3304
2977	3305	case TCP_SAVE_SYN:
2978		- if (val < 0 \|\| val > 1)
	3306	+ /* 0: disable, 1: enable, 2: start from ether_header */
	3307	+ if (val < 0 \|\| val > 2)
2979	3308	err = -EINVAL;
2980	3309	else
2981	3310	tp->save_syn = val;
..	..	@@ -2984,8 +3313,8 @@
2984	3313	case TCP_LINGER2:
2985	3314	if (val < 0)
2986	3315	tp->linger2 = -1;
2987		- else if (val > net->ipv4.sysctl_tcp_fin_timeout / HZ)
2988		- tp->linger2 = 0;
	3316	+ else if (val > TCP_FIN_TIMEOUT_MAX / HZ)
	3317	+ tp->linger2 = TCP_FIN_TIMEOUT_MAX;
2989	3318	else
2990	3319	tp->linger2 = val * HZ;
2991	3320	break;
..	..	@@ -3010,19 +3339,7 @@
3010	3339	break;
3011	3340
3012	3341	case TCP_QUICKACK:
3013		- if (!val) {
3014		- icsk->icsk_ack.pingpong = 1;
3015		- } else {
3016		- icsk->icsk_ack.pingpong = 0;
3017		- if ((1 << sk->sk_state) &
3018		- (TCPF_ESTABLISHED \| TCPF_CLOSE_WAIT) &&
3019		- inet_csk_ack_scheduled(sk)) {
3020		- icsk->icsk_ack.pending \|= ICSK_ACK_PUSHED;
3021		- tcp_cleanup_rbuf(sk, 1);
3022		- if (!(val & 1))
3023		- icsk->icsk_ack.pingpong = 1;
3024		- }
3025		- }
	3342	+ __tcp_sock_set_quickack(sk, val);
3026	3343	break;
3027	3344
3028	3345	#ifdef CONFIG_TCP_MD5SIG
..	..	@@ -3054,7 +3371,8 @@
3054	3371	case TCP_FASTOPEN_CONNECT:
3055	3372	if (val > 1 \|\| val < 0) {
3056	3373	err = -EINVAL;
3057		- } else if (net->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) {
	3374	+ } else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) &
	3375	+ TFO_CLIENT_ENABLE) {
3058	3376	if (sk->sk_state == TCP_CLOSE)
3059	3377	tp->fastopen_connect = val;
3060	3378	else
..	..	@@ -3090,6 +3408,11 @@
3090	3408	else
3091	3409	tp->recvmsg_inq = val;
3092	3410	break;
	3411	+ case TCP_TX_DELAY:
	3412	+ if (val)
	3413	+ tcp_enable_tx_delay();
	3414	+ tp->tcp_tx_delay = val;
	3415	+ break;
3093	3416	default:
3094	3417	err = -ENOPROTOOPT;
3095	3418	break;
..	..	@@ -3099,7 +3422,7 @@
3099	3422	return err;
3100	3423	}
3101	3424
3102		-int tcp_setsockopt(struct sock sk, int level, int optname, char __user optval,
	3425	+int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
3103	3426	unsigned int optlen)
3104	3427	{
3105	3428	const struct inet_connection_sock *icsk = inet_csk(sk);
..	..	@@ -3110,18 +3433,6 @@
3110	3433	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
3111	3434	}
3112	3435	EXPORT_SYMBOL(tcp_setsockopt);
3113		-
3114		-#ifdef CONFIG_COMPAT
3115		-int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
3116		- char __user *optval, unsigned int optlen)
3117		-{
3118		- if (level != SOL_TCP)
3119		- return inet_csk_compat_setsockopt(sk, level, optname,
3120		- optval, optlen);
3121		- return do_tcp_setsockopt(sk, level, optname, optval, optlen);
3122		-}
3123		-EXPORT_SYMBOL(compat_tcp_setsockopt);
3124		-#endif
3125	3436
3126	3437	static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
3127	3438	struct tcp_info *info)
..	..	@@ -3147,10 +3458,10 @@
3147	3458	{
3148	3459	const struct tcp_sock tp = tcp_sk(sk); / iff sk_type == SOCK_STREAM */
3149	3460	const struct inet_connection_sock *icsk = inet_csk(sk);
	3461	+ unsigned long rate;
3150	3462	u32 now;
3151	3463	u64 rate64;
3152	3464	bool slow;
3153		- u32 rate;
3154	3465
3155	3466	memset(info, 0, sizeof(*info));
3156	3467	if (sk->sk_type != SOCK_STREAM)
..	..	@@ -3160,11 +3471,11 @@
3160	3471
3161	3472	/* Report meaningful fields for all TCP states, including listeners */
3162	3473	rate = READ_ONCE(sk->sk_pacing_rate);
3163		- rate64 = rate != ~0U ? rate : ~0ULL;
	3474	+ rate64 = (rate != ~0UL) ? rate : ~0ULL;
3164	3475	info->tcpi_pacing_rate = rate64;
3165	3476
3166	3477	rate = READ_ONCE(sk->sk_max_pacing_rate);
3167		- rate64 = rate != ~0U ? rate : ~0ULL;
	3478	+ rate64 = (rate != ~0UL) ? rate : ~0ULL;
3168	3479	info->tcpi_max_pacing_rate = rate64;
3169	3480
3170	3481	info->tcpi_reordering = tp->reordering;
..	..	@@ -3175,8 +3486,8 @@
3175	3486	* tcpi_unacked -> Number of children ready for accept()
3176	3487	* tcpi_sacked -> max backlog
3177	3488	*/
3178		- info->tcpi_unacked = sk->sk_ack_backlog;
3179		- info->tcpi_sacked = sk->sk_max_ack_backlog;
	3489	+ info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog);
	3490	+ info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog);
3180	3491	return;
3181	3492	}
3182	3493
..	..	@@ -3254,6 +3565,9 @@
3254	3565	info->tcpi_bytes_retrans = tp->bytes_retrans;
3255	3566	info->tcpi_dsack_dups = tp->dsack_dups;
3256	3567	info->tcpi_reord_seen = tp->reord_seen;
	3568	+ info->tcpi_rcv_ooopack = tp->rcv_ooopack;
	3569	+ info->tcpi_snd_wnd = tp->snd_wnd;
	3570	+ info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;
3257	3571	unlock_sock_fast(sk, slow);
3258	3572	}
3259	3573	EXPORT_SYMBOL_GPL(tcp_get_info);
..	..	@@ -3282,16 +3596,21 @@
3282	3596	nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */
3283	3597	nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */
3284	3598	nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */
	3599	+ nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */
	3600	+ nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */
	3601	+ nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */
	3602	+ nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */
3285	3603	0;
3286	3604	}
3287	3605
3288		-struct sk_buff tcp_get_timestamping_opt_stats(const struct sock sk)
	3606	+struct sk_buff tcp_get_timestamping_opt_stats(const struct sock sk,
	3607	+ const struct sk_buff *orig_skb)
3289	3608	{
3290	3609	const struct tcp_sock *tp = tcp_sk(sk);
3291	3610	struct sk_buff *stats;
3292	3611	struct tcp_info info;
	3612	+ unsigned long rate;
3293	3613	u64 rate64;
3294		- u32 rate;
3295	3614
3296	3615	stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);
3297	3616	if (!stats)
..	..	@@ -3310,7 +3629,7 @@
3310	3629	tp->total_retrans, TCP_NLA_PAD);
3311	3630
3312	3631	rate = READ_ONCE(sk->sk_pacing_rate);
3313		- rate64 = rate != ~0U ? rate : ~0ULL;
	3632	+ rate64 = (rate != ~0UL) ? rate : ~0ULL;
3314	3633	nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);
3315	3634
3316	3635	rate64 = tcp_compute_delivery_rate(tp);
..	..	@@ -3335,6 +3654,12 @@
3335	3654	TCP_NLA_PAD);
3336	3655	nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups);
3337	3656	nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen);
	3657	+ nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3);
	3658	+ nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash);
	3659	+ nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT,
	3660	+ max_t(int, 0, tp->write_seq - tp->snd_nxt));
	3661	+ nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns,
	3662	+ TCP_NLA_PAD);
3338	3663
3339	3664	return stats;
3340	3665	}
..	..	@@ -3384,7 +3709,7 @@
3384	3709	case TCP_LINGER2:
3385	3710	val = tp->linger2;
3386	3711	if (val >= 0)
3387		- val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ;
	3712	+ val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ;
3388	3713	break;
3389	3714	case TCP_DEFER_ACCEPT:
3390	3715	val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
..	..	@@ -3429,7 +3754,7 @@
3429	3754	return 0;
3430	3755	}
3431	3756	case TCP_QUICKACK:
3432		- val = !icsk->icsk_ack.pingpong;
	3757	+ val = !inet_csk_in_pingpong_mode(sk);
3433	3758	break;
3434	3759
3435	3760	case TCP_CONGESTION:
..	..	@@ -3458,21 +3783,15 @@
3458	3783	return 0;
3459	3784
3460	3785	case TCP_FASTOPEN_KEY: {
3461		- __u8 key[TCP_FASTOPEN_KEY_LENGTH];
3462		- struct tcp_fastopen_context *ctx;
	3786	+ u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
	3787	+ unsigned int key_len;
3463	3788
3464	3789	if (get_user(len, optlen))
3465	3790	return -EFAULT;
3466	3791
3467		- rcu_read_lock();
3468		- ctx = rcu_dereference(icsk->icsk_accept_queue.fastopenq.ctx);
3469		- if (ctx)
3470		- memcpy(key, ctx->key, sizeof(key));
3471		- else
3472		- len = 0;
3473		- rcu_read_unlock();
3474		-
3475		- len = min_t(unsigned int, len, sizeof(key));
	3792	+ key_len = tcp_fastopen_get_cipher(net, icsk, key) *
	3793	+ TCP_FASTOPEN_KEY_LENGTH;
	3794	+ len = min_t(unsigned int, len, key_len);
3476	3795	if (put_user(len, optlen))
3477	3796	return -EFAULT;
3478	3797	if (copy_to_user(optval, key, len))
..	..	@@ -3545,6 +3864,10 @@
3545	3864	val = tp->fastopen_no_cookie;
3546	3865	break;
3547	3866
	3867	+ case TCP_TX_DELAY:
	3868	+ val = tp->tcp_tx_delay;
	3869	+ break;
	3870	+
3548	3871	case TCP_TIMESTAMP:
3549	3872	val = tcp_time_stamp_raw() + tp->tsoffset;
3550	3873	break;
..	..	@@ -3563,20 +3886,21 @@
3563	3886
3564	3887	lock_sock(sk);
3565	3888	if (tp->saved_syn) {
3566		- if (len < tp->saved_syn[0]) {
3567		- if (put_user(tp->saved_syn[0], optlen)) {
	3889	+ if (len < tcp_saved_syn_len(tp->saved_syn)) {
	3890	+ if (put_user(tcp_saved_syn_len(tp->saved_syn),
	3891	+ optlen)) {
3568	3892	release_sock(sk);
3569	3893	return -EFAULT;
3570	3894	}
3571	3895	release_sock(sk);
3572	3896	return -EINVAL;
3573	3897	}
3574		- len = tp->saved_syn[0];
	3898	+ len = tcp_saved_syn_len(tp->saved_syn);
3575	3899	if (put_user(len, optlen)) {
3576	3900	release_sock(sk);
3577	3901	return -EFAULT;
3578	3902	}
3579		- if (copy_to_user(optval, tp->saved_syn + 1, len)) {
	3903	+ if (copy_to_user(optval, tp->saved_syn->data, len)) {
3580	3904	release_sock(sk);
3581	3905	return -EFAULT;
3582	3906	}
..	..	@@ -3592,18 +3916,41 @@
3592	3916	}
3593	3917	#ifdef CONFIG_MMU
3594	3918	case TCP_ZEROCOPY_RECEIVE: {
3595		- struct tcp_zerocopy_receive zc;
	3919	+ struct tcp_zerocopy_receive zc = {};
3596	3920	int err;
3597	3921
3598	3922	if (get_user(len, optlen))
3599	3923	return -EFAULT;
3600		- if (len != sizeof(zc))
	3924	+ if (len < 0 \|\|
	3925	+ len < offsetofend(struct tcp_zerocopy_receive, length))
3601	3926	return -EINVAL;
	3927	+ if (len > sizeof(zc)) {
	3928	+ len = sizeof(zc);
	3929	+ if (put_user(len, optlen))
	3930	+ return -EFAULT;
	3931	+ }
3602	3932	if (copy_from_user(&zc, optval, len))
3603	3933	return -EFAULT;
3604	3934	lock_sock(sk);
3605	3935	err = tcp_zerocopy_receive(sk, &zc);
3606	3936	release_sock(sk);
	3937	+ if (len >= offsetofend(struct tcp_zerocopy_receive, err))
	3938	+ goto zerocopy_rcv_sk_err;
	3939	+ switch (len) {
	3940	+ case offsetofend(struct tcp_zerocopy_receive, err):
	3941	+ goto zerocopy_rcv_sk_err;
	3942	+ case offsetofend(struct tcp_zerocopy_receive, inq):
	3943	+ goto zerocopy_rcv_inq;
	3944	+ case offsetofend(struct tcp_zerocopy_receive, length):
	3945	+ default:
	3946	+ goto zerocopy_rcv_out;
	3947	+ }
	3948	+zerocopy_rcv_sk_err:
	3949	+ if (!err)
	3950	+ zc.err = sock_error(sk);
	3951	+zerocopy_rcv_inq:
	3952	+ zc.inq = tcp_inq_hint(sk);
	3953	+zerocopy_rcv_out:
3607	3954	if (!err && copy_to_user(optval, &zc, len))
3608	3955	err = -EFAULT;
3609	3956	return err;
..	..	@@ -3631,18 +3978,6 @@
3631	3978	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
3632	3979	}
3633	3980	EXPORT_SYMBOL(tcp_getsockopt);
3634		-
3635		-#ifdef CONFIG_COMPAT
3636		-int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
3637		- char __user optval, int __user optlen)
3638		-{
3639		- if (level != SOL_TCP)
3640		- return inet_csk_compat_getsockopt(sk, level, optname,
3641		- optval, optlen);
3642		- return do_tcp_getsockopt(sk, level, optname, optval, optlen);
3643		-}
3644		-EXPORT_SYMBOL(compat_tcp_getsockopt);
3645		-#endif
3646	3981
3647	3982	#ifdef CONFIG_TCP_MD5SIG
3648	3983	static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool);
..	..	@@ -3686,20 +4021,28 @@
3686	4021	* to memory. See smp_rmb() in tcp_get_md5sig_pool()
3687	4022	*/
3688	4023	smp_wmb();
3689		- tcp_md5sig_pool_populated = true;
	4024	+ /* Paired with READ_ONCE() from tcp_alloc_md5sig_pool()
	4025	+ * and tcp_get_md5sig_pool().
	4026	+ */
	4027	+ WRITE_ONCE(tcp_md5sig_pool_populated, true);
3690	4028	}
3691	4029
3692	4030	bool tcp_alloc_md5sig_pool(void)
3693	4031	{
3694		- if (unlikely(!tcp_md5sig_pool_populated)) {
	4032	+ /* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */
	4033	+ if (unlikely(!READ_ONCE(tcp_md5sig_pool_populated))) {
3695	4034	mutex_lock(&tcp_md5sig_mutex);
3696	4035
3697		- if (!tcp_md5sig_pool_populated)
	4036	+ if (!tcp_md5sig_pool_populated) {
3698	4037	__tcp_alloc_md5sig_pool();
	4038	+ if (tcp_md5sig_pool_populated)
	4039	+ static_branch_inc(&tcp_md5_needed);
	4040	+ }
3699	4041
3700	4042	mutex_unlock(&tcp_md5sig_mutex);
3701	4043	}
3702		- return tcp_md5sig_pool_populated;
	4044	+ /* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */
	4045	+ return READ_ONCE(tcp_md5sig_pool_populated);
3703	4046	}
3704	4047	EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
3705	4048
..	..	@@ -3715,7 +4058,8 @@
3715	4058	{
3716	4059	local_bh_disable();
3717	4060
3718		- if (tcp_md5sig_pool_populated) {
	4061	+ /* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */
	4062	+ if (READ_ONCE(tcp_md5sig_pool_populated)) {
3719	4063	/* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */
3720	4064	smp_rmb();
3721	4065	return this_cpu_ptr(&tcp_md5sig_pool);
..	..	@@ -3745,8 +4089,8 @@
3745	4089	return 1;
3746	4090
3747	4091	for (i = 0; i < shi->nr_frags; ++i) {
3748		- const struct skb_frag_struct *f = &shi->frags[i];
3749		- unsigned int offset = f->page_offset;
	4092	+ const skb_frag_t *f = &shi->frags[i];
	4093	+ unsigned int offset = skb_frag_off(f);
3750	4094	struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3751	4095
3752	4096	sg_set_page(&sg, page, skb_frag_size(f),
..	..	@@ -3772,8 +4116,8 @@
3772	4116	sg_init_one(&sg, key->key, keylen);
3773	4117	ahash_request_set_crypt(hp->md5_req, &sg, NULL, keylen);
3774	4118
3775		- /* tcp_md5_do_add() might change key->key under us */
3776		- return crypto_ahash_update(hp->md5_req);
	4119	+ /* We use data_race() because tcp_md5_do_add() might change key->key under us */
	4120	+ return data_race(crypto_ahash_update(hp->md5_req));
3777	4121	}
3778	4122	EXPORT_SYMBOL(tcp_md5_hash_key);
3779	4123
..	..	@@ -3781,7 +4125,13 @@
3781	4125
3782	4126	void tcp_done(struct sock *sk)
3783	4127	{
3784		- struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
	4128	+ struct request_sock *req;
	4129	+
	4130	+ /* We might be called with a new socket, after
	4131	+ * inet_csk_prepare_forced_close() has been called
	4132	+ * so we can not use lockdep_sock_is_held(sk)
	4133	+ */
	4134	+ req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1);
3785	4135
3786	4136	if (sk->sk_state == TCP_SYN_SENT \|\| sk->sk_state == TCP_SYN_RECV)
3787	4137	TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
..	..	@@ -3880,7 +4230,7 @@
3880	4230
3881	4231	BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
3882	4232	BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
3883		- FIELD_SIZEOF(struct sk_buff, cb));
	4233	+ sizeof_field(struct sk_buff, cb));
3884	4234
3885	4235	percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
3886	4236	percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
..	..	@@ -3954,4 +4304,5 @@
3954	4304	tcp_metrics_init();
3955	4305	BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
3956	4306	tcp_tasklet_init();
	4307	+ mptcp_init();
3957	4308	}