~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* INET An implementation of the TCP/IP protocol suite for the LINUX
3	4	* operating system. INET is implemented using the BSD Socket
..	..	@@ -43,13 +44,6 @@
43	44	* Chetan Loke : Implemented TPACKET_V3 block abstraction
44	45	* layer.
45	46	* Copyright (C) 2011, <lokec@ccs.neu.edu>
46		- *
47		- *
48		- * This program is free software; you can redistribute it and/or
49		- * modify it under the terms of the GNU General Public License
50		- * as published by the Free Software Foundation; either version
51		- * 2 of the License, or (at your option) any later version.
52		- *
53	47	*/
54	48
55	49	#include <linux/types.h>
..	..	@@ -99,52 +93,56 @@
99	93
100	94	/*
101	95	Assumptions:
102		- - if device has no dev->hard_header routine, it adds and removes ll header
103		- inside itself. In this case ll header is invisible outside of device,
104		- but higher levels still should reserve dev->hard_header_len.
105		- Some devices are enough clever to reallocate skb, when header
106		- will not fit to reserved space (tunnel), another ones are silly
107		- (PPP).
	96	+ - If the device has no dev->header_ops->create, there is no LL header
	97	+ visible above the device. In this case, its hard_header_len should be 0.
	98	+ The device may prepend its own header internally. In this case, its
	99	+ needed_headroom should be set to the space needed for it to add its
	100	+ internal header.
	101	+ For example, a WiFi driver pretending to be an Ethernet driver should
	102	+ set its hard_header_len to be the Ethernet header length, and set its
	103	+ needed_headroom to be (the real WiFi header length - the fake Ethernet
	104	+ header length).
108	105	- packet socket receives packets with pulled ll header,
109	106	so that SOCK_RAW should push it back.
110	107
111	108	On receive:
112	109	-----------
113	110
114		-Incoming, dev->hard_header!=NULL
	111	+Incoming, dev_has_header(dev) == true
115	112	mac_header -> ll header
116	113	data -> data
117	114
118		-Outgoing, dev->hard_header!=NULL
	115	+Outgoing, dev_has_header(dev) == true
119	116	mac_header -> ll header
120	117	data -> ll header
121	118
122		-Incoming, dev->hard_header==NULL
123		- mac_header -> UNKNOWN position. It is very likely, that it points to ll
124		- header. PPP makes it, that is wrong, because introduce
125		- assymetry between rx and tx paths.
	119	+Incoming, dev_has_header(dev) == false
	120	+ mac_header -> data
	121	+ However drivers often make it point to the ll header.
	122	+ This is incorrect because the ll header should be invisible to us.
126	123	data -> data
127	124
128		-Outgoing, dev->hard_header==NULL
129		- mac_header -> data. ll header is still not built!
	125	+Outgoing, dev_has_header(dev) == false
	126	+ mac_header -> data. ll header is invisible to us.
130	127	data -> data
131	128
132	129	Resume
133		- If dev->hard_header==NULL we are unlikely to restore sensible ll header.
	130	+ If dev_has_header(dev) == false we are unable to restore the ll header,
	131	+ because it is invisible to us.
134	132
135	133
136	134	On transmit:
137	135	------------
138	136
139		-dev->hard_header != NULL
	137	+dev->header_ops != NULL
140	138	mac_header -> ll header
141	139	data -> ll header
142	140
143		-dev->hard_header == NULL (ll header is added by device, we cannot control it)
	141	+dev->header_ops == NULL (ll header is invisible to us)
144	142	mac_header -> data
145	143	data -> data
146	144
147		- We should set nh.raw on output to correct posistion,
	145	+ We should set network_header on output to the correct position,
148	146	packet classifier depends on it.
149	147	*/
150	148
..	..	@@ -183,7 +181,6 @@
183	181	#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184	182	#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185	183	#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186		-#define BLOCK_PRIV(x) ((void )((char )(x) + BLOCK_O2PRIV(x)))
187	184
188	185	struct packet_sock;
189	186	static int tpacket_rcv(struct sk_buff skb, struct net_device dev,
..	..	@@ -272,27 +269,26 @@
272	269
273	270	static bool packet_use_direct_xmit(const struct packet_sock *po)
274	271	{
275		- return po->xmit == packet_direct_xmit;
276		-}
277		-
278		-static u16 __packet_pick_tx_queue(struct net_device dev, struct sk_buff skb,
279		- struct net_device *sb_dev)
280		-{
281		- return dev_pick_tx_cpu_id(dev, skb, sb_dev, NULL);
	272	+ /* Paired with WRITE_ONCE() in packet_setsockopt() */
	273	+ return READ_ONCE(po->xmit) == packet_direct_xmit;
282	274	}
283	275
284	276	static u16 packet_pick_tx_queue(struct sk_buff *skb)
285	277	{
286	278	struct net_device *dev = skb->dev;
287	279	const struct net_device_ops *ops = dev->netdev_ops;
	280	+ int cpu = raw_smp_processor_id();
288	281	u16 queue_index;
289	282
	283	+#ifdef CONFIG_XPS
	284	+ skb->sender_cpu = cpu + 1;
	285	+#endif
	286	+ skb_record_rx_queue(skb, cpu % dev->real_num_tx_queues);
290	287	if (ops->ndo_select_queue) {
291		- queue_index = ops->ndo_select_queue(dev, skb, NULL,
292		- __packet_pick_tx_queue);
	288	+ queue_index = ops->ndo_select_queue(dev, skb, NULL);
293	289	queue_index = netdev_cap_txqueue(dev, queue_index);
294	290	} else {
295		- queue_index = __packet_pick_tx_queue(dev, skb, NULL);
	291	+ queue_index = netdev_pick_tx(dev, skb, NULL);
296	292	}
297	293
298	294	return queue_index;
..	..	@@ -370,18 +366,20 @@
370	366	{
371	367	union tpacket_uhdr h;
372	368
	369	+ /* WRITE_ONCE() are paired with READ_ONCE() in __packet_get_status */
	370	+
373	371	h.raw = frame;
374	372	switch (po->tp_version) {
375	373	case TPACKET_V1:
376		- h.h1->tp_status = status;
	374	+ WRITE_ONCE(h.h1->tp_status, status);
377	375	flush_dcache_page(pgv_to_page(&h.h1->tp_status));
378	376	break;
379	377	case TPACKET_V2:
380		- h.h2->tp_status = status;
	378	+ WRITE_ONCE(h.h2->tp_status, status);
381	379	flush_dcache_page(pgv_to_page(&h.h2->tp_status));
382	380	break;
383	381	case TPACKET_V3:
384		- h.h3->tp_status = status;
	382	+ WRITE_ONCE(h.h3->tp_status, status);
385	383	flush_dcache_page(pgv_to_page(&h.h3->tp_status));
386	384	break;
387	385	default:
..	..	@@ -392,23 +390,25 @@
392	390	smp_wmb();
393	391	}
394	392
395		-static int __packet_get_status(struct packet_sock po, void frame)
	393	+static int __packet_get_status(const struct packet_sock po, void frame)
396	394	{
397	395	union tpacket_uhdr h;
398	396
399	397	smp_rmb();
400	398
	399	+ /* READ_ONCE() are paired with WRITE_ONCE() in __packet_set_status */
	400	+
401	401	h.raw = frame;
402	402	switch (po->tp_version) {
403	403	case TPACKET_V1:
404	404	flush_dcache_page(pgv_to_page(&h.h1->tp_status));
405		- return h.h1->tp_status;
	405	+ return READ_ONCE(h.h1->tp_status);
406	406	case TPACKET_V2:
407	407	flush_dcache_page(pgv_to_page(&h.h2->tp_status));
408		- return h.h2->tp_status;
	408	+ return READ_ONCE(h.h2->tp_status);
409	409	case TPACKET_V3:
410	410	flush_dcache_page(pgv_to_page(&h.h3->tp_status));
411		- return h.h3->tp_status;
	411	+ return READ_ONCE(h.h3->tp_status);
412	412	default:
413	413	WARN(1, "TPACKET version not supported.\n");
414	414	BUG();
..	..	@@ -416,17 +416,18 @@
416	416	}
417	417	}
418	418
419		-static __u32 tpacket_get_timestamp(struct sk_buff skb, struct timespec ts,
	419	+static __u32 tpacket_get_timestamp(struct sk_buff skb, struct timespec64 ts,
420	420	unsigned int flags)
421	421	{
422	422	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
423	423
424	424	if (shhwtstamps &&
425	425	(flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
426		- ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
	426	+ ktime_to_timespec64_cond(shhwtstamps->hwtstamp, ts))
427	427	return TP_STATUS_TS_RAW_HARDWARE;
428	428
429		- if (ktime_to_timespec_cond(skb->tstamp, ts))
	429	+ if ((flags & SOF_TIMESTAMPING_SOFTWARE) &&
	430	+ ktime_to_timespec64_cond(skb->tstamp, ts))
430	431	return TP_STATUS_TS_SOFTWARE;
431	432
432	433	return 0;
..	..	@@ -436,13 +437,20 @@
436	437	struct sk_buff *skb)
437	438	{
438	439	union tpacket_uhdr h;
439		- struct timespec ts;
	440	+ struct timespec64 ts;
440	441	__u32 ts_status;
441	442
442	443	if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
443	444	return 0;
444	445
445	446	h.raw = frame;
	447	+ /*
	448	+ * versions 1 through 3 overflow the timestamps in y2106, since they
	449	+ * all store the seconds in a 32-bit unsigned integer.
	450	+ * If we create a version 4, that should have a 64-bit timestamp,
	451	+ * either 64-bit seconds + 32-bit nanoseconds, or just 64-bit
	452	+ * nanoseconds.
	453	+ */
446	454	switch (po->tp_version) {
447	455	case TPACKET_V1:
448	456	h.h1->tp_sec = ts.tv_sec;
..	..	@@ -468,10 +476,10 @@
468	476	return ts_status;
469	477	}
470	478
471		-static void packet_lookup_frame(struct packet_sock po,
472		- struct packet_ring_buffer *rb,
473		- unsigned int position,
474		- int status)
	479	+static void packet_lookup_frame(const struct packet_sock po,
	480	+ const struct packet_ring_buffer *rb,
	481	+ unsigned int position,
	482	+ int status)
475	483	{
476	484	unsigned int pg_vec_pos, frame_offset;
477	485	union tpacket_uhdr h;
..	..	@@ -528,7 +536,7 @@
528	536	int blk_size_in_bytes)
529	537	{
530	538	struct net_device *dev;
531		- unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
	539	+ unsigned int mbits, div;
532	540	struct ethtool_link_ksettings ecmd;
533	541	int err;
534	542
..	..	@@ -540,31 +548,25 @@
540	548	}
541	549	err = __ethtool_get_link_ksettings(dev, &ecmd);
542	550	rtnl_unlock();
543		- if (!err) {
544		- /*
545		- * If the link speed is so slow you don't really
546		- * need to worry about perf anyways
547		- */
548		- if (ecmd.base.speed < SPEED_1000 \|\|
549		- ecmd.base.speed == SPEED_UNKNOWN) {
550		- return DEFAULT_PRB_RETIRE_TOV;
551		- } else {
552		- msec = 1;
553		- div = ecmd.base.speed / 1000;
554		- }
555		- } else
	551	+ if (err)
556	552	return DEFAULT_PRB_RETIRE_TOV;
557	553
	554	+ /* If the link speed is so slow you don't really
	555	+ * need to worry about perf anyways
	556	+ */
	557	+ if (ecmd.base.speed < SPEED_1000 \|\|
	558	+ ecmd.base.speed == SPEED_UNKNOWN)
	559	+ return DEFAULT_PRB_RETIRE_TOV;
	560	+
	561	+ div = ecmd.base.speed / 1000;
558	562	mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
559	563
560	564	if (div)
561	565	mbits /= div;
562	566
563		- tmo = mbits * msec;
564		-
565	567	if (div)
566		- return tmo+1;
567		- return tmo;
	568	+ return mbits + 1;
	569	+ return mbits;
568	570	}
569	571
570	572	static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
..	..	@@ -600,6 +602,7 @@
600	602	req_u->req3.tp_block_size);
601	603	p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
602	604	p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
	605	+ rwlock_init(&p1->blk_fill_in_prog_lock);
603	606
604	607	p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
605	608	prb_init_ft_ops(p1, req_u);
..	..	@@ -666,10 +669,9 @@
666	669	*
667	670	*/
668	671	if (BLOCK_NUM_PKTS(pbd)) {
669		- while (atomic_read(&pkc->blk_fill_in_prog)) {
670		- /* Waiting for skb_copy_bits to finish... */
671		- cpu_relax();
672		- }
	672	+ /* Waiting for skb_copy_bits to finish... */
	673	+ write_lock(&pkc->blk_fill_in_prog_lock);
	674	+ write_unlock(&pkc->blk_fill_in_prog_lock);
673	675	}
674	676
675	677	if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
..	..	@@ -767,7 +769,7 @@
767	769	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
768	770	struct sock *sk = &po->sk;
769	771
770		- if (po->stats.stats3.tp_drops)
	772	+ if (atomic_read(&po->tp_drops))
771	773	status \|= TP_STATUS_LOSING;
772	774
773	775	last_pkt = (struct tpacket3_hdr *)pkc1->prev;
..	..	@@ -783,8 +785,8 @@
783	785	* It shouldn't really happen as we don't close empty
784	786	* blocks. See prb_retire_rx_blk_timer_expired().
785	787	*/
786		- struct timespec ts;
787		- getnstimeofday(&ts);
	788	+ struct timespec64 ts;
	789	+ ktime_get_real_ts64(&ts);
788	790	h1->ts_last_pkt.ts_sec = ts.tv_sec;
789	791	h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
790	792	}
..	..	@@ -814,7 +816,7 @@
814	816	static void prb_open_block(struct tpacket_kbdq_core *pkc1,
815	817	struct tpacket_block_desc *pbd1)
816	818	{
817		- struct timespec ts;
	819	+ struct timespec64 ts;
818	820	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
819	821
820	822	smp_rmb();
..	..	@@ -827,7 +829,7 @@
827	829	BLOCK_NUM_PKTS(pbd1) = 0;
828	830	BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
829	831
830		- getnstimeofday(&ts);
	832	+ ktime_get_real_ts64(&ts);
831	833
832	834	h1->ts_first_pkt.ts_sec = ts.tv_sec;
833	835	h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
..	..	@@ -928,10 +930,9 @@
928	930	* the timer-handler already handled this case.
929	931	*/
930	932	if (!(status & TP_STATUS_BLK_TMO)) {
931		- while (atomic_read(&pkc->blk_fill_in_prog)) {
932		- /* Waiting for skb_copy_bits to finish... */
933		- cpu_relax();
934		- }
	933	+ /* Waiting for skb_copy_bits to finish... */
	934	+ write_lock(&pkc->blk_fill_in_prog_lock);
	935	+ write_unlock(&pkc->blk_fill_in_prog_lock);
935	936	}
936	937	prb_close_block(pkc, pbd, po, status);
937	938	return;
..	..	@@ -952,7 +953,8 @@
952	953	__releases(&pkc->blk_fill_in_prog_lock)
953	954	{
954	955	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
955		- atomic_dec(&pkc->blk_fill_in_prog);
	956	+
	957	+ read_unlock(&pkc->blk_fill_in_prog_lock);
956	958	}
957	959
958	960	static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
..	..	@@ -1007,14 +1009,13 @@
1007	1009	pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1008	1010	BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1009	1011	BLOCK_NUM_PKTS(pbd) += 1;
1010		- atomic_inc(&pkc->blk_fill_in_prog);
	1012	+ read_lock(&pkc->blk_fill_in_prog_lock);
1011	1013	prb_run_all_ft_ops(pkc, ppd);
1012	1014	}
1013	1015
1014	1016	/* Assumes caller has the sk->rx_queue.lock */
1015	1017	static void __packet_lookup_frame_in_block(struct packet_sock po,
1016	1018	struct sk_buff *skb,
1017		- int status,
1018	1019	unsigned int len
1019	1020	)
1020	1021	{
..	..	@@ -1086,7 +1087,7 @@
1086	1087	po->rx_ring.head, status);
1087	1088	return curr;
1088	1089	case TPACKET_V3:
1089		- return __packet_lookup_frame_in_block(po, skb, status, len);
	1090	+ return __packet_lookup_frame_in_block(po, skb, len);
1090	1091	default:
1091	1092	WARN(1, "TPACKET version not supported\n");
1092	1093	BUG();
..	..	@@ -1094,10 +1095,10 @@
1094	1095	}
1095	1096	}
1096	1097
1097		-static void prb_lookup_block(struct packet_sock po,
1098		- struct packet_ring_buffer *rb,
1099		- unsigned int idx,
1100		- int status)
	1098	+static void prb_lookup_block(const struct packet_sock po,
	1099	+ const struct packet_ring_buffer *rb,
	1100	+ unsigned int idx,
	1101	+ int status)
1101	1102	{
1102	1103	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1103	1104	struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
..	..	@@ -1210,12 +1211,12 @@
1210	1211	#define ROOM_LOW 0x1
1211	1212	#define ROOM_NORMAL 0x2
1212	1213
1213		-static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
	1214	+static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
1214	1215	{
1215	1216	int idx, len;
1216	1217
1217		- len = po->rx_ring.frame_max + 1;
1218		- idx = po->rx_ring.head;
	1218	+ len = READ_ONCE(po->rx_ring.frame_max) + 1;
	1219	+ idx = READ_ONCE(po->rx_ring.head);
1219	1220	if (pow_off)
1220	1221	idx += len >> pow_off;
1221	1222	if (idx >= len)
..	..	@@ -1223,12 +1224,12 @@
1223	1224	return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1224	1225	}
1225	1226
1226		-static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
	1227	+static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
1227	1228	{
1228	1229	int idx, len;
1229	1230
1230		- len = po->rx_ring.prb_bdqc.knum_blocks;
1231		- idx = po->rx_ring.prb_bdqc.kactive_blk_num;
	1231	+ len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
	1232	+ idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
1232	1233	if (pow_off)
1233	1234	idx += len >> pow_off;
1234	1235	if (idx >= len)
..	..	@@ -1236,15 +1237,18 @@
1236	1237	return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1237	1238	}
1238	1239
1239		-static int __packet_rcv_has_room(struct packet_sock po, struct sk_buff skb)
	1240	+static int __packet_rcv_has_room(const struct packet_sock *po,
	1241	+ const struct sk_buff *skb)
1240	1242	{
1241		- struct sock *sk = &po->sk;
	1243	+ const struct sock *sk = &po->sk;
1242	1244	int ret = ROOM_NONE;
1243	1245
1244	1246	if (po->prot_hook.func != tpacket_rcv) {
1245		- int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
1246		- - (skb ? skb->truesize : 0);
1247		- if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
	1247	+ int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
	1248	+ int avail = rcvbuf - atomic_read(&sk->sk_rmem_alloc)
	1249	+ - (skb ? skb->truesize : 0);
	1250	+
	1251	+ if (avail > (rcvbuf >> ROOM_POW_OFF))
1248	1252	return ROOM_NORMAL;
1249	1253	else if (avail > 0)
1250	1254	return ROOM_LOW;
..	..	@@ -1269,17 +1273,22 @@
1269	1273
1270	1274	static int packet_rcv_has_room(struct packet_sock po, struct sk_buff skb)
1271	1275	{
1272		- int ret;
1273		- bool has_room;
	1276	+ int pressure, ret;
1274	1277
1275		- spin_lock_bh(&po->sk.sk_receive_queue.lock);
1276	1278	ret = __packet_rcv_has_room(po, skb);
1277		- has_room = ret == ROOM_NORMAL;
1278		- if (po->pressure == has_room)
1279		- po->pressure = !has_room;
1280		- spin_unlock_bh(&po->sk.sk_receive_queue.lock);
	1279	+ pressure = ret != ROOM_NORMAL;
	1280	+
	1281	+ if (READ_ONCE(po->pressure) != pressure)
	1282	+ WRITE_ONCE(po->pressure, pressure);
1281	1283
1282	1284	return ret;
	1285	+}
	1286	+
	1287	+static void packet_rcv_try_clear_pressure(struct packet_sock *po)
	1288	+{
	1289	+ if (READ_ONCE(po->pressure) &&
	1290	+ __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
	1291	+ WRITE_ONCE(po->pressure, 0);
1283	1292	}
1284	1293
1285	1294	static void packet_sock_destruct(struct sock *sk)
..	..	@@ -1355,7 +1364,7 @@
1355	1364	struct packet_sock po, po_next, *po_skip = NULL;
1356	1365	unsigned int i, j, room = ROOM_NONE;
1357	1366
1358		- po = pkt_sk(f->arr[idx]);
	1367	+ po = pkt_sk(rcu_dereference(f->arr[idx]));
1359	1368
1360	1369	if (try_self) {
1361	1370	room = packet_rcv_has_room(po, skb);
..	..	@@ -1367,8 +1376,8 @@
1367	1376
1368	1377	i = j = min_t(int, po->rollover->sock, num - 1);
1369	1378	do {
1370		- po_next = pkt_sk(f->arr[i]);
1371		- if (po_next != po_skip && !po_next->pressure &&
	1379	+ po_next = pkt_sk(rcu_dereference(f->arr[i]));
	1380	+ if (po_next != po_skip && !READ_ONCE(po_next->pressure) &&
1372	1381	packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
1373	1382	if (i != j)
1374	1383	po->rollover->sock = i;
..	..	@@ -1462,7 +1471,7 @@
1462	1471	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1463	1472	idx = fanout_demux_rollover(f, skb, idx, true, num);
1464	1473
1465		- po = pkt_sk(f->arr[idx]);
	1474	+ po = pkt_sk(rcu_dereference(f->arr[idx]));
1466	1475	return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1467	1476	}
1468	1477
..	..	@@ -1476,7 +1485,7 @@
1476	1485	struct packet_fanout *f = po->fanout;
1477	1486
1478	1487	spin_lock(&f->lock);
1479		- f->arr[f->num_members] = sk;
	1488	+ rcu_assign_pointer(f->arr[f->num_members], sk);
1480	1489	smp_wmb();
1481	1490	f->num_members++;
1482	1491	if (f->num_members == 1)
..	..	@@ -1491,11 +1500,14 @@
1491	1500
1492	1501	spin_lock(&f->lock);
1493	1502	for (i = 0; i < f->num_members; i++) {
1494		- if (f->arr[i] == sk)
	1503	+ if (rcu_dereference_protected(f->arr[i],
	1504	+ lockdep_is_held(&f->lock)) == sk)
1495	1505	break;
1496	1506	}
1497	1507	BUG_ON(i >= f->num_members);
1498		- f->arr[i] = f->arr[f->num_members - 1];
	1508	+ rcu_assign_pointer(f->arr[i],
	1509	+ rcu_dereference_protected(f->arr[f->num_members - 1],
	1510	+ lockdep_is_held(&f->lock)));
1499	1511	f->num_members--;
1500	1512	if (f->num_members == 0)
1501	1513	__dev_remove_pack(&f->prot_hook);
..	..	@@ -1538,7 +1550,7 @@
1538	1550	}
1539	1551	}
1540	1552
1541		-static int fanout_set_data_cbpf(struct packet_sock po, char __user data,
	1553	+static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data,
1542	1554	unsigned int len)
1543	1555	{
1544	1556	struct bpf_prog *new;
..	..	@@ -1547,10 +1559,10 @@
1547	1559
1548	1560	if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1549	1561	return -EPERM;
1550		- if (len != sizeof(fprog))
1551		- return -EINVAL;
1552		- if (copy_from_user(&fprog, data, len))
1553		- return -EFAULT;
	1562	+
	1563	+ ret = copy_bpf_fprog_from_user(&fprog, data, len);
	1564	+ if (ret)
	1565	+ return ret;
1554	1566
1555	1567	ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
1556	1568	if (ret)
..	..	@@ -1560,7 +1572,7 @@
1560	1572	return 0;
1561	1573	}
1562	1574
1563		-static int fanout_set_data_ebpf(struct packet_sock po, char __user data,
	1575	+static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data,
1564	1576	unsigned int len)
1565	1577	{
1566	1578	struct bpf_prog *new;
..	..	@@ -1570,7 +1582,7 @@
1570	1582	return -EPERM;
1571	1583	if (len != sizeof(fd))
1572	1584	return -EINVAL;
1573		- if (copy_from_user(&fd, data, len))
	1585	+ if (copy_from_sockptr(&fd, data, len))
1574	1586	return -EFAULT;
1575	1587
1576	1588	new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
..	..	@@ -1581,7 +1593,7 @@
1581	1593	return 0;
1582	1594	}
1583	1595
1584		-static int fanout_set_data(struct packet_sock po, char __user data,
	1596	+static int fanout_set_data(struct packet_sock *po, sockptr_t data,
1585	1597	unsigned int len)
1586	1598	{
1587	1599	switch (po->fanout->type) {
..	..	@@ -1633,13 +1645,15 @@
1633	1645	return false;
1634	1646	}
1635	1647
1636		-static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
	1648	+static int fanout_add(struct sock sk, struct fanout_args args)
1637	1649	{
1638	1650	struct packet_rollover *rollover = NULL;
1639	1651	struct packet_sock *po = pkt_sk(sk);
	1652	+ u16 type_flags = args->type_flags;
1640	1653	struct packet_fanout f, match;
1641	1654	u8 type = type_flags & 0xff;
1642	1655	u8 flags = type_flags >> 8;
	1656	+ u16 id = args->id;
1643	1657	int err;
1644	1658
1645	1659	switch (type) {
..	..	@@ -1697,11 +1711,21 @@
1697	1711	}
1698	1712	}
1699	1713	err = -EINVAL;
1700		- if (match && match->flags != flags)
1701		- goto out;
1702		- if (!match) {
	1714	+ if (match) {
	1715	+ if (match->flags != flags)
	1716	+ goto out;
	1717	+ if (args->max_num_members &&
	1718	+ args->max_num_members != match->max_num_members)
	1719	+ goto out;
	1720	+ } else {
	1721	+ if (args->max_num_members > PACKET_FANOUT_MAX)
	1722	+ goto out;
	1723	+ if (!args->max_num_members)
	1724	+ /* legacy PACKET_FANOUT_MAX */
	1725	+ args->max_num_members = 256;
1703	1726	err = -ENOMEM;
1704		- match = kzalloc(sizeof(*match), GFP_KERNEL);
	1727	+ match = kvzalloc(struct_size(match, arr, args->max_num_members),
	1728	+ GFP_KERNEL);
1705	1729	if (!match)
1706	1730	goto out;
1707	1731	write_pnet(&match->net, sock_net(sk));
..	..	@@ -1718,6 +1742,7 @@
1718	1742	match->prot_hook.af_packet_priv = match;
1719	1743	match->prot_hook.af_packet_net = read_pnet(&match->net);
1720	1744	match->prot_hook.id_match = match_fanout_group;
	1745	+ match->max_num_members = args->max_num_members;
1721	1746	list_add(&match->list, &fanout_list);
1722	1747	}
1723	1748	err = -EINVAL;
..	..	@@ -1728,7 +1753,7 @@
1728	1753	match->prot_hook.type == po->prot_hook.type &&
1729	1754	match->prot_hook.dev == po->prot_hook.dev) {
1730	1755	err = -ENOSPC;
1731		- if (refcount_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
	1756	+ if (refcount_read(&match->sk_ref) < match->max_num_members) {
1732	1757	__dev_remove_pack(&po->prot_hook);
1733	1758
1734	1759	/* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */
..	..	@@ -1745,7 +1770,7 @@
1745	1770
1746	1771	if (err && !refcount_read(&match->sk_ref)) {
1747	1772	list_del(&match->list);
1748		- kfree(match);
	1773	+ kvfree(match);
1749	1774	}
1750	1775
1751	1776	out:
..	..	@@ -1835,7 +1860,7 @@
1835	1860	skb_dst_drop(skb);
1836	1861
1837	1862	/* drop conntrack reference */
1838		- nf_reset(skb);
	1863	+ nf_reset_ct(skb);
1839	1864
1840	1865	spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1841	1866
..	..	@@ -1863,6 +1888,24 @@
1863	1888	return 0;
1864	1889	}
1865	1890
	1891	+static void packet_parse_headers(struct sk_buff skb, struct socket sock)
	1892	+{
	1893	+ int depth;
	1894	+
	1895	+ if ((!skb->protocol \|\| skb->protocol == htons(ETH_P_ALL)) &&
	1896	+ sock->type == SOCK_RAW) {
	1897	+ skb_reset_mac_header(skb);
	1898	+ skb->protocol = dev_parse_header_protocol(skb);
	1899	+ }
	1900	+
	1901	+ /* Move network header to the right position for VLAN tagged packets */
	1902	+ if (likely(skb->dev->type == ARPHRD_ETHER) &&
	1903	+ eth_type_vlan(skb->protocol) &&
	1904	+ vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0)
	1905	+ skb_set_network_header(skb, depth);
	1906	+
	1907	+ skb_probe_transport_header(skb);
	1908	+}
1866	1909
1867	1910	/*
1868	1911	* Output a raw packet to a device layer. This bypasses all the other
..	..	@@ -1955,7 +1998,7 @@
1955	1998	goto retry;
1956	1999	}
1957	2000
1958		- if (!dev_validate_header(dev, skb->data, len)) {
	2001	+ if (!dev_validate_header(dev, skb->data, len) \|\| !skb->len) {
1959	2002	err = -EINVAL;
1960	2003	goto out_unlock;
1961	2004	}
..	..	@@ -1978,12 +2021,12 @@
1978	2021	skb->mark = sk->sk_mark;
1979	2022	skb->tstamp = sockc.transmit_time;
1980	2023
1981		- sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
	2024	+ skb_setup_tx_timestamp(skb, sockc.tsflags);
1982	2025
1983	2026	if (unlikely(extra_len == 4))
1984	2027	skb->no_fcs = 1;
1985	2028
1986		- skb_probe_transport_header(skb, 0);
	2029	+ packet_parse_headers(skb, sock);
1987	2030
1988	2031	dev_queue_xmit(skb);
1989	2032	rcu_read_unlock();
..	..	@@ -2060,7 +2103,7 @@
2060	2103
2061	2104	skb->dev = dev;
2062	2105
2063		- if (dev->header_ops) {
	2106	+ if (dev_has_header(dev)) {
2064	2107	/* The device has an explicit notion of ll header,
2065	2108	* exported to higher levels.
2066	2109	*
..	..	@@ -2105,7 +2148,7 @@
2105	2148	sll = &PACKET_SKB_CB(skb)->sa.ll;
2106	2149	sll->sll_hatype = dev->type;
2107	2150	sll->sll_pkttype = skb->pkt_type;
2108		- if (unlikely(po->origdev))
	2151	+ if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
2109	2152	sll->sll_ifindex = orig_dev->ifindex;
2110	2153	else
2111	2154	sll->sll_ifindex = dev->ifindex;
..	..	@@ -2125,7 +2168,7 @@
2125	2168	skb_dst_drop(skb);
2126	2169
2127	2170	/* drop conntrack reference */
2128		- nf_reset(skb);
	2171	+ nf_reset_ct(skb);
2129	2172
2130	2173	spin_lock(&sk->sk_receive_queue.lock);
2131	2174	po->stats.stats1.tp_packets++;
..	..	@@ -2137,10 +2180,8 @@
2137	2180
2138	2181	drop_n_acct:
2139	2182	is_drop_n_account = true;
2140		- spin_lock(&sk->sk_receive_queue.lock);
2141		- po->stats.stats1.tp_drops++;
	2183	+ atomic_inc(&po->tp_drops);
2142	2184	atomic_inc(&sk->sk_drops);
2143		- spin_unlock(&sk->sk_receive_queue.lock);
2144	2185
2145	2186	drop_n_restore:
2146	2187	if (skb_head != skb->data && skb_shared(skb)) {
..	..	@@ -2169,7 +2210,7 @@
2169	2210	unsigned short macoff, hdrlen;
2170	2211	unsigned int netoff;
2171	2212	struct sk_buff *copy_skb = NULL;
2172		- struct timespec ts;
	2213	+ struct timespec64 ts;
2173	2214	__u32 ts_status;
2174	2215	bool is_drop_n_account = false;
2175	2216	unsigned int slot_id = 0;
..	..	@@ -2191,7 +2232,7 @@
2191	2232	if (!net_eq(dev_net(dev), sock_net(sk)))
2192	2233	goto drop;
2193	2234
2194		- if (dev->header_ops) {
	2235	+ if (dev_has_header(dev)) {
2195	2236	if (sk->sk_type != SOCK_DGRAM)
2196	2237	skb_push(skb, skb->data - skb_mac_header(skb));
2197	2238	else if (skb->pkt_type == PACKET_OUTGOING) {
..	..	@@ -2206,11 +2247,16 @@
2206	2247	if (!res)
2207	2248	goto drop_n_restore;
2208	2249
	2250	+ /* If we are flooded, just give up */
	2251	+ if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
	2252	+ atomic_inc(&po->tp_drops);
	2253	+ goto drop_n_restore;
	2254	+ }
	2255	+
2209	2256	if (skb->ip_summed == CHECKSUM_PARTIAL)
2210	2257	status \|= TP_STATUS_CSUMNOTREADY;
2211	2258	else if (skb->pkt_type != PACKET_OUTGOING &&
2212		- (skb->ip_summed == CHECKSUM_COMPLETE \|\|
2213		- skb_csum_unnecessary(skb)))
	2259	+ skb_csum_unnecessary(skb))
2214	2260	status \|= TP_STATUS_CSUM_VALID;
2215	2261
2216	2262	if (snaplen > res)
..	..	@@ -2231,9 +2277,7 @@
2231	2277	macoff = netoff - maclen;
2232	2278	}
2233	2279	if (netoff > USHRT_MAX) {
2234		- spin_lock(&sk->sk_receive_queue.lock);
2235		- po->stats.stats1.tp_drops++;
2236		- spin_unlock(&sk->sk_receive_queue.lock);
	2280	+ atomic_inc(&po->tp_drops);
2237	2281	goto drop_n_restore;
2238	2282	}
2239	2283	if (po->tp_version <= TPACKET_V2) {
..	..	@@ -2246,8 +2290,11 @@
2246	2290	copy_skb = skb_get(skb);
2247	2291	skb_head = skb->data;
2248	2292	}
2249		- if (copy_skb)
	2293	+ if (copy_skb) {
	2294	+ memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
	2295	+ sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
2250	2296	skb_set_owner_r(copy_skb, sk);
	2297	+ }
2251	2298	}
2252	2299	snaplen = po->rx_ring.frame_size - macoff;
2253	2300	if ((int)snaplen < 0) {
..	..	@@ -2299,7 +2346,7 @@
2299	2346	* Anyways, moving it for V1/V2 only as V3 doesn't need this
2300	2347	* at packet level.
2301	2348	*/
2302		- if (po->stats.stats1.tp_drops)
	2349	+ if (atomic_read(&po->tp_drops))
2303	2350	status \|= TP_STATUS_LOSING;
2304	2351	}
2305	2352
..	..	@@ -2312,8 +2359,13 @@
2312	2359
2313	2360	skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
2314	2361
2315		- if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
2316		- getnstimeofday(&ts);
	2362	+ /* Always timestamp; prefer an existing software timestamp taken
	2363	+ * closer to the time of capture.
	2364	+ */
	2365	+ ts_status = tpacket_get_timestamp(skb, &ts,
	2366	+ po->tp_tstamp \| SOF_TIMESTAMPING_SOFTWARE);
	2367	+ if (!ts_status)
	2368	+ ktime_get_real_ts64(&ts);
2317	2369
2318	2370	status \|= ts_status;
2319	2371
..	..	@@ -2369,7 +2421,7 @@
2369	2421	sll->sll_hatype = dev->type;
2370	2422	sll->sll_protocol = skb->protocol;
2371	2423	sll->sll_pkttype = skb->pkt_type;
2372		- if (unlikely(po->origdev))
	2424	+ if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
2373	2425	sll->sll_ifindex = orig_dev->ifindex;
2374	2426	else
2375	2427	sll->sll_ifindex = dev->ifindex;
..	..	@@ -2412,9 +2464,9 @@
2412	2464	return 0;
2413	2465
2414	2466	drop_n_account:
2415		- is_drop_n_account = true;
2416		- po->stats.stats1.tp_drops++;
2417	2467	spin_unlock(&sk->sk_receive_queue.lock);
	2468	+ atomic_inc(&po->tp_drops);
	2469	+ is_drop_n_account = true;
2418	2470
2419	2471	sk->sk_data_ready(sk);
2420	2472	kfree_skb(copy_skb);
..	..	@@ -2440,15 +2492,6 @@
2440	2492	}
2441	2493
2442	2494	sock_wfree(skb);
2443		-}
2444		-
2445		-static void tpacket_set_protocol(const struct net_device *dev,
2446		- struct sk_buff *skb)
2447		-{
2448		- if (dev->type == ARPHRD_ETHER) {
2449		- skb_reset_mac_header(skb);
2450		- skb->protocol = eth_hdr(skb)->h_proto;
2451		- }
2452	2495	}
2453	2496
2454	2497	static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
..	..	@@ -2498,7 +2541,7 @@
2498	2541	skb->priority = po->sk.sk_priority;
2499	2542	skb->mark = po->sk.sk_mark;
2500	2543	skb->tstamp = sockc->transmit_time;
2501		- sock_tx_timestamp(&po->sk, sockc->tsflags, &skb_shinfo(skb)->tx_flags);
	2544	+ skb_setup_tx_timestamp(skb, sockc->tsflags);
2502	2545	skb_zcopy_set_nouarg(skb, ph.raw);
2503	2546
2504	2547	skb_reserve(skb, hlen);
..	..	@@ -2521,8 +2564,6 @@
2521	2564	return err;
2522	2565	if (!dev_validate_header(dev, skb->data, hdrlen))
2523	2566	return -EINVAL;
2524		- if (!skb->protocol)
2525		- tpacket_set_protocol(dev, skb);
2526	2567
2527	2568	data += hdrlen;
2528	2569	to_write -= hdrlen;
..	..	@@ -2557,7 +2598,7 @@
2557	2598	len = ((to_write > len_max) ? len_max : to_write);
2558	2599	}
2559	2600
2560		- skb_probe_transport_header(skb, 0);
	2601	+ packet_parse_headers(skb, sock);
2561	2602
2562	2603	return tp_len;
2563	2604	}
..	..	@@ -2787,9 +2828,11 @@
2787	2828	packet_inc_pending(&po->tx_ring);
2788	2829
2789	2830	status = TP_STATUS_SEND_REQUEST;
2790		- err = po->xmit(skb);
2791		- if (unlikely(err > 0)) {
2792		- err = net_xmit_errno(err);
	2831	+ /* Paired with WRITE_ONCE() in packet_setsockopt() */
	2832	+ err = READ_ONCE(po->xmit)(skb);
	2833	+ if (unlikely(err != 0)) {
	2834	+ if (err > 0)
	2835	+ err = net_xmit_errno(err);
2793	2836	if (err && __packet_get_status(po, ph) ==
2794	2837	TP_STATUS_AVAILABLE) {
2795	2838	/* skb was destructed already */
..	..	@@ -2956,13 +2999,13 @@
2956	2999	if (err)
2957	3000	goto out_free;
2958	3001
2959		- if (sock->type == SOCK_RAW &&
2960		- !dev_validate_header(dev, skb->data, len)) {
	3002	+ if ((sock->type == SOCK_RAW &&
	3003	+ !dev_validate_header(dev, skb->data, len)) \|\| !skb->len) {
2961	3004	err = -EINVAL;
2962	3005	goto out_free;
2963	3006	}
2964	3007
2965		- sock_tx_timestamp(sk, sockc.tsflags, &skb_shinfo(skb)->tx_flags);
	3008	+ skb_setup_tx_timestamp(skb, sockc.tsflags);
2966	3009
2967	3010	if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
2968	3011	!packet_extra_vlan_len_allowed(dev, skb)) {
..	..	@@ -2976,6 +3019,11 @@
2976	3019	skb->mark = sockc.mark;
2977	3020	skb->tstamp = sockc.transmit_time;
2978	3021
	3022	+ if (unlikely(extra_len == 4))
	3023	+ skb->no_fcs = 1;
	3024	+
	3025	+ packet_parse_headers(skb, sock);
	3026	+
2979	3027	if (has_vnet_hdr) {
2980	3028	err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
2981	3029	if (err)
..	..	@@ -2984,14 +3032,14 @@
2984	3032	virtio_net_hdr_set_proto(skb, &vnet_hdr);
2985	3033	}
2986	3034
2987		- skb_probe_transport_header(skb, reserve);
2988		-
2989		- if (unlikely(extra_len == 4))
2990		- skb->no_fcs = 1;
2991		-
2992		- err = po->xmit(skb);
2993		- if (err > 0 && (err = net_xmit_errno(err)) != 0)
2994		- goto out_unlock;
	3035	+ /* Paired with WRITE_ONCE() in packet_setsockopt() */
	3036	+ err = READ_ONCE(po->xmit)(skb);
	3037	+ if (unlikely(err != 0)) {
	3038	+ if (err > 0)
	3039	+ err = net_xmit_errno(err);
	3040	+ if (err)
	3041	+ goto out_unlock;
	3042	+ }
2995	3043
2996	3044	dev_put(dev);
2997	3045
..	..	@@ -3011,10 +3059,13 @@
3011	3059	struct sock *sk = sock->sk;
3012	3060	struct packet_sock *po = pkt_sk(sk);
3013	3061
3014		- if (po->tx_ring.pg_vec)
	3062	+ /* Reading tx_ring.pg_vec without holding pg_vec_lock is racy.
	3063	+ * tpacket_snd() will redo the check safely.
	3064	+ */
	3065	+ if (data_race(po->tx_ring.pg_vec))
3015	3066	return tpacket_snd(po, msg);
3016		- else
3017		- return packet_snd(sock, msg, len);
	3067	+
	3068	+ return packet_snd(sock, msg, len);
3018	3069	}
3019	3070
3020	3071	/*
..	..	@@ -3075,7 +3126,7 @@
3075	3126	kfree(po->rollover);
3076	3127	if (f) {
3077	3128	fanout_release_data(f);
3078		- kfree(f);
	3129	+ kvfree(f);
3079	3130	}
3080	3131	/*
3081	3132	* Now the socket is dead. No more input will appear.
..	..	@@ -3110,6 +3161,9 @@
3110	3161
3111	3162	lock_sock(sk);
3112	3163	spin_lock(&po->bind_lock);
	3164	+ if (!proto)
	3165	+ proto = po->num;
	3166	+
3113	3167	rcu_read_lock();
3114	3168
3115	3169	if (po->fanout) {
..	..	@@ -3212,7 +3266,7 @@
3212	3266	memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
3213	3267	name[sizeof(uaddr->sa_data)] = 0;
3214	3268
3215		- return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
	3269	+ return packet_do_bind(sk, name, 0, 0);
3216	3270	}
3217	3271
3218	3272	static int packet_bind(struct socket sock, struct sockaddr uaddr, int addr_len)
..	..	@@ -3229,8 +3283,7 @@
3229	3283	if (sll->sll_family != AF_PACKET)
3230	3284	return -EINVAL;
3231	3285
3232		- return packet_do_bind(sk, NULL, sll->sll_ifindex,
3233		- sll->sll_protocol ? : pkt_sk(sk)->num);
	3286	+ return packet_do_bind(sk, NULL, sll->sll_ifindex, sll->sll_protocol);
3234	3287	}
3235	3288
3236	3289	static struct proto packet_proto = {
..	..	@@ -3370,8 +3423,7 @@
3370	3423	if (skb == NULL)
3371	3424	goto out;
3372	3425
3373		- if (pkt_sk(sk)->pressure)
3374		- packet_rcv_has_room(pkt_sk(sk), NULL);
	3426	+ packet_rcv_try_clear_pressure(pkt_sk(sk));
3375	3427
3376	3428	if (pkt_sk(sk)->has_vnet_hdr) {
3377	3429	err = packet_rcv_vnet(msg, skb, &len);
..	..	@@ -3406,6 +3458,8 @@
3406	3458	sock_recv_ts_and_drops(msg, sk, skb);
3407	3459
3408	3460	if (msg->msg_name) {
	3461	+ const size_t max_len = min(sizeof(skb->cb),
	3462	+ sizeof(struct sockaddr_storage));
3409	3463	int copy_len;
3410	3464
3411	3465	/* If the address length field is there to be filled
..	..	@@ -3428,18 +3482,21 @@
3428	3482	msg->msg_namelen = sizeof(struct sockaddr_ll);
3429	3483	}
3430	3484	}
	3485	+ if (WARN_ON_ONCE(copy_len > max_len)) {
	3486	+ copy_len = max_len;
	3487	+ msg->msg_namelen = copy_len;
	3488	+ }
3431	3489	memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
3432	3490	}
3433	3491
3434		- if (pkt_sk(sk)->auxdata) {
	3492	+ if (packet_sock_flag(pkt_sk(sk), PACKET_SOCK_AUXDATA)) {
3435	3493	struct tpacket_auxdata aux;
3436	3494
3437	3495	aux.tp_status = TP_STATUS_USER;
3438	3496	if (skb->ip_summed == CHECKSUM_PARTIAL)
3439	3497	aux.tp_status \|= TP_STATUS_CSUMNOTREADY;
3440	3498	else if (skb->pkt_type != PACKET_OUTGOING &&
3441		- (skb->ip_summed == CHECKSUM_COMPLETE \|\|
3442		- skb_csum_unnecessary(skb)))
	3499	+ skb_csum_unnecessary(skb))
3443	3500	aux.tp_status \|= TP_STATUS_CSUM_VALID;
3444	3501
3445	3502	aux.tp_len = origlen;
..	..	@@ -3669,7 +3726,8 @@
3669	3726	}
3670	3727
3671	3728	static int
3672		-packet_setsockopt(struct socket sock, int level, int optname, char __user optval, unsigned int optlen)
	3729	+packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
	3730	+ unsigned int optlen)
3673	3731	{
3674	3732	struct sock *sk = sock->sk;
3675	3733	struct packet_sock *po = pkt_sk(sk);
..	..	@@ -3689,7 +3747,7 @@
3689	3747	return -EINVAL;
3690	3748	if (len > sizeof(mreq))
3691	3749	len = sizeof(mreq);
3692		- if (copy_from_user(&mreq, optval, len))
	3750	+ if (copy_from_sockptr(&mreq, optval, len))
3693	3751	return -EFAULT;
3694	3752	if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3695	3753	return -EINVAL;
..	..	@@ -3720,7 +3778,7 @@
3720	3778	if (optlen < len) {
3721	3779	ret = -EINVAL;
3722	3780	} else {
3723		- if (copy_from_user(&req_u.req, optval, len))
	3781	+ if (copy_from_sockptr(&req_u.req, optval, len))
3724	3782	ret = -EFAULT;
3725	3783	else
3726	3784	ret = packet_set_ring(sk, &req_u, 0,
..	..	@@ -3735,7 +3793,7 @@
3735	3793
3736	3794	if (optlen != sizeof(val))
3737	3795	return -EINVAL;
3738		- if (copy_from_user(&val, optval, sizeof(val)))
	3796	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3739	3797	return -EFAULT;
3740	3798
3741	3799	pkt_sk(sk)->copy_thresh = val;
..	..	@@ -3747,7 +3805,7 @@
3747	3805
3748	3806	if (optlen != sizeof(val))
3749	3807	return -EINVAL;
3750		- if (copy_from_user(&val, optval, sizeof(val)))
	3808	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3751	3809	return -EFAULT;
3752	3810	switch (val) {
3753	3811	case TPACKET_V1:
..	..	@@ -3773,7 +3831,7 @@
3773	3831
3774	3832	if (optlen != sizeof(val))
3775	3833	return -EINVAL;
3776		- if (copy_from_user(&val, optval, sizeof(val)))
	3834	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3777	3835	return -EFAULT;
3778	3836	if (val > INT_MAX)
3779	3837	return -EINVAL;
..	..	@@ -3793,7 +3851,7 @@
3793	3851
3794	3852	if (optlen != sizeof(val))
3795	3853	return -EINVAL;
3796		- if (copy_from_user(&val, optval, sizeof(val)))
	3854	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3797	3855	return -EFAULT;
3798	3856
3799	3857	lock_sock(sk);
..	..	@@ -3812,12 +3870,10 @@
3812	3870
3813	3871	if (optlen < sizeof(val))
3814	3872	return -EINVAL;
3815		- if (copy_from_user(&val, optval, sizeof(val)))
	3873	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3816	3874	return -EFAULT;
3817	3875
3818		- lock_sock(sk);
3819		- po->auxdata = !!val;
3820		- release_sock(sk);
	3876	+ packet_sock_flag_set(po, PACKET_SOCK_AUXDATA, val);
3821	3877	return 0;
3822	3878	}
3823	3879	case PACKET_ORIGDEV:
..	..	@@ -3826,12 +3882,10 @@
3826	3882
3827	3883	if (optlen < sizeof(val))
3828	3884	return -EINVAL;
3829		- if (copy_from_user(&val, optval, sizeof(val)))
	3885	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3830	3886	return -EFAULT;
3831	3887
3832		- lock_sock(sk);
3833		- po->origdev = !!val;
3834		- release_sock(sk);
	3888	+ packet_sock_flag_set(po, PACKET_SOCK_ORIGDEV, val);
3835	3889	return 0;
3836	3890	}
3837	3891	case PACKET_VNET_HDR:
..	..	@@ -3842,7 +3896,7 @@
3842	3896	return -EINVAL;
3843	3897	if (optlen < sizeof(val))
3844	3898	return -EINVAL;
3845		- if (copy_from_user(&val, optval, sizeof(val)))
	3899	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3846	3900	return -EFAULT;
3847	3901
3848	3902	lock_sock(sk);
..	..	@@ -3861,7 +3915,7 @@
3861	3915
3862	3916	if (optlen != sizeof(val))
3863	3917	return -EINVAL;
3864		- if (copy_from_user(&val, optval, sizeof(val)))
	3918	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3865	3919	return -EFAULT;
3866	3920
3867	3921	po->tp_tstamp = val;
..	..	@@ -3869,14 +3923,14 @@
3869	3923	}
3870	3924	case PACKET_FANOUT:
3871	3925	{
3872		- int val;
	3926	+ struct fanout_args args = { 0 };
3873	3927
3874		- if (optlen != sizeof(val))
	3928	+ if (optlen != sizeof(int) && optlen != sizeof(args))
3875	3929	return -EINVAL;
3876		- if (copy_from_user(&val, optval, sizeof(val)))
	3930	+ if (copy_from_sockptr(&args, optval, optlen))
3877	3931	return -EFAULT;
3878	3932
3879		- return fanout_add(sk, val & 0xffff, val >> 16);
	3933	+ return fanout_add(sk, &args);
3880	3934	}
3881	3935	case PACKET_FANOUT_DATA:
3882	3936	{
..	..	@@ -3886,13 +3940,27 @@
3886	3940
3887	3941	return fanout_set_data(po, optval, optlen);
3888	3942	}
	3943	+ case PACKET_IGNORE_OUTGOING:
	3944	+ {
	3945	+ int val;
	3946	+
	3947	+ if (optlen != sizeof(val))
	3948	+ return -EINVAL;
	3949	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
	3950	+ return -EFAULT;
	3951	+ if (val < 0 \|\| val > 1)
	3952	+ return -EINVAL;
	3953	+
	3954	+ po->prot_hook.ignore_outgoing = !!val;
	3955	+ return 0;
	3956	+ }
3889	3957	case PACKET_TX_HAS_OFF:
3890	3958	{
3891	3959	unsigned int val;
3892	3960
3893	3961	if (optlen != sizeof(val))
3894	3962	return -EINVAL;
3895		- if (copy_from_user(&val, optval, sizeof(val)))
	3963	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3896	3964	return -EFAULT;
3897	3965
3898	3966	lock_sock(sk);
..	..	@@ -3911,10 +3979,11 @@
3911	3979
3912	3980	if (optlen != sizeof(val))
3913	3981	return -EINVAL;
3914		- if (copy_from_user(&val, optval, sizeof(val)))
	3982	+ if (copy_from_sockptr(&val, optval, sizeof(val)))
3915	3983	return -EFAULT;
3916	3984
3917		- po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
	3985	+ /* Paired with all lockless reads of po->xmit */
	3986	+ WRITE_ONCE(po->xmit, val ? packet_direct_xmit : dev_queue_xmit);
3918	3987	return 0;
3919	3988	}
3920	3989	default:
..	..	@@ -3932,6 +4001,7 @@
3932	4001	void *data = &val;
3933	4002	union tpacket_stats_u st;
3934	4003	struct tpacket_rollover_stats rstats;
	4004	+ int drops;
3935	4005
3936	4006	if (level != SOL_PACKET)
3937	4007	return -ENOPROTOOPT;
..	..	@@ -3948,23 +4018,26 @@
3948	4018	memcpy(&st, &po->stats, sizeof(st));
3949	4019	memset(&po->stats, 0, sizeof(po->stats));
3950	4020	spin_unlock_bh(&sk->sk_receive_queue.lock);
	4021	+ drops = atomic_xchg(&po->tp_drops, 0);
3951	4022
3952	4023	if (po->tp_version == TPACKET_V3) {
3953	4024	lv = sizeof(struct tpacket_stats_v3);
3954		- st.stats3.tp_packets += st.stats3.tp_drops;
	4025	+ st.stats3.tp_drops = drops;
	4026	+ st.stats3.tp_packets += drops;
3955	4027	data = &st.stats3;
3956	4028	} else {
3957	4029	lv = sizeof(struct tpacket_stats);
3958		- st.stats1.tp_packets += st.stats1.tp_drops;
	4030	+ st.stats1.tp_drops = drops;
	4031	+ st.stats1.tp_packets += drops;
3959	4032	data = &st.stats1;
3960	4033	}
3961	4034
3962	4035	break;
3963	4036	case PACKET_AUXDATA:
3964		- val = po->auxdata;
	4037	+ val = packet_sock_flag(po, PACKET_SOCK_AUXDATA);
3965	4038	break;
3966	4039	case PACKET_ORIGDEV:
3967		- val = po->origdev;
	4040	+ val = packet_sock_flag(po, PACKET_SOCK_ORIGDEV);
3968	4041	break;
3969	4042	case PACKET_VNET_HDR:
3970	4043	val = po->has_vnet_hdr;
..	..	@@ -4009,6 +4082,9 @@
4009	4082	((u32)po->fanout->flags << 24)) :
4010	4083	0);
4011	4084	break;
	4085	+ case PACKET_IGNORE_OUTGOING:
	4086	+ val = po->prot_hook.ignore_outgoing;
	4087	+ break;
4012	4088	case PACKET_ROLLOVER_STATS:
4013	4089	if (!po->rollover)
4014	4090	return -EINVAL;
..	..	@@ -4037,28 +4113,6 @@
4037	4113	return 0;
4038	4114	}
4039	4115
4040		-
4041		-#ifdef CONFIG_COMPAT
4042		-static int compat_packet_setsockopt(struct socket *sock, int level, int optname,
4043		- char __user *optval, unsigned int optlen)
4044		-{
4045		- struct packet_sock *po = pkt_sk(sock->sk);
4046		-
4047		- if (level != SOL_PACKET)
4048		- return -ENOPROTOOPT;
4049		-
4050		- if (optname == PACKET_FANOUT_DATA &&
4051		- po->fanout && po->fanout->type == PACKET_FANOUT_CBPF) {
4052		- optval = (char __user *)get_compat_bpf_fprog(optval);
4053		- if (!optval)
4054		- return -EFAULT;
4055		- optlen = sizeof(struct sock_fprog);
4056		- }
4057		-
4058		- return packet_setsockopt(sock, level, optname, optval, optlen);
4059		-}
4060		-#endif
4061		-
4062	4116	static int packet_notifier(struct notifier_block *this,
4063	4117	unsigned long msg, void *ptr)
4064	4118	{
..	..	@@ -4074,7 +4128,7 @@
4074	4128	case NETDEV_UNREGISTER:
4075	4129	if (po->mclist)
4076	4130	packet_dev_mclist_delete(dev, &po->mclist);
4077		- /* fallthrough */
	4131	+ fallthrough;
4078	4132
4079	4133	case NETDEV_DOWN:
4080	4134	if (dev->ifindex == po->ifindex) {
..	..	@@ -4134,11 +4188,6 @@
4134	4188	spin_unlock_bh(&sk->sk_receive_queue.lock);
4135	4189	return put_user(amount, (int __user *)arg);
4136	4190	}
4137		- case SIOCGSTAMP:
4138		- return sock_get_timestamp(sk, (struct timeval __user *)arg);
4139		- case SIOCGSTAMPNS:
4140		- return sock_get_timestampns(sk, (struct timespec __user *)arg);
4141		-
4142	4191	#ifdef CONFIG_INET
4143	4192	case SIOCADDRT:
4144	4193	case SIOCDELRT:
..	..	@@ -4176,8 +4225,7 @@
4176	4225	TP_STATUS_KERNEL))
4177	4226	mask \|= EPOLLIN \| EPOLLRDNORM;
4178	4227	}
4179		- if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
4180		- po->pressure = 0;
	4228	+ packet_rcv_try_clear_pressure(po);
4181	4229	spin_unlock_bh(&sk->sk_receive_queue.lock);
4182	4230	spin_lock_bh(&sk->sk_write_queue.lock);
4183	4231	if (po->tx_ring.pg_vec) {
..	..	@@ -4296,7 +4344,7 @@
4296	4344	struct packet_ring_buffer *rb;
4297	4345	struct sk_buff_head *rb_queue;
4298	4346	__be16 num;
4299		- int err = -EINVAL;
	4347	+ int err;
4300	4348	/* Added to avoid minimal code churn */
4301	4349	struct tpacket_req *req = &req_u->req;
4302	4350
..	..	@@ -4526,10 +4574,9 @@
4526	4574	.getname = packet_getname_spkt,
4527	4575	.poll = datagram_poll,
4528	4576	.ioctl = packet_ioctl,
	4577	+ .gettstamp = sock_gettstamp,
4529	4578	.listen = sock_no_listen,
4530	4579	.shutdown = sock_no_shutdown,
4531		- .setsockopt = sock_no_setsockopt,
4532		- .getsockopt = sock_no_getsockopt,
4533	4580	.sendmsg = packet_sendmsg_spkt,
4534	4581	.recvmsg = packet_recvmsg,
4535	4582	.mmap = sock_no_mmap,
..	..	@@ -4547,13 +4594,11 @@
4547	4594	.getname = packet_getname,
4548	4595	.poll = packet_poll,
4549	4596	.ioctl = packet_ioctl,
	4597	+ .gettstamp = sock_gettstamp,
4550	4598	.listen = sock_no_listen,
4551	4599	.shutdown = sock_no_shutdown,
4552	4600	.setsockopt = packet_setsockopt,
4553	4601	.getsockopt = packet_getsockopt,
4554		-#ifdef CONFIG_COMPAT
4555		- .compat_setsockopt = compat_packet_setsockopt,
4556		-#endif
4557	4602	.sendmsg = packet_sendmsg,
4558	4603	.recvmsg = packet_recvmsg,
4559	4604	.mmap = packet_mmap,
..	..	@@ -4630,9 +4675,11 @@
4630	4675	mutex_init(&net->packet.sklist_lock);
4631	4676	INIT_HLIST_HEAD(&net->packet.sklist);
4632	4677
	4678	+#ifdef CONFIG_PROC_FS
4633	4679	if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
4634	4680	sizeof(struct seq_net_private)))
4635	4681	return -ENOMEM;
	4682	+#endif /* CONFIG_PROC_FS */
4636	4683
4637	4684	return 0;
4638	4685	}