~ljy/RK356X_SDK_RELEASE.git

..	..	@@ -1,10 +1,6 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* NET3 Protocol independent device support routines.
3		- *
4		- * This program is free software; you can redistribute it and/or
5		- * modify it under the terms of the GNU General Public License
6		- * as published by the Free Software Foundation; either version
7		- * 2 of the License, or (at your option) any later version.
8	4	*
9	5	* Derived from the non IP parts of dev.c 1.0.19
10	6	* Authors: Ross Biro
..	..	@@ -102,6 +98,7 @@
102	98	#include <net/busy_poll.h>
103	99	#include <linux/rtnetlink.h>
104	100	#include <linux/stat.h>
	101	+#include <net/dsa.h>
105	102	#include <net/dst.h>
106	103	#include <net/dst_metadata.h>
107	104	#include <net/pkt_sched.h>
..	..	@@ -132,7 +129,6 @@
132	129	#include <trace/events/napi.h>
133	130	#include <trace/events/net.h>
134	131	#include <trace/events/skb.h>
135		-#include <linux/pci.h>
136	132	#include <linux/inetdevice.h>
137	133	#include <linux/cpu_rmap.h>
138	134	#include <linux/static_key.h>
..	..	@@ -146,11 +142,15 @@
146	142	#include <linux/sctp.h>
147	143	#include <net/udp_tunnel.h>
148	144	#include <linux/net_namespace.h>
	145	+#include <linux/indirect_call_wrapper.h>
	146	+#include <net/devlink.h>
	147	+#include <linux/pm_runtime.h>
	148	+#include <linux/prandom.h>
	149	+#include <trace/hooks/net.h>
149	150
150	151	#include "net-sysfs.h"
151	152
152	153	#define MAX_GRO_SKBS 8
153		-#define MAX_NEST_DEV 8
154	154
155	155	/* This should be increased if a protocol with a bigger head is added. */
156	156	#define GRO_MAX_HEAD (MAX_HEADER + 128)
..	..	@@ -164,6 +164,9 @@
164	164	static int netif_rx_internal(struct sk_buff *skb);
165	165	static int call_netdevice_notifiers_info(unsigned long val,
166	166	struct netdev_notifier_info *info);
	167	+static int call_netdevice_notifiers_extack(unsigned long val,
	168	+ struct net_device *dev,
	169	+ struct netlink_ext_ack *extack);
167	170	static struct napi_struct *napi_by_id(unsigned int napi_id);
168	171
169	172	/*
..	..	@@ -219,15 +222,137 @@
219	222	static inline void rps_lock(struct softnet_data *sd)
220	223	{
221	224	#ifdef CONFIG_RPS
222		- spin_lock(&sd->input_pkt_queue.lock);
	225	+ raw_spin_lock(&sd->input_pkt_queue.raw_lock);
223	226	#endif
224	227	}
225	228
226	229	static inline void rps_unlock(struct softnet_data *sd)
227	230	{
228	231	#ifdef CONFIG_RPS
229		- spin_unlock(&sd->input_pkt_queue.lock);
	232	+ raw_spin_unlock(&sd->input_pkt_queue.raw_lock);
230	233	#endif
	234	+}
	235	+
	236	+static struct netdev_name_node netdev_name_node_alloc(struct net_device dev,
	237	+ const char *name)
	238	+{
	239	+ struct netdev_name_node *name_node;
	240	+
	241	+ name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
	242	+ if (!name_node)
	243	+ return NULL;
	244	+ INIT_HLIST_NODE(&name_node->hlist);
	245	+ name_node->dev = dev;
	246	+ name_node->name = name;
	247	+ return name_node;
	248	+}
	249	+
	250	+static struct netdev_name_node *
	251	+netdev_name_node_head_alloc(struct net_device *dev)
	252	+{
	253	+ struct netdev_name_node *name_node;
	254	+
	255	+ name_node = netdev_name_node_alloc(dev, dev->name);
	256	+ if (!name_node)
	257	+ return NULL;
	258	+ INIT_LIST_HEAD(&name_node->list);
	259	+ return name_node;
	260	+}
	261	+
	262	+static void netdev_name_node_free(struct netdev_name_node *name_node)
	263	+{
	264	+ kfree(name_node);
	265	+}
	266	+
	267	+static void netdev_name_node_add(struct net *net,
	268	+ struct netdev_name_node *name_node)
	269	+{
	270	+ hlist_add_head_rcu(&name_node->hlist,
	271	+ dev_name_hash(net, name_node->name));
	272	+}
	273	+
	274	+static void netdev_name_node_del(struct netdev_name_node *name_node)
	275	+{
	276	+ hlist_del_rcu(&name_node->hlist);
	277	+}
	278	+
	279	+static struct netdev_name_node netdev_name_node_lookup(struct net net,
	280	+ const char *name)
	281	+{
	282	+ struct hlist_head *head = dev_name_hash(net, name);
	283	+ struct netdev_name_node *name_node;
	284	+
	285	+ hlist_for_each_entry(name_node, head, hlist)
	286	+ if (!strcmp(name_node->name, name))
	287	+ return name_node;
	288	+ return NULL;
	289	+}
	290	+
	291	+static struct netdev_name_node netdev_name_node_lookup_rcu(struct net net,
	292	+ const char *name)
	293	+{
	294	+ struct hlist_head *head = dev_name_hash(net, name);
	295	+ struct netdev_name_node *name_node;
	296	+
	297	+ hlist_for_each_entry_rcu(name_node, head, hlist)
	298	+ if (!strcmp(name_node->name, name))
	299	+ return name_node;
	300	+ return NULL;
	301	+}
	302	+
	303	+int netdev_name_node_alt_create(struct net_device dev, const char name)
	304	+{
	305	+ struct netdev_name_node *name_node;
	306	+ struct net *net = dev_net(dev);
	307	+
	308	+ name_node = netdev_name_node_lookup(net, name);
	309	+ if (name_node)
	310	+ return -EEXIST;
	311	+ name_node = netdev_name_node_alloc(dev, name);
	312	+ if (!name_node)
	313	+ return -ENOMEM;
	314	+ netdev_name_node_add(net, name_node);
	315	+ /* The node that holds dev->name acts as a head of per-device list. */
	316	+ list_add_tail(&name_node->list, &dev->name_node->list);
	317	+
	318	+ return 0;
	319	+}
	320	+EXPORT_SYMBOL(netdev_name_node_alt_create);
	321	+
	322	+static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
	323	+{
	324	+ list_del(&name_node->list);
	325	+ netdev_name_node_del(name_node);
	326	+ kfree(name_node->name);
	327	+ netdev_name_node_free(name_node);
	328	+}
	329	+
	330	+int netdev_name_node_alt_destroy(struct net_device dev, const char name)
	331	+{
	332	+ struct netdev_name_node *name_node;
	333	+ struct net *net = dev_net(dev);
	334	+
	335	+ name_node = netdev_name_node_lookup(net, name);
	336	+ if (!name_node)
	337	+ return -ENOENT;
	338	+ /* lookup might have found our primary name or a name belonging
	339	+ * to another device.
	340	+ */
	341	+ if (name_node == dev->name_node \|\| name_node->dev != dev)
	342	+ return -EINVAL;
	343	+
	344	+ __netdev_name_node_alt_destroy(name_node);
	345	+
	346	+ return 0;
	347	+}
	348	+EXPORT_SYMBOL(netdev_name_node_alt_destroy);
	349	+
	350	+static void netdev_name_node_alt_flush(struct net_device *dev)
	351	+{
	352	+ struct netdev_name_node name_node, tmp;
	353	+
	354	+ list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
	355	+ __netdev_name_node_alt_destroy(name_node);
231	356	}
232	357
233	358	/* Device list insertion */
..	..	@@ -239,7 +364,7 @@
239	364
240	365	write_lock_bh(&dev_base_lock);
241	366	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
242		- hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
	367	+ netdev_name_node_add(net, dev->name_node);
243	368	hlist_add_head_rcu(&dev->index_hlist,
244	369	dev_index_hash(net, dev->ifindex));
245	370	write_unlock_bh(&dev_base_lock);
..	..	@@ -257,7 +382,7 @@
257	382	/* Unlink dev from the device chain */
258	383	write_lock_bh(&dev_base_lock);
259	384	list_del_rcu(&dev->dev_list);
260		- hlist_del_rcu(&dev->name_hlist);
	385	+ netdev_name_node_del(dev->name_node);
261	386	hlist_del_rcu(&dev->index_hlist);
262	387	write_unlock_bh(&dev_base_lock);
263	388
..	..	@@ -355,6 +480,7 @@
355	480	unsigned short dev_type)
356	481	{
357	482	}
	483	+
358	484	static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
359	485	{
360	486	}
..	..	@@ -385,6 +511,12 @@
385	511
386	512	static inline struct list_head ptype_head(const struct packet_type pt)
387	513	{
	514	+ struct list_head vendor_pt = { .next = NULL, };
	515	+
	516	+ trace_android_vh_ptype_head(pt, &vendor_pt);
	517	+ if (vendor_pt.next)
	518	+ return vendor_pt.next;
	519	+
388	520	if (pt->type == htons(ETH_P_ALL))
389	521	return pt->dev ? &pt->dev->ptype_all : &ptype_all;
390	522	else
..	..	@@ -735,14 +867,10 @@
735	867
736	868	struct net_device __dev_get_by_name(struct net net, const char *name)
737	869	{
738		- struct net_device *dev;
739		- struct hlist_head *head = dev_name_hash(net, name);
	870	+ struct netdev_name_node *node_name;
740	871
741		- hlist_for_each_entry(dev, head, name_hlist)
742		- if (!strncmp(dev->name, name, IFNAMSIZ))
743		- return dev;
744		-
745		- return NULL;
	872	+ node_name = netdev_name_node_lookup(net, name);
	873	+ return node_name ? node_name->dev : NULL;
746	874	}
747	875	EXPORT_SYMBOL(__dev_get_by_name);
748	876
..	..	@@ -760,14 +888,10 @@
760	888
761	889	struct net_device dev_get_by_name_rcu(struct net net, const char *name)
762	890	{
763		- struct net_device *dev;
764		- struct hlist_head *head = dev_name_hash(net, name);
	891	+ struct netdev_name_node *node_name;
765	892
766		- hlist_for_each_entry_rcu(dev, head, name_hlist)
767		- if (!strncmp(dev->name, name, IFNAMSIZ))
768		- return dev;
769		-
770		- return NULL;
	893	+ node_name = netdev_name_node_lookup_rcu(net, name);
	894	+ return node_name ? node_name->dev : NULL;
771	895	}
772	896	EXPORT_SYMBOL(dev_get_by_name_rcu);
773	897
..	..	@@ -1015,7 +1139,7 @@
1015	1139	* @name: name string
1016	1140	*
1017	1141	* Network device names need to be valid file names to
1018		- * to allow sysfs to work. We also disallow any kind of
	1142	+ * allow sysfs to work. We also disallow any kind of
1019	1143	* whitespace.
1020	1144	*/
1021	1145	bool dev_valid_name(const char *name)
..	..	@@ -1078,6 +1202,18 @@
1078	1202	return -ENOMEM;
1079	1203
1080	1204	for_each_netdev(net, d) {
	1205	+ struct netdev_name_node *name_node;
	1206	+ list_for_each_entry(name_node, &d->name_node->list, list) {
	1207	+ if (!sscanf(name_node->name, name, &i))
	1208	+ continue;
	1209	+ if (i < 0 \|\| i >= max_netdevices)
	1210	+ continue;
	1211	+
	1212	+ /* avoid cases where sscanf is not exact inverse of printf */
	1213	+ snprintf(buf, IFNAMSIZ, name, i);
	1214	+ if (!strncmp(buf, name_node->name, IFNAMSIZ))
	1215	+ set_bit(i, inuse);
	1216	+ }
1081	1217	if (!sscanf(d->name, name, &i))
1082	1218	continue;
1083	1219	if (i < 0 \|\| i >= max_netdevices)
..	..	@@ -1138,8 +1274,8 @@
1138	1274	}
1139	1275	EXPORT_SYMBOL(dev_alloc_name);
1140	1276
1141		-int dev_get_valid_name(struct net net, struct net_device dev,
1142		- const char *name)
	1277	+static int dev_get_valid_name(struct net net, struct net_device dev,
	1278	+ const char *name)
1143	1279	{
1144	1280	BUG_ON(!net);
1145	1281
..	..	@@ -1155,7 +1291,6 @@
1155	1291
1156	1292	return 0;
1157	1293	}
1158		-EXPORT_SYMBOL(dev_get_valid_name);
1159	1294
1160	1295	/**
1161	1296	* dev_change_name - change name of a device
..	..	@@ -1229,13 +1364,13 @@
1229	1364	netdev_adjacent_rename_links(dev, oldname);
1230	1365
1231	1366	write_lock_bh(&dev_base_lock);
1232		- hlist_del_rcu(&dev->name_hlist);
	1367	+ netdev_name_node_del(dev->name_node);
1233	1368	write_unlock_bh(&dev_base_lock);
1234	1369
1235	1370	synchronize_rcu();
1236	1371
1237	1372	write_lock_bh(&dev_base_lock);
1238		- hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
	1373	+ netdev_name_node_add(net, dev->name_node);
1239	1374	write_unlock_bh(&dev_base_lock);
1240	1375
1241	1376	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
..	..	@@ -1285,8 +1420,8 @@
1285	1420	}
1286	1421
1287	1422	mutex_lock(&ifalias_mutex);
1288		- rcu_swap_protected(dev->ifalias, new_alias,
1289		- mutex_is_locked(&ifalias_mutex));
	1423	+ new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
	1424	+ mutex_is_locked(&ifalias_mutex));
1290	1425	mutex_unlock(&ifalias_mutex);
1291	1426
1292	1427	if (new_alias)
..	..	@@ -1372,15 +1507,20 @@
1372	1507	}
1373	1508	EXPORT_SYMBOL(netdev_notify_peers);
1374	1509
1375		-static int __dev_open(struct net_device *dev)
	1510	+static int __dev_open(struct net_device dev, struct netlink_ext_ack extack)
1376	1511	{
1377	1512	const struct net_device_ops *ops = dev->netdev_ops;
1378	1513	int ret;
1379	1514
1380	1515	ASSERT_RTNL();
1381	1516
1382		- if (!netif_device_present(dev))
1383		- return -ENODEV;
	1517	+ if (!netif_device_present(dev)) {
	1518	+ /* may be detached because parent is runtime-suspended */
	1519	+ if (dev->dev.parent)
	1520	+ pm_runtime_resume(dev->dev.parent);
	1521	+ if (!netif_device_present(dev))
	1522	+ return -ENODEV;
	1523	+ }
1384	1524
1385	1525	/* Block netpoll from trying to do any rx path servicing.
1386	1526	* If we don't do this there is a chance ndo_poll_controller
..	..	@@ -1388,7 +1528,7 @@
1388	1528	*/
1389	1529	netpoll_poll_disable(dev);
1390	1530
1391		- ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
	1531	+ ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1392	1532	ret = notifier_to_errno(ret);
1393	1533	if (ret)
1394	1534	return ret;
..	..	@@ -1417,7 +1557,8 @@
1417	1557
1418	1558	/**
1419	1559	* dev_open - prepare an interface for use.
1420		- * @dev: device to open
	1560	+ * @dev: device to open
	1561	+ * @extack: netlink extended ack
1421	1562	*
1422	1563	* Takes a device from down to up state. The device's private open
1423	1564	* function is invoked and then the multicast lists are loaded. Finally
..	..	@@ -1427,14 +1568,14 @@
1427	1568	* Calling this function on an active interface is a nop. On a failure
1428	1569	* a negative errno code is returned.
1429	1570	*/
1430		-int dev_open(struct net_device *dev)
	1571	+int dev_open(struct net_device dev, struct netlink_ext_ack extack)
1431	1572	{
1432	1573	int ret;
1433	1574
1434	1575	if (dev->flags & IFF_UP)
1435	1576	return 0;
1436	1577
1437		- ret = __dev_open(dev);
	1578	+ ret = __dev_open(dev, extack);
1438	1579	if (ret < 0)
1439	1580	return ret;
1440	1581
..	..	@@ -1596,6 +1737,7 @@
1596	1737	N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1597	1738	N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1598	1739	N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
	1740	+ N(PRE_CHANGEADDR)
1599	1741	}
1600	1742	#undef N
1601	1743	return "UNKNOWN_NETDEV_EVENT";
..	..	@@ -1610,6 +1752,62 @@
1610	1752	};
1611	1753
1612	1754	return nb->notifier_call(nb, val, &info);
	1755	+}
	1756	+
	1757	+static int call_netdevice_register_notifiers(struct notifier_block *nb,
	1758	+ struct net_device *dev)
	1759	+{
	1760	+ int err;
	1761	+
	1762	+ err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
	1763	+ err = notifier_to_errno(err);
	1764	+ if (err)
	1765	+ return err;
	1766	+
	1767	+ if (!(dev->flags & IFF_UP))
	1768	+ return 0;
	1769	+
	1770	+ call_netdevice_notifier(nb, NETDEV_UP, dev);
	1771	+ return 0;
	1772	+}
	1773	+
	1774	+static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
	1775	+ struct net_device *dev)
	1776	+{
	1777	+ if (dev->flags & IFF_UP) {
	1778	+ call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
	1779	+ dev);
	1780	+ call_netdevice_notifier(nb, NETDEV_DOWN, dev);
	1781	+ }
	1782	+ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
	1783	+}
	1784	+
	1785	+static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
	1786	+ struct net *net)
	1787	+{
	1788	+ struct net_device *dev;
	1789	+ int err;
	1790	+
	1791	+ for_each_netdev(net, dev) {
	1792	+ err = call_netdevice_register_notifiers(nb, dev);
	1793	+ if (err)
	1794	+ goto rollback;
	1795	+ }
	1796	+ return 0;
	1797	+
	1798	+rollback:
	1799	+ for_each_netdev_continue_reverse(net, dev)
	1800	+ call_netdevice_unregister_notifiers(nb, dev);
	1801	+ return err;
	1802	+}
	1803	+
	1804	+static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
	1805	+ struct net *net)
	1806	+{
	1807	+ struct net_device *dev;
	1808	+
	1809	+ for_each_netdev(net, dev)
	1810	+ call_netdevice_unregister_notifiers(nb, dev);
1613	1811	}
1614	1812
1615	1813	static int dev_boot_phase = 1;
..	..	@@ -1630,8 +1828,6 @@
1630	1828
1631	1829	int register_netdevice_notifier(struct notifier_block *nb)
1632	1830	{
1633		- struct net_device *dev;
1634		- struct net_device *last;
1635	1831	struct net *net;
1636	1832	int err;
1637	1833
..	..	@@ -1644,17 +1840,9 @@
1644	1840	if (dev_boot_phase)
1645	1841	goto unlock;
1646	1842	for_each_net(net) {
1647		- for_each_netdev(net, dev) {
1648		- err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1649		- err = notifier_to_errno(err);
1650		- if (err)
1651		- goto rollback;
1652		-
1653		- if (!(dev->flags & IFF_UP))
1654		- continue;
1655		-
1656		- call_netdevice_notifier(nb, NETDEV_UP, dev);
1657		- }
	1843	+ err = call_netdevice_register_net_notifiers(nb, net);
	1844	+ if (err)
	1845	+ goto rollback;
1658	1846	}
1659	1847
1660	1848	unlock:
..	..	@@ -1663,22 +1851,9 @@
1663	1851	return err;
1664	1852
1665	1853	rollback:
1666		- last = dev;
1667		- for_each_net(net) {
1668		- for_each_netdev(net, dev) {
1669		- if (dev == last)
1670		- goto outroll;
	1854	+ for_each_net_continue_reverse(net)
	1855	+ call_netdevice_unregister_net_notifiers(nb, net);
1671	1856
1672		- if (dev->flags & IFF_UP) {
1673		- call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1674		- dev);
1675		- call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1676		- }
1677		- call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1678		- }
1679		- }
1680		-
1681		-outroll:
1682	1857	raw_notifier_chain_unregister(&netdev_chain, nb);
1683	1858	goto unlock;
1684	1859	}
..	..	@@ -1700,7 +1875,6 @@
1700	1875
1701	1876	int unregister_netdevice_notifier(struct notifier_block *nb)
1702	1877	{
1703		- struct net_device *dev;
1704	1878	struct net *net;
1705	1879	int err;
1706	1880
..	..	@@ -1711,22 +1885,147 @@
1711	1885	if (err)
1712	1886	goto unlock;
1713	1887
1714		- for_each_net(net) {
1715		- for_each_netdev(net, dev) {
1716		- if (dev->flags & IFF_UP) {
1717		- call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1718		- dev);
1719		- call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1720		- }
1721		- call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1722		- }
1723		- }
	1888	+ for_each_net(net)
	1889	+ call_netdevice_unregister_net_notifiers(nb, net);
	1890	+
1724	1891	unlock:
1725	1892	rtnl_unlock();
1726	1893	up_write(&pernet_ops_rwsem);
1727	1894	return err;
1728	1895	}
1729	1896	EXPORT_SYMBOL(unregister_netdevice_notifier);
	1897	+
	1898	+static int __register_netdevice_notifier_net(struct net *net,
	1899	+ struct notifier_block *nb,
	1900	+ bool ignore_call_fail)
	1901	+{
	1902	+ int err;
	1903	+
	1904	+ err = raw_notifier_chain_register(&net->netdev_chain, nb);
	1905	+ if (err)
	1906	+ return err;
	1907	+ if (dev_boot_phase)
	1908	+ return 0;
	1909	+
	1910	+ err = call_netdevice_register_net_notifiers(nb, net);
	1911	+ if (err && !ignore_call_fail)
	1912	+ goto chain_unregister;
	1913	+
	1914	+ return 0;
	1915	+
	1916	+chain_unregister:
	1917	+ raw_notifier_chain_unregister(&net->netdev_chain, nb);
	1918	+ return err;
	1919	+}
	1920	+
	1921	+static int __unregister_netdevice_notifier_net(struct net *net,
	1922	+ struct notifier_block *nb)
	1923	+{
	1924	+ int err;
	1925	+
	1926	+ err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
	1927	+ if (err)
	1928	+ return err;
	1929	+
	1930	+ call_netdevice_unregister_net_notifiers(nb, net);
	1931	+ return 0;
	1932	+}
	1933	+
	1934	+/**
	1935	+ * register_netdevice_notifier_net - register a per-netns network notifier block
	1936	+ * @net: network namespace
	1937	+ * @nb: notifier
	1938	+ *
	1939	+ * Register a notifier to be called when network device events occur.
	1940	+ * The notifier passed is linked into the kernel structures and must
	1941	+ * not be reused until it has been unregistered. A negative errno code
	1942	+ * is returned on a failure.
	1943	+ *
	1944	+ * When registered all registration and up events are replayed
	1945	+ * to the new notifier to allow device to have a race free
	1946	+ * view of the network device list.
	1947	+ */
	1948	+
	1949	+int register_netdevice_notifier_net(struct net net, struct notifier_block nb)
	1950	+{
	1951	+ int err;
	1952	+
	1953	+ rtnl_lock();
	1954	+ err = __register_netdevice_notifier_net(net, nb, false);
	1955	+ rtnl_unlock();
	1956	+ return err;
	1957	+}
	1958	+EXPORT_SYMBOL(register_netdevice_notifier_net);
	1959	+
	1960	+/**
	1961	+ * unregister_netdevice_notifier_net - unregister a per-netns
	1962	+ * network notifier block
	1963	+ * @net: network namespace
	1964	+ * @nb: notifier
	1965	+ *
	1966	+ * Unregister a notifier previously registered by
	1967	+ * register_netdevice_notifier(). The notifier is unlinked into the
	1968	+ * kernel structures and may then be reused. A negative errno code
	1969	+ * is returned on a failure.
	1970	+ *
	1971	+ * After unregistering unregister and down device events are synthesized
	1972	+ * for all devices on the device list to the removed notifier to remove
	1973	+ * the need for special case cleanup code.
	1974	+ */
	1975	+
	1976	+int unregister_netdevice_notifier_net(struct net *net,
	1977	+ struct notifier_block *nb)
	1978	+{
	1979	+ int err;
	1980	+
	1981	+ rtnl_lock();
	1982	+ err = __unregister_netdevice_notifier_net(net, nb);
	1983	+ rtnl_unlock();
	1984	+ return err;
	1985	+}
	1986	+EXPORT_SYMBOL(unregister_netdevice_notifier_net);
	1987	+
	1988	+int register_netdevice_notifier_dev_net(struct net_device *dev,
	1989	+ struct notifier_block *nb,
	1990	+ struct netdev_net_notifier *nn)
	1991	+{
	1992	+ int err;
	1993	+
	1994	+ rtnl_lock();
	1995	+ err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
	1996	+ if (!err) {
	1997	+ nn->nb = nb;
	1998	+ list_add(&nn->list, &dev->net_notifier_list);
	1999	+ }
	2000	+ rtnl_unlock();
	2001	+ return err;
	2002	+}
	2003	+EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
	2004	+
	2005	+int unregister_netdevice_notifier_dev_net(struct net_device *dev,
	2006	+ struct notifier_block *nb,
	2007	+ struct netdev_net_notifier *nn)
	2008	+{
	2009	+ int err;
	2010	+
	2011	+ rtnl_lock();
	2012	+ list_del(&nn->list);
	2013	+ err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
	2014	+ rtnl_unlock();
	2015	+ return err;
	2016	+}
	2017	+EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
	2018	+
	2019	+static void move_netdevice_notifiers_dev_net(struct net_device *dev,
	2020	+ struct net *net)
	2021	+{
	2022	+ struct netdev_net_notifier *nn;
	2023	+
	2024	+ list_for_each_entry(nn, &dev->net_notifier_list, list) {
	2025	+ __unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
	2026	+ __register_netdevice_notifier_net(net, nn->nb, true);
	2027	+ }
	2028	+}
1730	2029
1731	2030	/**
1732	2031	* call_netdevice_notifiers_info - call all network notifier blocks
..	..	@@ -1740,8 +2039,31 @@
1740	2039	static int call_netdevice_notifiers_info(unsigned long val,
1741	2040	struct netdev_notifier_info *info)
1742	2041	{
	2042	+ struct net *net = dev_net(info->dev);
	2043	+ int ret;
	2044	+
1743	2045	ASSERT_RTNL();
	2046	+
	2047	+ /* Run per-netns notifier block chain first, then run the global one.
	2048	+ * Hopefully, one day, the global one is going to be removed after
	2049	+ * all notifier block registrators get converted to be per-netns.
	2050	+ */
	2051	+ ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
	2052	+ if (ret & NOTIFY_STOP_MASK)
	2053	+ return ret;
1744	2054	return raw_notifier_call_chain(&netdev_chain, val, info);
	2055	+}
	2056	+
	2057	+static int call_netdevice_notifiers_extack(unsigned long val,
	2058	+ struct net_device *dev,
	2059	+ struct netlink_ext_ack *extack)
	2060	+{
	2061	+ struct netdev_notifier_info info = {
	2062	+ .dev = dev,
	2063	+ .extack = extack,
	2064	+ };
	2065	+
	2066	+ return call_netdevice_notifiers_info(val, &info);
1745	2067	}
1746	2068
1747	2069	/**
..	..	@@ -1755,11 +2077,7 @@
1755	2077
1756	2078	int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1757	2079	{
1758		- struct netdev_notifier_info info = {
1759		- .dev = dev,
1760		- };
1761		-
1762		- return call_netdevice_notifiers_info(val, &info);
	2080	+ return call_netdevice_notifiers_extack(val, dev, NULL);
1763	2081	}
1764	2082	EXPORT_SYMBOL(call_netdevice_notifiers);
1765	2083
..	..	@@ -1987,6 +2305,17 @@
1987	2305	return false;
1988	2306	}
1989	2307
	2308	+/**
	2309	+ * dev_nit_active - return true if any network interface taps are in use
	2310	+ *
	2311	+ * @dev: network device to check for the presence of taps
	2312	+ */
	2313	+bool dev_nit_active(struct net_device *dev)
	2314	+{
	2315	+ return !list_empty(&ptype_all) \|\| !list_empty(&dev->ptype_all);
	2316	+}
	2317	+EXPORT_SYMBOL_GPL(dev_nit_active);
	2318	+
1990	2319	/*
1991	2320	* Support routine. Sends outgoing frames to any network
1992	2321	* taps currently in use.
..	..	@@ -2002,6 +2331,9 @@
2002	2331	rcu_read_lock();
2003	2332	again:
2004	2333	list_for_each_entry_rcu(ptype, ptype_list, list) {
	2334	+ if (ptype->ignore_outgoing)
	2335	+ continue;
	2336	+
2005	2337	/* Never send packets back to the socket
2006	2338	* they originated from - MvS (miquels@drinkel.ow.org)
2007	2339	*/
..	..	@@ -2723,6 +3055,7 @@
2723	3055	sd->output_queue_tailp = &q->next_sched;
2724	3056	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2725	3057	local_irq_restore(flags);
	3058	+ preempt_check_resched_rt();
2726	3059	}
2727	3060
2728	3061	void __netif_schedule(struct Qdisc *q)
..	..	@@ -2744,7 +3077,7 @@
2744	3077	void netif_schedule_queue(struct netdev_queue *txq)
2745	3078	{
2746	3079	rcu_read_lock();
2747		- if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
	3080	+ if (!netif_xmit_stopped(txq)) {
2748	3081	struct Qdisc *q = rcu_dereference(txq->qdisc);
2749	3082
2750	3083	__netif_schedule(q);
..	..	@@ -2785,6 +3118,7 @@
2785	3118	__this_cpu_write(softnet_data.completion_queue, skb);
2786	3119	raise_softirq_irqoff(NET_TX_SOFTIRQ);
2787	3120	local_irq_restore(flags);
	3121	+ preempt_check_resched_rt();
2788	3122	}
2789	3123	EXPORT_SYMBOL(__dev_kfree_skb_irq);
2790	3124
..	..	@@ -2881,12 +3215,10 @@
2881	3215	else
2882	3216	name = netdev_name(dev);
2883	3217	}
2884		- WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2885		- "gso_type=%d ip_summed=%d\n",
	3218	+ skb_dump(KERN_WARNING, skb, false);
	3219	+ WARN(1, "%s: caps=(%pNF, %pNF)\n",
2886	3220	name, dev ? &dev->features : &null_features,
2887		- skb->sk ? &skb->sk->sk_route_caps : &null_features,
2888		- skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2889		- skb_shinfo(skb)->gso_type, skb->ip_summed);
	3221	+ skb->sk ? &skb->sk->sk_route_caps : &null_features);
2890	3222	}
2891	3223
2892	3224	/*
..	..	@@ -2916,18 +3248,19 @@
2916	3248	}
2917	3249
2918	3250	offset = skb_checksum_start_offset(skb);
2919		- BUG_ON(offset >= skb_headlen(skb));
	3251	+ ret = -EINVAL;
	3252	+ if (WARN_ON_ONCE(offset >= skb_headlen(skb)))
	3253	+ goto out;
	3254	+
2920	3255	csum = skb_checksum(skb, offset, skb->len - offset, 0);
2921	3256
2922	3257	offset += skb->csum_offset;
2923		- BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
	3258	+ if (WARN_ON_ONCE(offset + sizeof(__sum16) > skb_headlen(skb)))
	3259	+ goto out;
2924	3260
2925		- if (skb_cloned(skb) &&
2926		- !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2927		- ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2928		- if (ret)
2929		- goto out;
2930		- }
	3261	+ ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
	3262	+ if (ret)
	3263	+ goto out;
2931	3264
2932	3265	(__sum16 )(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2933	3266	out_set_summed:
..	..	@@ -2962,12 +3295,11 @@
2962	3295	ret = -EINVAL;
2963	3296	goto out;
2964	3297	}
2965		- if (skb_cloned(skb) &&
2966		- !skb_clone_writable(skb, offset + sizeof(__le32))) {
2967		- ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2968		- if (ret)
2969		- goto out;
2970		- }
	3298	+
	3299	+ ret = skb_ensure_writable(skb, offset + sizeof(__le32));
	3300	+ if (ret)
	3301	+ goto out;
	3302	+
2971	3303	crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2972	3304	skb->len - start, ~(__u32)0,
2973	3305	crc32c_csum_stub));
..	..	@@ -3052,7 +3384,7 @@
3052	3384	* It may return NULL if the skb requires no segmentation. This is
3053	3385	* only possible when GSO is used for verifying header integrity.
3054	3386	*
3055		- * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
	3387	+ * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3056	3388	*/
3057	3389	struct sk_buff __skb_gso_segment(struct sk_buff skb,
3058	3390	netdev_features_t features, bool tx_path)
..	..	@@ -3081,7 +3413,7 @@
3081	3413	features &= ~NETIF_F_GSO_PARTIAL;
3082	3414	}
3083	3415
3084		- BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
	3416	+ BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3085	3417	sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3086	3418
3087	3419	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
..	..	@@ -3092,7 +3424,7 @@
3092	3424
3093	3425	segs = skb_mac_gso_segment(skb, features);
3094	3426
3095		- if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
	3427	+ if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3096	3428	skb_warn_bad_offload(skb);
3097	3429
3098	3430	return segs;
..	..	@@ -3101,10 +3433,11 @@
3101	3433
3102	3434	/* Take action when hardware reception checksum errors are detected. */
3103	3435	#ifdef CONFIG_BUG
3104		-void netdev_rx_csum_fault(struct net_device *dev)
	3436	+void netdev_rx_csum_fault(struct net_device dev, struct sk_buff skb)
3105	3437	{
3106	3438	if (net_ratelimit()) {
3107	3439	pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
	3440	+ skb_dump(KERN_ERR, skb, true);
3108	3441	dump_stack();
3109	3442	}
3110	3443	}
..	..	@@ -3154,10 +3487,9 @@
3154	3487	static netdev_features_t harmonize_features(struct sk_buff *skb,
3155	3488	netdev_features_t features)
3156	3489	{
3157		- int tmp;
3158	3490	__be16 type;
3159	3491
3160		- type = skb_network_protocol(skb, &tmp);
	3492	+ type = skb_network_protocol(skb, NULL);
3161	3493	features = net_mpls_features(skb, features, type);
3162	3494
3163	3495	if (skb->ip_summed != CHECKSUM_NONE &&
..	..	@@ -3254,10 +3586,11 @@
3254	3586	unsigned int len;
3255	3587	int rc;
3256	3588
3257		- if (!list_empty(&ptype_all) \|\| !list_empty(&dev->ptype_all))
	3589	+ if (dev_nit_active(dev))
3258	3590	dev_queue_xmit_nit(skb, dev);
3259	3591
3260	3592	len = skb->len;
	3593	+ PRANDOM_ADD_NOISE(skb, dev, txq, len + jiffies);
3261	3594	trace_net_dev_start_xmit(skb, dev);
3262	3595	rc = netdev_start_xmit(skb, dev, txq, more);
3263	3596	trace_net_dev_xmit(skb, rc, dev, len);
..	..	@@ -3274,7 +3607,7 @@
3274	3607	while (skb) {
3275	3608	struct sk_buff *next = skb->next;
3276	3609
3277		- skb->next = NULL;
	3610	+ skb_mark_not_on_list(skb);
3278	3611	rc = xmit_one(skb, dev, txq, next != NULL);
3279	3612	if (unlikely(!dev_xmit_complete(rc))) {
3280	3613	skb->next = next;
..	..	@@ -3374,7 +3707,7 @@
3374	3707
3375	3708	for (; skb != NULL; skb = next) {
3376	3709	next = skb->next;
3377		- skb->next = NULL;
	3710	+ skb_mark_not_on_list(skb);
3378	3711
3379	3712	/* in case skb wont be segmented, point to itself */
3380	3713	skb->prev = skb;
..	..	@@ -3405,7 +3738,7 @@
3405	3738	/* To get more precise estimation of bytes sent on wire,
3406	3739	* we add to pkt_len the headers size of all segments
3407	3740	*/
3408		- if (shinfo->gso_size) {
	3741	+ if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3409	3742	unsigned int hdr_len;
3410	3743	u16 gso_segs = shinfo->gso_segs;
3411	3744
..	..	@@ -3449,13 +3782,9 @@
3449	3782	qdisc_calculate_pkt_len(skb, q);
3450	3783
3451	3784	if (q->flags & TCQ_F_NOLOCK) {
3452		- if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3453		- __qdisc_drop(skb, &to_free);
3454		- rc = NET_XMIT_DROP;
3455		- } else {
3456		- rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
	3785	+ rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
	3786	+ if (likely(!netif_xmit_frozen_or_stopped(txq)))
3457	3787	qdisc_run(q);
3458		- }
3459	3788
3460	3789	if (unlikely(to_free))
3461	3790	kfree_skb_list(to_free);
..	..	@@ -3468,7 +3797,11 @@
3468	3797	* This permits qdisc->running owner to get the lock more
3469	3798	* often and dequeue packets faster.
3470	3799	*/
	3800	+#ifdef CONFIG_PREEMPT_RT
	3801	+ contended = true;
	3802	+#else
3471	3803	contended = qdisc_is_running(q);
	3804	+#endif
3472	3805	if (unlikely(contended))
3473	3806	spin_lock(&q->busylock);
3474	3807
..	..	@@ -3551,7 +3884,8 @@
3551	3884	skb_reset_mac_header(skb);
3552	3885	__skb_pull(skb, skb_network_offset(skb));
3553	3886	skb->pkt_type = PACKET_LOOPBACK;
3554		- skb->ip_summed = CHECKSUM_UNNECESSARY;
	3887	+ if (skb->ip_summed == CHECKSUM_NONE)
	3888	+ skb->ip_summed = CHECKSUM_UNNECESSARY;
3555	3889	WARN_ON(!skb_dst(skb));
3556	3890	skb_dst_force(skb);
3557	3891	netif_rx_ni(skb);
..	..	@@ -3570,6 +3904,7 @@
3570	3904	return skb;
3571	3905
3572	3906	/* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
	3907	+ qdisc_skb_cb(skb)->mru = 0;
3573	3908	mini_qdisc_bstats_cpu_update(miniq, skb);
3574	3909
3575	3910	switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
..	..	@@ -3670,23 +4005,21 @@
3670	4005	}
3671	4006
3672	4007	u16 dev_pick_tx_zero(struct net_device dev, struct sk_buff skb,
3673		- struct net_device *sb_dev,
3674		- select_queue_fallback_t fallback)
	4008	+ struct net_device *sb_dev)
3675	4009	{
3676	4010	return 0;
3677	4011	}
3678	4012	EXPORT_SYMBOL(dev_pick_tx_zero);
3679	4013
3680	4014	u16 dev_pick_tx_cpu_id(struct net_device dev, struct sk_buff skb,
3681		- struct net_device *sb_dev,
3682		- select_queue_fallback_t fallback)
	4015	+ struct net_device *sb_dev)
3683	4016	{
3684	4017	return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
3685	4018	}
3686	4019	EXPORT_SYMBOL(dev_pick_tx_cpu_id);
3687	4020
3688		-static u16 __netdev_pick_tx(struct net_device dev, struct sk_buff skb,
3689		- struct net_device *sb_dev)
	4021	+u16 netdev_pick_tx(struct net_device dev, struct sk_buff skb,
	4022	+ struct net_device *sb_dev)
3690	4023	{
3691	4024	struct sock *sk = skb->sk;
3692	4025	int queue_index = sk_tx_queue_get(sk);
..	..	@@ -3710,10 +4043,11 @@
3710	4043
3711	4044	return queue_index;
3712	4045	}
	4046	+EXPORT_SYMBOL(netdev_pick_tx);
3713	4047
3714		-struct netdev_queue netdev_pick_tx(struct net_device dev,
3715		- struct sk_buff *skb,
3716		- struct net_device *sb_dev)
	4048	+struct netdev_queue netdev_core_pick_tx(struct net_device dev,
	4049	+ struct sk_buff *skb,
	4050	+ struct net_device *sb_dev)
3717	4051	{
3718	4052	int queue_index = 0;
3719	4053
..	..	@@ -3728,10 +4062,9 @@
3728	4062	const struct net_device_ops *ops = dev->netdev_ops;
3729	4063
3730	4064	if (ops->ndo_select_queue)
3731		- queue_index = ops->ndo_select_queue(dev, skb, sb_dev,
3732		- __netdev_pick_tx);
	4065	+ queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
3733	4066	else
3734		- queue_index = __netdev_pick_tx(dev, skb, sb_dev);
	4067	+ queue_index = netdev_pick_tx(dev, skb, sb_dev);
3735	4068
3736	4069	queue_index = netdev_cap_txqueue(dev, queue_index);
3737	4070	}
..	..	@@ -3775,6 +4108,7 @@
3775	4108	bool again = false;
3776	4109
3777	4110	skb_reset_mac_header(skb);
	4111	+ skb_assert_len(skb);
3778	4112
3779	4113	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3780	4114	__skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
..	..	@@ -3805,7 +4139,7 @@
3805	4139	else
3806	4140	skb_dst_force(skb);
3807	4141
3808		- txq = netdev_pick_tx(dev, skb, sb_dev);
	4142	+ txq = netdev_core_pick_tx(dev, skb, sb_dev);
3809	4143	q = rcu_dereference_bh(txq->qdisc);
3810	4144
3811	4145	trace_net_dev_queue(skb);
..	..	@@ -3840,6 +4174,7 @@
3840	4174	if (!skb)
3841	4175	goto out;
3842	4176
	4177	+ PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
3843	4178	HARD_TX_LOCK(dev, txq, cpu);
3844	4179
3845	4180	if (!netif_xmit_stopped(txq)) {
..	..	@@ -3887,7 +4222,7 @@
3887	4222	}
3888	4223	EXPORT_SYMBOL(dev_queue_xmit_accel);
3889	4224
3890		-int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
	4225	+int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3891	4226	{
3892	4227	struct net_device *dev = skb->dev;
3893	4228	struct sk_buff *orig_skb = skb;
..	..	@@ -3905,6 +4240,7 @@
3905	4240
3906	4241	skb_set_queue_mapping(skb, queue_id);
3907	4242	txq = skb_get_tx_queue(dev, skb);
	4243	+ PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
3908	4244
3909	4245	local_bh_disable();
3910	4246
..	..	@@ -3916,17 +4252,13 @@
3916	4252	dev_xmit_recursion_dec();
3917	4253
3918	4254	local_bh_enable();
3919		-
3920		- if (!dev_xmit_complete(ret))
3921		- kfree_skb(skb);
3922		-
3923	4255	return ret;
3924	4256	drop:
3925	4257	atomic_long_inc(&dev->tx_dropped);
3926	4258	kfree_skb_list(skb);
3927	4259	return NET_XMIT_DROP;
3928	4260	}
3929		-EXPORT_SYMBOL(dev_direct_xmit);
	4261	+EXPORT_SYMBOL(__dev_direct_xmit);
3930	4262
3931	4263	/*************************************************************************
3932	4264	* Receiver routines
..	..	@@ -3944,6 +4276,8 @@
3944	4276	int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
3945	4277	int dev_rx_weight __read_mostly = 64;
3946	4278	int dev_tx_weight __read_mostly = 64;
	4279	+/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
	4280	+int gro_normal_batch __read_mostly = 8;
3947	4281
3948	4282	/* Called with irq disabled */
3949	4283	static inline void ____napi_schedule(struct softnet_data *sd,
..	..	@@ -3961,9 +4295,9 @@
3961	4295	u32 rps_cpu_mask __read_mostly;
3962	4296	EXPORT_SYMBOL(rps_cpu_mask);
3963	4297
3964		-struct static_key rps_needed __read_mostly;
	4298	+struct static_key_false rps_needed __read_mostly;
3965	4299	EXPORT_SYMBOL(rps_needed);
3966		-struct static_key rfs_needed __read_mostly;
	4300	+struct static_key_false rfs_needed __read_mostly;
3967	4301	EXPORT_SYMBOL(rfs_needed);
3968	4302
3969	4303	static struct rps_dev_flow *
..	..	@@ -4194,7 +4528,7 @@
4194	4528	struct softnet_data *sd;
4195	4529	unsigned int old_flow, new_flow;
4196	4530
4197		- if (qlen < (netdev_max_backlog >> 1))
	4531	+ if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))
4198	4532	return false;
4199	4533
4200	4534	sd = this_cpu_ptr(&softnet_data);
..	..	@@ -4242,7 +4576,7 @@
4242	4576	if (!netif_running(skb->dev))
4243	4577	goto drop;
4244	4578	qlen = skb_queue_len(&sd->input_pkt_queue);
4245		- if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
	4579	+ if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {
4246	4580	if (qlen) {
4247	4581	enqueue:
4248	4582	__skb_queue_tail(&sd->input_pkt_queue, skb);
..	..	@@ -4267,6 +4601,7 @@
4267	4601	rps_unlock(sd);
4268	4602
4269	4603	local_irq_restore(flags);
	4604	+ preempt_check_resched_rt();
4270	4605
4271	4606	atomic_long_inc(&skb->dev->rx_dropped);
4272	4607	kfree_skb(skb);
..	..	@@ -4312,7 +4647,7 @@
4312	4647	/* Reinjected packets coming from act_mirred or similar should
4313	4648	* not get XDP generic processing.
4314	4649	*/
4315		- if (skb_is_tc_redirected(skb))
	4650	+ if (skb_is_redirected(skb))
4316	4651	return XDP_PASS;
4317	4652
4318	4653	/* XDP packets must be linear and must have sufficient headroom
..	..	@@ -4344,6 +4679,11 @@
4344	4679	xdp->data_meta = xdp->data;
4345	4680	xdp->data_end = xdp->data + hlen;
4346	4681	xdp->data_hard_start = skb->data - skb_headroom(skb);
	4682	+
	4683	+ /* SKB "head" area always have tailroom for skb_shared_info */
	4684	+ xdp->frame_sz = (void *)skb_end_pointer(skb) - xdp->data_hard_start;
	4685	+ xdp->frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	4686	+
4347	4687	orig_data_end = xdp->data_end;
4348	4688	orig_data = xdp->data;
4349	4689	eth = (struct ethhdr *)xdp->data;
..	..	@@ -4367,14 +4707,11 @@
4367	4707	skb_reset_network_header(skb);
4368	4708	}
4369	4709
4370		- /* check if bpf_xdp_adjust_tail was used. it can only "shrink"
4371		- * pckt.
4372		- */
4373		- off = orig_data_end - xdp->data_end;
	4710	+ /* check if bpf_xdp_adjust_tail was used */
	4711	+ off = xdp->data_end - orig_data_end;
4374	4712	if (off != 0) {
4375	4713	skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4376		- skb->len -= off;
4377		-
	4714	+ skb->len += off; /* positive on grow, negative on shrink */
4378	4715	}
4379	4716
4380	4717	/* check if XDP changed eth hdr such SKB needs update */
..	..	@@ -4397,10 +4734,10 @@
4397	4734	break;
4398	4735	default:
4399	4736	bpf_warn_invalid_xdp_action(act);
4400		- /* fall through */
	4737	+ fallthrough;
4401	4738	case XDP_ABORTED:
4402	4739	trace_xdp_exception(skb->dev, xdp_prog, act);
4403		- /* fall through */
	4740	+ fallthrough;
4404	4741	case XDP_DROP:
4405	4742	do_drop:
4406	4743	kfree_skb(skb);
..	..	@@ -4420,7 +4757,7 @@
4420	4757	bool free_skb = true;
4421	4758	int cpu, rc;
4422	4759
4423		- txq = netdev_pick_tx(dev, skb, NULL);
	4760	+ txq = netdev_core_pick_tx(dev, skb, NULL);
4424	4761	cpu = smp_processor_id();
4425	4762	HARD_TX_LOCK(dev, txq, cpu);
4426	4763	if (!netif_xmit_stopped(txq)) {
..	..	@@ -4434,7 +4771,6 @@
4434	4771	kfree_skb(skb);
4435	4772	}
4436	4773	}
4437		-EXPORT_SYMBOL_GPL(generic_xdp_tx);
4438	4774
4439	4775	static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4440	4776
..	..	@@ -4472,16 +4808,16 @@
4472	4808	{
4473	4809	int ret;
4474	4810
4475		- net_timestamp_check(netdev_tstamp_prequeue, skb);
	4811	+ net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
4476	4812
4477	4813	trace_netif_rx(skb);
4478	4814
4479	4815	#ifdef CONFIG_RPS
4480		- if (static_key_false(&rps_needed)) {
	4816	+ if (static_branch_unlikely(&rps_needed)) {
4481	4817	struct rps_dev_flow voidflow, *rflow = &voidflow;
4482	4818	int cpu;
4483	4819
4484		- preempt_disable();
	4820	+ migrate_disable();
4485	4821	rcu_read_lock();
4486	4822
4487	4823	cpu = get_rps_cpu(skb->dev, skb, &rflow);
..	..	@@ -4491,14 +4827,14 @@
4491	4827	ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4492	4828
4493	4829	rcu_read_unlock();
4494		- preempt_enable();
	4830	+ migrate_enable();
4495	4831	} else
4496	4832	#endif
4497	4833	{
4498	4834	unsigned int qtail;
4499	4835
4500		- ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4501		- put_cpu();
	4836	+ ret = enqueue_to_backlog(skb, get_cpu_light(), &qtail);
	4837	+ put_cpu_light();
4502	4838	}
4503	4839	return ret;
4504	4840	}
..	..	@@ -4520,9 +4856,14 @@
4520	4856
4521	4857	int netif_rx(struct sk_buff *skb)
4522	4858	{
	4859	+ int ret;
	4860	+
4523	4861	trace_netif_rx_entry(skb);
4524	4862
4525		- return netif_rx_internal(skb);
	4863	+ ret = netif_rx_internal(skb);
	4864	+ trace_netif_rx_exit(ret);
	4865	+
	4866	+ return ret;
4526	4867	}
4527	4868	EXPORT_SYMBOL(netif_rx);
4528	4869
..	..	@@ -4532,15 +4873,29 @@
4532	4873
4533	4874	trace_netif_rx_ni_entry(skb);
4534	4875
4535		- preempt_disable();
	4876	+ local_bh_disable();
4536	4877	err = netif_rx_internal(skb);
4537		- if (local_softirq_pending())
4538		- do_softirq();
4539		- preempt_enable();
	4878	+ local_bh_enable();
	4879	+ trace_netif_rx_ni_exit(err);
4540	4880
4541	4881	return err;
4542	4882	}
4543	4883	EXPORT_SYMBOL(netif_rx_ni);
	4884	+
	4885	+int netif_rx_any_context(struct sk_buff *skb)
	4886	+{
	4887	+ /*
	4888	+ * If invoked from contexts which do not invoke bottom half
	4889	+ * processing either at return from interrupt or when softrqs are
	4890	+ * reenabled, use netif_rx_ni() which invokes bottomhalf processing
	4891	+ * directly.
	4892	+ */
	4893	+ if (in_interrupt())
	4894	+ return netif_rx(skb);
	4895	+ else
	4896	+ return netif_rx_ni(skb);
	4897	+}
	4898	+EXPORT_SYMBOL(netif_rx_any_context);
4544	4899
4545	4900	static __latent_entropy void net_tx_action(struct softirq_action *h)
4546	4901	{
..	..	@@ -4583,25 +4938,43 @@
4583	4938	sd->output_queue_tailp = &sd->output_queue;
4584	4939	local_irq_enable();
4585	4940
	4941	+ rcu_read_lock();
	4942	+
4586	4943	while (head) {
4587	4944	struct Qdisc *q = head;
4588	4945	spinlock_t *root_lock = NULL;
4589	4946
4590	4947	head = head->next_sched;
4591	4948
4592		- if (!(q->flags & TCQ_F_NOLOCK)) {
4593		- root_lock = qdisc_lock(q);
4594		- spin_lock(root_lock);
4595		- }
4596	4949	/* We need to make sure head->next_sched is read
4597	4950	* before clearing __QDISC_STATE_SCHED
4598	4951	*/
4599	4952	smp_mb__before_atomic();
	4953	+
	4954	+ if (!(q->flags & TCQ_F_NOLOCK)) {
	4955	+ root_lock = qdisc_lock(q);
	4956	+ spin_lock(root_lock);
	4957	+ } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
	4958	+ &q->state))) {
	4959	+ /* There is a synchronize_net() between
	4960	+ * STATE_DEACTIVATED flag being set and
	4961	+ * qdisc_reset()/some_qdisc_is_busy() in
	4962	+ * dev_deactivate(), so we can safely bail out
	4963	+ * early here to avoid data race between
	4964	+ * qdisc_deactivate() and some_qdisc_is_busy()
	4965	+ * for lockless qdisc.
	4966	+ */
	4967	+ clear_bit(__QDISC_STATE_SCHED, &q->state);
	4968	+ continue;
	4969	+ }
	4970	+
4600	4971	clear_bit(__QDISC_STATE_SCHED, &q->state);
4601	4972	qdisc_run(q);
4602	4973	if (root_lock)
4603	4974	spin_unlock(root_lock);
4604	4975	}
	4976	+
	4977	+ rcu_read_unlock();
4605	4978	}
4606	4979
4607	4980	xfrm_dev_backlog(sd);
..	..	@@ -4616,7 +4989,7 @@
4616	4989
4617	4990	static inline struct sk_buff *
4618	4991	sch_handle_ingress(struct sk_buff skb, struct packet_type pt_prev, int ret,
4619		- struct net_device *orig_dev)
	4992	+ struct net_device orig_dev, bool another)
4620	4993	{
4621	4994	#ifdef CONFIG_NET_CLS_ACT
4622	4995	struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
..	..	@@ -4636,10 +5009,12 @@
4636	5009	}
4637	5010
4638	5011	qdisc_skb_cb(skb)->pkt_len = skb->len;
	5012	+ qdisc_skb_cb(skb)->mru = 0;
4639	5013	skb->tc_at_ingress = 1;
4640	5014	mini_qdisc_bstats_cpu_update(miniq, skb);
4641	5015
4642		- switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
	5016	+ switch (tcf_classify_ingress(skb, miniq->block, miniq->filter_list,
	5017	+ &cl_res, false)) {
4643	5018	case TC_ACT_OK:
4644	5019	case TC_ACT_RECLASSIFY:
4645	5020	skb->tc_index = TC_H_MIN(cl_res.classid);
..	..	@@ -4659,11 +5034,13 @@
4659	5034	* redirecting to another netdev
4660	5035	*/
4661	5036	__skb_push(skb, skb->mac_len);
4662		- skb_do_redirect(skb);
	5037	+ if (skb_do_redirect(skb) == -EAGAIN) {
	5038	+ __skb_pull(skb, skb->mac_len);
	5039	+ *another = true;
	5040	+ break;
	5041	+ }
4663	5042	return NULL;
4664		- case TC_ACT_REINSERT:
4665		- /* this does not scrub the packet, and updates stats on error */
4666		- skb_tc_reinsert(skb, &cl_res);
	5043	+ case TC_ACT_CONSUMED:
4667	5044	return NULL;
4668	5045	default:
4669	5046	break;
..	..	@@ -4763,7 +5140,6 @@
4763	5140	static inline int nf_ingress(struct sk_buff skb, struct packet_type *pt_prev,
4764	5141	int ret, struct net_device orig_dev)
4765	5142	{
4766		-#ifdef CONFIG_NETFILTER_INGRESS
4767	5143	if (nf_hook_ingress_active(skb)) {
4768	5144	int ingress_retval;
4769	5145
..	..	@@ -4777,7 +5153,6 @@
4777	5153	rcu_read_unlock();
4778	5154	return ingress_retval;
4779	5155	}
4780		-#endif /* CONFIG_NETFILTER_INGRESS */
4781	5156	return 0;
4782	5157	}
4783	5158
..	..	@@ -4792,7 +5167,7 @@
4792	5167	int ret = NET_RX_DROP;
4793	5168	__be16 type;
4794	5169
4795		- net_timestamp_check(!netdev_tstamp_prequeue, skb);
	5170	+ net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb);
4796	5171
4797	5172	trace_netif_receive_skb(skb);
4798	5173
..	..	@@ -4852,7 +5227,12 @@
4852	5227	skip_taps:
4853	5228	#ifdef CONFIG_NET_INGRESS
4854	5229	if (static_branch_unlikely(&ingress_needed_key)) {
4855		- skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
	5230	+ bool another = false;
	5231	+
	5232	+ skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
	5233	+ &another);
	5234	+ if (another)
	5235	+ goto another_round;
4856	5236	if (!skb)
4857	5237	goto out;
4858	5238
..	..	@@ -4860,7 +5240,7 @@
4860	5240	goto out;
4861	5241	}
4862	5242	#endif
4863		- skb_reset_tc(skb);
	5243	+ skb_reset_redirect(skb);
4864	5244	skip_classify:
4865	5245	if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4866	5246	goto drop;
..	..	@@ -4897,14 +5277,42 @@
4897	5277	}
4898	5278	}
4899	5279
4900		- if (unlikely(skb_vlan_tag_present(skb))) {
4901		- if (skb_vlan_tag_get_id(skb))
	5280	+ if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
	5281	+check_vlan_id:
	5282	+ if (skb_vlan_tag_get_id(skb)) {
	5283	+ /* Vlan id is non 0 and vlan_do_receive() above couldn't
	5284	+ * find vlan device.
	5285	+ */
4902	5286	skb->pkt_type = PACKET_OTHERHOST;
	5287	+ } else if (skb->protocol == cpu_to_be16(ETH_P_8021Q) \|\|
	5288	+ skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
	5289	+ /* Outer header is 802.1P with vlan 0, inner header is
	5290	+ * 802.1Q or 802.1AD and vlan_do_receive() above could
	5291	+ * not find vlan dev for vlan id 0.
	5292	+ */
	5293	+ __vlan_hwaccel_clear_tag(skb);
	5294	+ skb = skb_vlan_untag(skb);
	5295	+ if (unlikely(!skb))
	5296	+ goto out;
	5297	+ if (vlan_do_receive(&skb))
	5298	+ /* After stripping off 802.1P header with vlan 0
	5299	+ * vlan dev is found for inner header.
	5300	+ */
	5301	+ goto another_round;
	5302	+ else if (unlikely(!skb))
	5303	+ goto out;
	5304	+ else
	5305	+ /* We have stripped outer 802.1P vlan 0 header.
	5306	+ * But could not find vlan dev.
	5307	+ * check again for vlan id to set OTHERHOST.
	5308	+ */
	5309	+ goto check_vlan_id;
	5310	+ }
4903	5311	/* Note: we might in the future use prio bits
4904	5312	* and set skb->priority like in vlan_do_receive()
4905	5313	* For the time being, just ignore Priority Code Point
4906	5314	*/
4907		- skb->vlan_tci = 0;
	5315	+ __vlan_hwaccel_clear_tag(skb);
4908	5316	}
4909	5317
4910	5318	type = skb->protocol;
..	..	@@ -4960,7 +5368,8 @@
4960	5368
4961	5369	ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
4962	5370	if (pt_prev)
4963		- ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
	5371	+ ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
	5372	+ skb->dev, pt_prev, orig_dev);
4964	5373	return ret;
4965	5374	}
4966	5375
..	..	@@ -4970,7 +5379,7 @@
4970	5379	*
4971	5380	* More direct receive version of netif_receive_skb(). It should
4972	5381	* only be used by callers that have a need to skip RPS and Generic XDP.
4973		- * Caller must also take care of handling if (page_is_)pfmemalloc.
	5382	+ * Caller must also take care of handling if ``(page_is_)pfmemalloc``.
4974	5383	*
4975	5384	* This function may only be called from softirq context and interrupts
4976	5385	* should be enabled.
..	..	@@ -5002,7 +5411,8 @@
5002	5411	if (list_empty(head))
5003	5412	return;
5004	5413	if (pt_prev->list_func != NULL)
5005		- pt_prev->list_func(head, pt_prev, orig_dev);
	5414	+ INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
	5415	+ ip_list_rcv, head, pt_prev, orig_dev);
5006	5416	else
5007	5417	list_for_each_entry_safe(skb, next, head, list) {
5008	5418	skb_list_del_init(skb);
..	..	@@ -5113,6 +5523,25 @@
5113	5523	struct bpf_prog *new = xdp->prog;
5114	5524	int ret = 0;
5115	5525
	5526	+ if (new) {
	5527	+ u32 i;
	5528	+
	5529	+ mutex_lock(&new->aux->used_maps_mutex);
	5530	+
	5531	+ /* generic XDP does not work with DEVMAPs that can
	5532	+ * have a bpf_prog installed on an entry
	5533	+ */
	5534	+ for (i = 0; i < new->aux->used_map_cnt; i++) {
	5535	+ if (dev_map_can_have_prog(new->aux->used_maps[i]) \|\|
	5536	+ cpu_map_prog_allowed(new->aux->used_maps[i])) {
	5537	+ mutex_unlock(&new->aux->used_maps_mutex);
	5538	+ return -EINVAL;
	5539	+ }
	5540	+ }
	5541	+
	5542	+ mutex_unlock(&new->aux->used_maps_mutex);
	5543	+ }
	5544	+
5116	5545	switch (xdp->command) {
5117	5546	case XDP_SETUP_PROG:
5118	5547	rcu_assign_pointer(dev->xdp_prog, new);
..	..	@@ -5128,10 +5557,6 @@
5128	5557	}
5129	5558	break;
5130	5559
5131		- case XDP_QUERY_PROG:
5132		- xdp->prog_id = old ? old->aux->id : 0;
5133		- break;
5134		-
5135	5560	default:
5136	5561	ret = -EINVAL;
5137	5562	break;
..	..	@@ -5144,14 +5569,14 @@
5144	5569	{
5145	5570	int ret;
5146	5571
5147		- net_timestamp_check(netdev_tstamp_prequeue, skb);
	5572	+ net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5148	5573
5149	5574	if (skb_defer_rx_timestamp(skb))
5150	5575	return NET_RX_SUCCESS;
5151	5576
5152	5577	rcu_read_lock();
5153	5578	#ifdef CONFIG_RPS
5154		- if (static_key_false(&rps_needed)) {
	5579	+ if (static_branch_unlikely(&rps_needed)) {
5155	5580	struct rps_dev_flow voidflow, *rflow = &voidflow;
5156	5581	int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5157	5582
..	..	@@ -5174,7 +5599,7 @@
5174	5599
5175	5600	INIT_LIST_HEAD(&sublist);
5176	5601	list_for_each_entry_safe(skb, next, head, list) {
5177		- net_timestamp_check(netdev_tstamp_prequeue, skb);
	5602	+ net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5178	5603	skb_list_del_init(skb);
5179	5604	if (!skb_defer_rx_timestamp(skb))
5180	5605	list_add_tail(&skb->list, &sublist);
..	..	@@ -5183,7 +5608,7 @@
5183	5608
5184	5609	rcu_read_lock();
5185	5610	#ifdef CONFIG_RPS
5186		- if (static_key_false(&rps_needed)) {
	5611	+ if (static_branch_unlikely(&rps_needed)) {
5187	5612	list_for_each_entry_safe(skb, next, head, list) {
5188	5613	struct rps_dev_flow voidflow, *rflow = &voidflow;
5189	5614	int cpu = get_rps_cpu(skb->dev, skb, &rflow);
..	..	@@ -5217,9 +5642,14 @@
5217	5642	*/
5218	5643	int netif_receive_skb(struct sk_buff *skb)
5219	5644	{
	5645	+ int ret;
	5646	+
5220	5647	trace_netif_receive_skb_entry(skb);
5221	5648
5222		- return netif_receive_skb_internal(skb);
	5649	+ ret = netif_receive_skb_internal(skb);
	5650	+ trace_netif_receive_skb_exit(ret);
	5651	+
	5652	+ return ret;
5223	5653	}
5224	5654	EXPORT_SYMBOL(netif_receive_skb);
5225	5655
..	..	@@ -5239,13 +5669,16 @@
5239	5669
5240	5670	if (list_empty(head))
5241	5671	return;
5242		- list_for_each_entry(skb, head, list)
5243		- trace_netif_receive_skb_list_entry(skb);
	5672	+ if (trace_netif_receive_skb_list_entry_enabled()) {
	5673	+ list_for_each_entry(skb, head, list)
	5674	+ trace_netif_receive_skb_list_entry(skb);
	5675	+ }
5244	5676	netif_receive_skb_list_internal(head);
	5677	+ trace_netif_receive_skb_list_exit(0);
5245	5678	}
5246	5679	EXPORT_SYMBOL(netif_receive_skb_list);
5247	5680
5248		-DEFINE_PER_CPU(struct work_struct, flush_works);
	5681	+static DEFINE_PER_CPU(struct work_struct, flush_works);
5249	5682
5250	5683	/* Network device is going away, flush any packets still pending */
5251	5684	static void flush_backlog(struct work_struct *work)
..	..	@@ -5278,23 +5711,89 @@
5278	5711	local_bh_enable();
5279	5712	}
5280	5713
	5714	+static bool flush_required(int cpu)
	5715	+{
	5716	+#if IS_ENABLED(CONFIG_RPS)
	5717	+ struct softnet_data *sd = &per_cpu(softnet_data, cpu);
	5718	+ bool do_flush;
	5719	+
	5720	+ local_irq_disable();
	5721	+ rps_lock(sd);
	5722	+
	5723	+ /* as insertion into process_queue happens with the rps lock held,
	5724	+ * process_queue access may race only with dequeue
	5725	+ */
	5726	+ do_flush = !skb_queue_empty(&sd->input_pkt_queue) \|\|
	5727	+ !skb_queue_empty_lockless(&sd->process_queue);
	5728	+ rps_unlock(sd);
	5729	+ local_irq_enable();
	5730	+
	5731	+ return do_flush;
	5732	+#endif
	5733	+ /* without RPS we can't safely check input_pkt_queue: during a
	5734	+ * concurrent remote skb_queue_splice() we can detect as empty both
	5735	+ * input_pkt_queue and process_queue even if the latter could end-up
	5736	+ * containing a lot of packets.
	5737	+ */
	5738	+ return true;
	5739	+}
	5740	+
5281	5741	static void flush_all_backlogs(void)
5282	5742	{
	5743	+ static cpumask_t flush_cpus;
5283	5744	unsigned int cpu;
	5745	+
	5746	+ /* since we are under rtnl lock protection we can use static data
	5747	+ * for the cpumask and avoid allocating on stack the possibly
	5748	+ * large mask
	5749	+ */
	5750	+ ASSERT_RTNL();
5284	5751
5285	5752	get_online_cpus();
5286	5753
5287		- for_each_online_cpu(cpu)
5288		- queue_work_on(cpu, system_highpri_wq,
5289		- per_cpu_ptr(&flush_works, cpu));
	5754	+ cpumask_clear(&flush_cpus);
	5755	+ for_each_online_cpu(cpu) {
	5756	+ if (flush_required(cpu)) {
	5757	+ queue_work_on(cpu, system_highpri_wq,
	5758	+ per_cpu_ptr(&flush_works, cpu));
	5759	+ cpumask_set_cpu(cpu, &flush_cpus);
	5760	+ }
	5761	+ }
5290	5762
5291		- for_each_online_cpu(cpu)
	5763	+ /* we can have in flight packet[s] on the cpus we are not flushing,
	5764	+ * synchronize_net() in unregister_netdevice_many() will take care of
	5765	+ * them
	5766	+ */
	5767	+ for_each_cpu(cpu, &flush_cpus)
5292	5768	flush_work(per_cpu_ptr(&flush_works, cpu));
5293	5769
5294	5770	put_online_cpus();
5295	5771	}
5296	5772
5297		-static int napi_gro_complete(struct sk_buff *skb)
	5773	+/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
	5774	+static void gro_normal_list(struct napi_struct *napi)
	5775	+{
	5776	+ if (!napi->rx_count)
	5777	+ return;
	5778	+ netif_receive_skb_list_internal(&napi->rx_list);
	5779	+ INIT_LIST_HEAD(&napi->rx_list);
	5780	+ napi->rx_count = 0;
	5781	+}
	5782	+
	5783	+/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
	5784	+ * pass the whole batch up to the stack.
	5785	+ */
	5786	+static void gro_normal_one(struct napi_struct napi, struct sk_buff skb, int segs)
	5787	+{
	5788	+ list_add_tail(&skb->list, &napi->rx_list);
	5789	+ napi->rx_count += segs;
	5790	+ if (napi->rx_count >= gro_normal_batch)
	5791	+ gro_normal_list(napi);
	5792	+}
	5793	+
	5794	+INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
	5795	+INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
	5796	+static int napi_gro_complete(struct napi_struct napi, struct sk_buff skb)
5298	5797	{
5299	5798	struct packet_offload *ptype;
5300	5799	__be16 type = skb->protocol;
..	..	@@ -5313,7 +5812,9 @@
5313	5812	if (ptype->type != type \|\| !ptype->callbacks.gro_complete)
5314	5813	continue;
5315	5814
5316		- err = ptype->callbacks.gro_complete(skb, 0);
	5815	+ err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
	5816	+ ipv6_gro_complete, inet_gro_complete,
	5817	+ skb, 0);
5317	5818	break;
5318	5819	}
5319	5820	rcu_read_unlock();
..	..	@@ -5325,7 +5826,8 @@
5325	5826	}
5326	5827
5327	5828	out:
5328		- return netif_receive_skb_internal(skb);
	5829	+ gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
	5830	+ return NET_RX_SUCCESS;
5329	5831	}
5330	5832
5331	5833	static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
..	..	@@ -5337,9 +5839,8 @@
5337	5839	list_for_each_entry_safe_reverse(skb, p, head, list) {
5338	5840	if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
5339	5841	return;
5340		- list_del(&skb->list);
5341		- skb->next = NULL;
5342		- napi_gro_complete(skb);
	5842	+ skb_list_del_init(skb);
	5843	+ napi_gro_complete(napi, skb);
5343	5844	napi->gro_hash[index].count--;
5344	5845	}
5345	5846
..	..	@@ -5353,11 +5854,13 @@
5353	5854	*/
5354	5855	void napi_gro_flush(struct napi_struct *napi, bool flush_old)
5355	5856	{
5356		- u32 i;
	5857	+ unsigned long bitmask = napi->gro_bitmask;
	5858	+ unsigned int i, base = ~0U;
5357	5859
5358		- for (i = 0; i < GRO_HASH_BUCKETS; i++) {
5359		- if (test_bit(i, &napi->gro_bitmask))
5360		- __napi_gro_flush_chain(napi, i, flush_old);
	5860	+ while ((i = ffs(bitmask)) != 0) {
	5861	+ bitmask >>= i;
	5862	+ base += i;
	5863	+ __napi_gro_flush_chain(napi, base, flush_old);
5361	5864	}
5362	5865	}
5363	5866	EXPORT_SYMBOL(napi_gro_flush);
..	..	@@ -5382,7 +5885,9 @@
5382	5885	}
5383	5886
5384	5887	diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
5385		- diffs \|= p->vlan_tci ^ skb->vlan_tci;
	5888	+ diffs \|= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
	5889	+ if (skb_vlan_tag_present(p))
	5890	+ diffs \|= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);
5386	5891	diffs \|= skb_metadata_dst_cmp(p, skb);
5387	5892	diffs \|= skb_metadata_differs(p, skb);
5388	5893	if (maclen == ETH_HLEN)
..	..	@@ -5392,13 +5897,26 @@
5392	5897	diffs = memcmp(skb_mac_header(p),
5393	5898	skb_mac_header(skb),
5394	5899	maclen);
	5900	+
	5901	+ diffs \|= skb_get_nfct(p) ^ skb_get_nfct(skb);
	5902	+#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
	5903	+ if (!diffs) {
	5904	+ struct tc_skb_ext *skb_ext = skb_ext_find(skb, TC_SKB_EXT);
	5905	+ struct tc_skb_ext *p_ext = skb_ext_find(p, TC_SKB_EXT);
	5906	+
	5907	+ diffs \|= (!!p_ext) ^ (!!skb_ext);
	5908	+ if (!diffs && unlikely(skb_ext))
	5909	+ diffs \|= p_ext->chain ^ skb_ext->chain;
	5910	+ }
	5911	+#endif
	5912	+
5395	5913	NAPI_GRO_CB(p)->same_flow = !diffs;
5396	5914	}
5397	5915
5398	5916	return head;
5399	5917	}
5400	5918
5401		-static void skb_gro_reset_offset(struct sk_buff *skb)
	5919	+static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
5402	5920	{
5403	5921	const struct skb_shared_info *pinfo = skb_shinfo(skb);
5404	5922	const skb_frag_t *frag0 = &pinfo->frags[0];
..	..	@@ -5407,10 +5925,9 @@
5407	5925	NAPI_GRO_CB(skb)->frag0 = NULL;
5408	5926	NAPI_GRO_CB(skb)->frag0_len = 0;
5409	5927
5410		- if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
5411		- pinfo->nr_frags &&
	5928	+ if (!skb_headlen(skb) && pinfo->nr_frags &&
5412	5929	!PageHighMem(skb_frag_page(frag0)) &&
5413		- (!NET_IP_ALIGN \|\| !(skb_frag_off(frag0) & 3))) {
	5930	+ (!NET_IP_ALIGN \|\| !((skb_frag_off(frag0) + nhoff) & 3))) {
5414	5931	NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
5415	5932	NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
5416	5933	skb_frag_size(frag0),
..	..	@@ -5429,7 +5946,7 @@
5429	5946	skb->data_len -= grow;
5430	5947	skb->tail += grow;
5431	5948
5432		- pinfo->frags[0].page_offset += grow;
	5949	+ skb_frag_off_add(&pinfo->frags[0], grow);
5433	5950	skb_frag_size_sub(&pinfo->frags[0], grow);
5434	5951
5435	5952	if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
..	..	@@ -5439,7 +5956,7 @@
5439	5956	}
5440	5957	}
5441	5958
5442		-static void gro_flush_oldest(struct list_head *head)
	5959	+static void gro_flush_oldest(struct napi_struct napi, struct list_head head)
5443	5960	{
5444	5961	struct sk_buff *oldest;
5445	5962
..	..	@@ -5454,11 +5971,14 @@
5454	5971	/* Do not adjust napi->gro_hash[].count, caller is adding a new
5455	5972	* SKB to the chain.
5456	5973	*/
5457		- list_del(&oldest->list);
5458		- oldest->next = NULL;
5459		- napi_gro_complete(oldest);
	5974	+ skb_list_del_init(oldest);
	5975	+ napi_gro_complete(napi, oldest);
5460	5976	}
5461	5977
	5978	+INDIRECT_CALLABLE_DECLARE(struct sk_buff inet_gro_receive(struct list_head ,
	5979	+ struct sk_buff *));
	5980	+INDIRECT_CALLABLE_DECLARE(struct sk_buff ipv6_gro_receive(struct list_head ,
	5981	+ struct sk_buff *));
5462	5982	static enum gro_result dev_gro_receive(struct napi_struct napi, struct sk_buff skb)
5463	5983	{
5464	5984	u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
..	..	@@ -5508,7 +6028,9 @@
5508	6028	NAPI_GRO_CB(skb)->csum_valid = 0;
5509	6029	}
5510	6030
5511		- pp = ptype->callbacks.gro_receive(gro_head, skb);
	6031	+ pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
	6032	+ ipv6_gro_receive, inet_gro_receive,
	6033	+ gro_head, skb);
5512	6034	break;
5513	6035	}
5514	6036	rcu_read_unlock();
..	..	@@ -5516,7 +6038,7 @@
5516	6038	if (&ptype->list == head)
5517	6039	goto normal;
5518	6040
5519		- if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
	6041	+ if (PTR_ERR(pp) == -EINPROGRESS) {
5520	6042	ret = GRO_CONSUMED;
5521	6043	goto ok;
5522	6044	}
..	..	@@ -5525,9 +6047,8 @@
5525	6047	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
5526	6048
5527	6049	if (pp) {
5528		- list_del(&pp->list);
5529		- pp->next = NULL;
5530		- napi_gro_complete(pp);
	6050	+ skb_list_del_init(pp);
	6051	+ napi_gro_complete(napi, pp);
5531	6052	napi->gro_hash[hash].count--;
5532	6053	}
5533	6054
..	..	@@ -5538,7 +6059,7 @@
5538	6059	goto normal;
5539	6060
5540	6061	if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
5541		- gro_flush_oldest(gro_head);
	6062	+ gro_flush_oldest(napi, gro_head);
5542	6063	} else {
5543	6064	napi->gro_hash[hash].count++;
5544	6065	}
..	..	@@ -5599,16 +6120,17 @@
5599	6120	static void napi_skb_free_stolen_head(struct sk_buff *skb)
5600	6121	{
5601	6122	skb_dst_drop(skb);
5602		- secpath_reset(skb);
	6123	+ skb_ext_put(skb);
5603	6124	kmem_cache_free(skbuff_head_cache, skb);
5604	6125	}
5605	6126
5606		-static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
	6127	+static gro_result_t napi_skb_finish(struct napi_struct *napi,
	6128	+ struct sk_buff *skb,
	6129	+ gro_result_t ret)
5607	6130	{
5608	6131	switch (ret) {
5609	6132	case GRO_NORMAL:
5610		- if (netif_receive_skb_internal(skb))
5611		- ret = GRO_DROP;
	6133	+ gro_normal_one(napi, skb, 1);
5612	6134	break;
5613	6135
5614	6136	case GRO_DROP:
..	..	@@ -5633,12 +6155,17 @@
5633	6155
5634	6156	gro_result_t napi_gro_receive(struct napi_struct napi, struct sk_buff skb)
5635	6157	{
	6158	+ gro_result_t ret;
	6159	+
5636	6160	skb_mark_napi_id(skb, napi);
5637	6161	trace_napi_gro_receive_entry(skb);
5638	6162
5639		- skb_gro_reset_offset(skb);
	6163	+ skb_gro_reset_offset(skb, 0);
5640	6164
5641		- return napi_skb_finish(dev_gro_receive(napi, skb), skb);
	6165	+ ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
	6166	+ trace_napi_gro_receive_exit(ret);
	6167	+
	6168	+ return ret;
5642	6169	}
5643	6170	EXPORT_SYMBOL(napi_gro_receive);
5644	6171
..	..	@@ -5651,7 +6178,7 @@
5651	6178	__skb_pull(skb, skb_headlen(skb));
5652	6179	/* restore the reserve we had after netdev_alloc_skb_ip_align() */
5653	6180	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
5654		- skb->vlan_tci = 0;
	6181	+ __vlan_hwaccel_clear_tag(skb);
5655	6182	skb->dev = napi->dev;
5656	6183	skb->skb_iif = 0;
5657	6184
..	..	@@ -5661,7 +6188,8 @@
5661	6188	skb->encapsulation = 0;
5662	6189	skb_shinfo(skb)->gso_type = 0;
5663	6190	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5664		- secpath_reset(skb);
	6191	+ skb_ext_reset(skb);
	6192	+ nf_reset_ct(skb);
5665	6193
5666	6194	napi->skb = skb;
5667	6195	}
..	..	@@ -5690,8 +6218,8 @@
5690	6218	case GRO_HELD:
5691	6219	__skb_push(skb, ETH_HLEN);
5692	6220	skb->protocol = eth_type_trans(skb, skb->dev);
5693		- if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
5694		- ret = GRO_DROP;
	6221	+ if (ret == GRO_NORMAL)
	6222	+ gro_normal_one(napi, skb, 1);
5695	6223	break;
5696	6224
5697	6225	case GRO_DROP:
..	..	@@ -5726,7 +6254,7 @@
5726	6254	napi->skb = NULL;
5727	6255
5728	6256	skb_reset_mac_header(skb);
5729		- skb_gro_reset_offset(skb);
	6257	+ skb_gro_reset_offset(skb, hlen);
5730	6258
5731	6259	if (unlikely(skb_gro_header_hard(skb, hlen))) {
5732	6260	eth = skb_gro_header_slow(skb, hlen, 0);
..	..	@@ -5756,6 +6284,7 @@
5756	6284
5757	6285	gro_result_t napi_gro_frags(struct napi_struct *napi)
5758	6286	{
	6287	+ gro_result_t ret;
5759	6288	struct sk_buff *skb = napi_frags_skb(napi);
5760	6289
5761	6290	if (!skb)
..	..	@@ -5763,7 +6292,10 @@
5763	6292
5764	6293	trace_napi_gro_frags_entry(skb);
5765	6294
5766		- return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
	6295	+ ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
	6296	+ trace_napi_gro_frags_exit(ret);
	6297	+
	6298	+ return ret;
5767	6299	}
5768	6300	EXPORT_SYMBOL(napi_gro_frags);
5769	6301
..	..	@@ -5779,10 +6311,11 @@
5779	6311
5780	6312	/* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
5781	6313	sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
	6314	+ /* See comments in __skb_checksum_complete(). */
5782	6315	if (likely(!sum)) {
5783	6316	if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5784	6317	!skb->csum_complete_sw)
5785		- netdev_rx_csum_fault(skb->dev);
	6318	+ netdev_rx_csum_fault(skb->dev, skb);
5786	6319	}
5787	6320
5788	6321	NAPI_GRO_CB(skb)->csum = wsum;
..	..	@@ -5818,12 +6351,14 @@
5818	6351	sd->rps_ipi_list = NULL;
5819	6352
5820	6353	local_irq_enable();
	6354	+ preempt_check_resched_rt();
5821	6355
5822	6356	/* Send pending IPI's to kick RPS processing on remote cpus. */
5823	6357	net_rps_send_ipi(remsd);
5824	6358	} else
5825	6359	#endif
5826	6360	local_irq_enable();
	6361	+ preempt_check_resched_rt();
5827	6362	}
5828	6363
5829	6364	static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
..	..	@@ -5849,7 +6384,7 @@
5849	6384	net_rps_action_and_irq_enable(sd);
5850	6385	}
5851	6386
5852		- napi->weight = dev_rx_weight;
	6387	+ napi->weight = READ_ONCE(dev_rx_weight);
5853	6388	while (again) {
5854	6389	struct sk_buff *skb;
5855	6390
..	..	@@ -5859,7 +6394,8 @@
5859	6394	rcu_read_unlock();
5860	6395	input_queue_head_incr(sd);
5861	6396	if (++work >= quota)
5862		- goto state_changed;
	6397	+ return work;
	6398	+
5863	6399	}
5864	6400
5865	6401	local_irq_disable();
..	..	@@ -5883,10 +6419,6 @@
5883	6419	local_irq_enable();
5884	6420	}
5885	6421
5886		-state_changed:
5887		- napi_gro_flush(napi, false);
5888		- sd->current_napi = NULL;
5889		-
5890	6422	return work;
5891	6423	}
5892	6424
..	..	@@ -5904,6 +6436,7 @@
5904	6436	local_irq_save(flags);
5905	6437	____napi_schedule(this_cpu_ptr(&softnet_data), n);
5906	6438	local_irq_restore(flags);
	6439	+ preempt_check_resched_rt();
5907	6440	}
5908	6441	EXPORT_SYMBOL(__napi_schedule);
5909	6442
..	..	@@ -5912,7 +6445,7 @@
5912	6445	* @n: napi context
5913	6446	*
5914	6447	* Test if NAPI routine is already running, and if not mark
5915		- * it as running. This is used as a condition variable
	6448	+ * it as running. This is used as a condition variable to
5916	6449	* insure only one NAPI poll instance runs. We also make
5917	6450	* sure there is no pending NAPI disable.
5918	6451	*/
..	..	@@ -5961,7 +6494,8 @@
5961	6494
5962	6495	bool napi_complete_done(struct napi_struct *n, int work_done)
5963	6496	{
5964		- unsigned long flags, val, new;
	6497	+ unsigned long flags, val, new, timeout = 0;
	6498	+ bool ret = true;
5965	6499
5966	6500	/*
5967	6501	* 1) Don't let napi dequeue from the cpu poll list
..	..	@@ -5973,28 +6507,31 @@
5973	6507	NAPIF_STATE_IN_BUSY_POLL)))
5974	6508	return false;
5975	6509
	6510	+ if (work_done) {
	6511	+ if (n->gro_bitmask)
	6512	+ timeout = READ_ONCE(n->dev->gro_flush_timeout);
	6513	+ n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
	6514	+ }
	6515	+ if (n->defer_hard_irqs_count > 0) {
	6516	+ n->defer_hard_irqs_count--;
	6517	+ timeout = READ_ONCE(n->dev->gro_flush_timeout);
	6518	+ if (timeout)
	6519	+ ret = false;
	6520	+ }
5976	6521	if (n->gro_bitmask) {
5977		- unsigned long timeout = 0;
5978		-
5979		- if (work_done)
5980		- timeout = n->dev->gro_flush_timeout;
5981		-
5982	6522	/* When the NAPI instance uses a timeout and keeps postponing
5983	6523	* it, we need to bound somehow the time packets are kept in
5984	6524	* the GRO layer
5985	6525	*/
5986	6526	napi_gro_flush(n, !!timeout);
5987		- if (timeout)
5988		- hrtimer_start(&n->timer, ns_to_ktime(timeout),
5989		- HRTIMER_MODE_REL_PINNED);
5990	6527	}
5991		- if (unlikely(!list_empty(&n->poll_list))) {
5992		- struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5993	6528
	6529	+ gro_normal_list(n);
	6530	+
	6531	+ if (unlikely(!list_empty(&n->poll_list))) {
5994	6532	/* If n->poll_list is not empty, we need to mask irqs */
5995	6533	local_irq_save(flags);
5996	6534	list_del_init(&n->poll_list);
5997		- sd->current_napi = NULL;
5998	6535	local_irq_restore(flags);
5999	6536	}
6000	6537
..	..	@@ -6018,7 +6555,10 @@
6018	6555	return false;
6019	6556	}
6020	6557
6021		- return true;
	6558	+ if (timeout)
	6559	+ hrtimer_start(&n->timer, ns_to_ktime(timeout),
	6560	+ HRTIMER_MODE_REL_PINNED);
	6561	+ return ret;
6022	6562	}
6023	6563	EXPORT_SYMBOL(napi_complete_done);
6024	6564
..	..	@@ -6061,10 +6601,19 @@
6061	6601	* Ideally, a new ndo_busy_poll_stop() could avoid another round.
6062	6602	*/
6063	6603	rc = napi->poll(napi, BUSY_POLL_BUDGET);
	6604	+ /* We can't gro_normal_list() here, because napi->poll() might have
	6605	+ * rearmed the napi (napi_complete_done()) in which case it could
	6606	+ * already be running on another CPU.
	6607	+ */
6064	6608	trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
6065	6609	netpoll_poll_unlock(have_poll_lock);
6066		- if (rc == BUSY_POLL_BUDGET)
	6610	+ if (rc == BUSY_POLL_BUDGET) {
	6611	+ /* As the whole budget was spent, we still own the napi so can
	6612	+ * safely handle the rx_list.
	6613	+ */
	6614	+ gro_normal_list(napi);
6067	6615	__napi_schedule(napi);
	6616	+ }
6068	6617	local_bh_enable();
6069	6618	}
6070	6619
..	..	@@ -6109,6 +6658,7 @@
6109	6658	}
6110	6659	work = napi_poll(napi, BUSY_POLL_BUDGET);
6111	6660	trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
	6661	+ gro_normal_list(napi);
6112	6662	count:
6113	6663	if (work > 0)
6114	6664	__NET_ADD_STATS(dev_net(napi->dev),
..	..	@@ -6142,8 +6692,7 @@
6142	6692
6143	6693	static void napi_hash_add(struct napi_struct *napi)
6144	6694	{
6145		- if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) \|\|
6146		- test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
	6695	+ if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6147	6696	return;
6148	6697
6149	6698	spin_lock(&napi_hash_lock);
..	..	@@ -6164,20 +6713,14 @@
6164	6713	/* Warning : caller is responsible to make sure rcu grace period
6165	6714	* is respected before freeing memory containing @napi
6166	6715	*/
6167		-bool napi_hash_del(struct napi_struct *napi)
	6716	+static void napi_hash_del(struct napi_struct *napi)
6168	6717	{
6169		- bool rcu_sync_needed = false;
6170		-
6171	6718	spin_lock(&napi_hash_lock);
6172	6719
6173		- if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
6174		- rcu_sync_needed = true;
6175		- hlist_del_rcu(&napi->napi_hash_node);
6176		- }
	6720	+ hlist_del_init_rcu(&napi->napi_hash_node);
	6721	+
6177	6722	spin_unlock(&napi_hash_lock);
6178		- return rcu_sync_needed;
6179	6723	}
6180		-EXPORT_SYMBOL_GPL(napi_hash_del);
6181	6724
6182	6725	static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6183	6726	{
..	..	@@ -6188,7 +6731,7 @@
6188	6731	/* Note : we use a relaxed variant of napi_schedule_prep() not setting
6189	6732	* NAPI_STATE_MISSED, since we do not react to a device IRQ.
6190	6733	*/
6191		- if (napi->gro_bitmask && !napi_disable_pending(napi) &&
	6734	+ if (!napi_disable_pending(napi) &&
6192	6735	!test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
6193	6736	__napi_schedule_irqoff(napi);
6194	6737
..	..	@@ -6209,15 +6752,21 @@
6209	6752	void netif_napi_add(struct net_device dev, struct napi_struct napi,
6210	6753	int (poll)(struct napi_struct , int), int weight)
6211	6754	{
	6755	+ if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
	6756	+ return;
	6757	+
6212	6758	INIT_LIST_HEAD(&napi->poll_list);
	6759	+ INIT_HLIST_NODE(&napi->napi_hash_node);
6213	6760	hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6214	6761	napi->timer.function = napi_watchdog;
6215	6762	init_gro_hash(napi);
6216	6763	napi->skb = NULL;
	6764	+ INIT_LIST_HEAD(&napi->rx_list);
	6765	+ napi->rx_count = 0;
6217	6766	napi->poll = poll;
6218	6767	if (weight > NAPI_POLL_WEIGHT)
6219		- pr_err_once("netif_napi_add() called with weight %d on device %s\n",
6220		- weight, dev->name);
	6768	+ netdev_err_once(dev, "%s() called with weight %d\n", __func__,
	6769	+ weight);
6221	6770	napi->weight = weight;
6222	6771	napi->dev = dev;
6223	6772	#ifdef CONFIG_NETPOLL
..	..	@@ -6260,26 +6809,19 @@
6260	6809	}
6261	6810
6262	6811	/* Must be called in process context */
6263		-void netif_napi_del(struct napi_struct *napi)
	6812	+void __netif_napi_del(struct napi_struct *napi)
6264	6813	{
6265		- might_sleep();
6266		- if (napi_hash_del(napi))
6267		- synchronize_net();
6268		- list_del_init(&napi->dev_list);
	6814	+ if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
	6815	+ return;
	6816	+
	6817	+ napi_hash_del(napi);
	6818	+ list_del_rcu(&napi->dev_list);
6269	6819	napi_free_frags(napi);
6270	6820
6271	6821	flush_gro_hash(napi);
6272	6822	napi->gro_bitmask = 0;
6273	6823	}
6274		-EXPORT_SYMBOL(netif_napi_del);
6275		-
6276		-struct napi_struct *get_current_napi_context(void)
6277		-{
6278		- struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6279		-
6280		- return sd->current_napi;
6281		-}
6282		-EXPORT_SYMBOL(get_current_napi_context);
	6824	+EXPORT_SYMBOL(__netif_napi_del);
6283	6825
6284	6826	static int napi_poll(struct napi_struct n, struct list_head repoll)
6285	6827	{
..	..	@@ -6300,14 +6842,13 @@
6300	6842	*/
6301	6843	work = 0;
6302	6844	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
6303		- struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6304		-
6305		- sd->current_napi = n;
6306	6845	work = n->poll(n, weight);
6307	6846	trace_napi_poll(n, work, weight);
6308	6847	}
6309	6848
6310		- WARN_ON_ONCE(work > weight);
	6849	+ if (unlikely(work > weight))
	6850	+ pr_err_once("NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
	6851	+ n->poll, work, weight);
6311	6852
6312	6853	if (likely(work < weight))
6313	6854	goto out_unlock;
..	..	@@ -6328,6 +6869,8 @@
6328	6869	*/
6329	6870	napi_gro_flush(n, HZ >= 1000);
6330	6871	}
	6872	+
	6873	+ gro_normal_list(n);
6331	6874
6332	6875	/* Some drivers may have called napi_schedule
6333	6876	* prior to exhausting their budget.
..	..	@@ -6350,8 +6893,8 @@
6350	6893	{
6351	6894	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6352	6895	unsigned long time_limit = jiffies +
6353		- usecs_to_jiffies(netdev_budget_usecs);
6354		- int budget = netdev_budget;
	6896	+ usecs_to_jiffies(READ_ONCE(netdev_budget_usecs));
	6897	+ int budget = READ_ONCE(netdev_budget);
6355	6898	LIST_HEAD(list);
6356	6899	LIST_HEAD(repoll);
6357	6900
..	..	@@ -6401,6 +6944,9 @@
6401	6944	/* upper master flag, there can only be one master device per list */
6402	6945	bool master;
6403	6946
	6947	+ /* lookup ignore flag */
	6948	+ bool ignore;
	6949	+
6404	6950	/* counter for the number of times this device was added to us */
6405	6951	u16 ref_nr;
6406	6952
..	..	@@ -6423,9 +6969,10 @@
6423	6969	return NULL;
6424	6970	}
6425	6971
6426		-static int __netdev_has_upper_dev(struct net_device upper_dev, void data)
	6972	+static int ____netdev_has_upper_dev(struct net_device *upper_dev,
	6973	+ struct netdev_nested_priv *priv)
6427	6974	{
6428		- struct net_device *dev = data;
	6975	+ struct net_device dev = (struct net_device )priv->data;
6429	6976
6430	6977	return upper_dev == dev;
6431	6978	}
..	..	@@ -6442,10 +6989,14 @@
6442	6989	bool netdev_has_upper_dev(struct net_device *dev,
6443	6990	struct net_device *upper_dev)
6444	6991	{
	6992	+ struct netdev_nested_priv priv = {
	6993	+ .data = (void *)upper_dev,
	6994	+ };
	6995	+
6445	6996	ASSERT_RTNL();
6446	6997
6447		- return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
6448		- upper_dev);
	6998	+ return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
	6999	+ &priv);
6449	7000	}
6450	7001	EXPORT_SYMBOL(netdev_has_upper_dev);
6451	7002
..	..	@@ -6462,8 +7013,12 @@
6462	7013	bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
6463	7014	struct net_device *upper_dev)
6464	7015	{
6465		- return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
6466		- upper_dev);
	7016	+ struct netdev_nested_priv priv = {
	7017	+ .data = (void *)upper_dev,
	7018	+ };
	7019	+
	7020	+ return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
	7021	+ &priv);
6467	7022	}
6468	7023	EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
6469	7024
..	..	@@ -6505,6 +7060,22 @@
6505	7060	return NULL;
6506	7061	}
6507	7062	EXPORT_SYMBOL(netdev_master_upper_dev_get);
	7063	+
	7064	+static struct net_device __netdev_master_upper_dev_get(struct net_device dev)
	7065	+{
	7066	+ struct netdev_adjacent *upper;
	7067	+
	7068	+ ASSERT_RTNL();
	7069	+
	7070	+ if (list_empty(&dev->adj_list.upper))
	7071	+ return NULL;
	7072	+
	7073	+ upper = list_first_entry(&dev->adj_list.upper,
	7074	+ struct netdev_adjacent, list);
	7075	+ if (likely(upper->master) && !upper->ignore)
	7076	+ return upper->dev;
	7077	+ return NULL;
	7078	+}
6508	7079
6509	7080	/**
6510	7081	* netdev_has_any_lower_dev - Check if device is linked to some device
..	..	@@ -6556,8 +7127,9 @@
6556	7127	}
6557	7128	EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
6558	7129
6559		-static struct net_device netdev_next_upper_dev(struct net_device dev,
6560		- struct list_head **iter)
	7130	+static struct net_device __netdev_next_upper_dev(struct net_device dev,
	7131	+ struct list_head **iter,
	7132	+ bool *ignore)
6561	7133	{
6562	7134	struct netdev_adjacent *upper;
6563	7135
..	..	@@ -6567,6 +7139,7 @@
6567	7139	return NULL;
6568	7140
6569	7141	*iter = &upper->list;
	7142	+ *ignore = upper->ignore;
6570	7143
6571	7144	return upper->dev;
6572	7145	}
..	..	@@ -6588,30 +7161,33 @@
6588	7161	return upper->dev;
6589	7162	}
6590	7163
6591		-static int netdev_walk_all_upper_dev(struct net_device *dev,
6592		- int (fn)(struct net_device dev,
6593		- void *data),
6594		- void *data)
	7164	+static int __netdev_walk_all_upper_dev(struct net_device *dev,
	7165	+ int (fn)(struct net_device dev,
	7166	+ struct netdev_nested_priv *priv),
	7167	+ struct netdev_nested_priv *priv)
6595	7168	{
6596	7169	struct net_device udev, next, now, dev_stack[MAX_NEST_DEV + 1];
6597	7170	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
6598	7171	int ret, cur = 0;
	7172	+ bool ignore;
6599	7173
6600	7174	now = dev;
6601	7175	iter = &dev->adj_list.upper;
6602	7176
6603	7177	while (1) {
6604	7178	if (now != dev) {
6605		- ret = fn(now, data);
	7179	+ ret = fn(now, priv);
6606	7180	if (ret)
6607	7181	return ret;
6608	7182	}
6609	7183
6610	7184	next = NULL;
6611	7185	while (1) {
6612		- udev = netdev_next_upper_dev(now, &iter);
	7186	+ udev = __netdev_next_upper_dev(now, &iter, &ignore);
6613	7187	if (!udev)
6614	7188	break;
	7189	+ if (ignore)
	7190	+ continue;
6615	7191
6616	7192	next = udev;
6617	7193	niter = &udev->adj_list.upper;
..	..	@@ -6636,8 +7212,8 @@
6636	7212
6637	7213	int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
6638	7214	int (fn)(struct net_device dev,
6639		- void *data),
6640		- void *data)
	7215	+ struct netdev_nested_priv *priv),
	7216	+ struct netdev_nested_priv *priv)
6641	7217	{
6642	7218	struct net_device udev, next, now, dev_stack[MAX_NEST_DEV + 1];
6643	7219	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
..	..	@@ -6648,7 +7224,7 @@
6648	7224
6649	7225	while (1) {
6650	7226	if (now != dev) {
6651		- ret = fn(now, data);
	7227	+ ret = fn(now, priv);
6652	7228	if (ret)
6653	7229	return ret;
6654	7230	}
..	..	@@ -6680,6 +7256,20 @@
6680	7256	return 0;
6681	7257	}
6682	7258	EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
	7259	+
	7260	+static bool __netdev_has_upper_dev(struct net_device *dev,
	7261	+ struct net_device *upper_dev)
	7262	+{
	7263	+ struct netdev_nested_priv priv = {
	7264	+ .flags = 0,
	7265	+ .data = (void *)upper_dev,
	7266	+ };
	7267	+
	7268	+ ASSERT_RTNL();
	7269	+
	7270	+ return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
	7271	+ &priv);
	7272	+}
6683	7273
6684	7274	/**
6685	7275	* netdev_lower_get_next_private - Get the next ->private from the
..	..	@@ -6777,10 +7367,27 @@
6777	7367	return lower->dev;
6778	7368	}
6779	7369
	7370	+static struct net_device __netdev_next_lower_dev(struct net_device dev,
	7371	+ struct list_head **iter,
	7372	+ bool *ignore)
	7373	+{
	7374	+ struct netdev_adjacent *lower;
	7375	+
	7376	+ lower = list_entry((*iter)->next, struct netdev_adjacent, list);
	7377	+
	7378	+ if (&lower->list == &dev->adj_list.lower)
	7379	+ return NULL;
	7380	+
	7381	+ *iter = &lower->list;
	7382	+ *ignore = lower->ignore;
	7383	+
	7384	+ return lower->dev;
	7385	+}
	7386	+
6780	7387	int netdev_walk_all_lower_dev(struct net_device *dev,
6781	7388	int (fn)(struct net_device dev,
6782		- void *data),
6783		- void *data)
	7389	+ struct netdev_nested_priv *priv),
	7390	+ struct netdev_nested_priv *priv)
6784	7391	{
6785	7392	struct net_device ldev, next, now, dev_stack[MAX_NEST_DEV + 1];
6786	7393	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
..	..	@@ -6791,7 +7398,7 @@
6791	7398
6792	7399	while (1) {
6793	7400	if (now != dev) {
6794		- ret = fn(now, data);
	7401	+ ret = fn(now, priv);
6795	7402	if (ret)
6796	7403	return ret;
6797	7404	}
..	..	@@ -6824,8 +7431,57 @@
6824	7431	}
6825	7432	EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
6826	7433
6827		-static struct net_device netdev_next_lower_dev_rcu(struct net_device dev,
6828		- struct list_head **iter)
	7434	+static int __netdev_walk_all_lower_dev(struct net_device *dev,
	7435	+ int (fn)(struct net_device dev,
	7436	+ struct netdev_nested_priv *priv),
	7437	+ struct netdev_nested_priv *priv)
	7438	+{
	7439	+ struct net_device ldev, next, now, dev_stack[MAX_NEST_DEV + 1];
	7440	+ struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
	7441	+ int ret, cur = 0;
	7442	+ bool ignore;
	7443	+
	7444	+ now = dev;
	7445	+ iter = &dev->adj_list.lower;
	7446	+
	7447	+ while (1) {
	7448	+ if (now != dev) {
	7449	+ ret = fn(now, priv);
	7450	+ if (ret)
	7451	+ return ret;
	7452	+ }
	7453	+
	7454	+ next = NULL;
	7455	+ while (1) {
	7456	+ ldev = __netdev_next_lower_dev(now, &iter, &ignore);
	7457	+ if (!ldev)
	7458	+ break;
	7459	+ if (ignore)
	7460	+ continue;
	7461	+
	7462	+ next = ldev;
	7463	+ niter = &ldev->adj_list.lower;
	7464	+ dev_stack[cur] = now;
	7465	+ iter_stack[cur++] = iter;
	7466	+ break;
	7467	+ }
	7468	+
	7469	+ if (!next) {
	7470	+ if (!cur)
	7471	+ return 0;
	7472	+ next = dev_stack[--cur];
	7473	+ niter = iter_stack[cur];
	7474	+ }
	7475	+
	7476	+ now = next;
	7477	+ iter = niter;
	7478	+ }
	7479	+
	7480	+ return 0;
	7481	+}
	7482	+
	7483	+struct net_device netdev_next_lower_dev_rcu(struct net_device dev,
	7484	+ struct list_head **iter)
6829	7485	{
6830	7486	struct netdev_adjacent *lower;
6831	7487
..	..	@@ -6837,17 +7493,21 @@
6837	7493
6838	7494	return lower->dev;
6839	7495	}
	7496	+EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
6840	7497
6841	7498	static u8 __netdev_upper_depth(struct net_device *dev)
6842	7499	{
6843	7500	struct net_device *udev;
6844	7501	struct list_head *iter;
6845	7502	u8 max_depth = 0;
	7503	+ bool ignore;
6846	7504
6847	7505	for (iter = &dev->adj_list.upper,
6848		- udev = netdev_next_upper_dev(dev, &iter);
	7506	+ udev = __netdev_next_upper_dev(dev, &iter, &ignore);
6849	7507	udev;
6850		- udev = netdev_next_upper_dev(dev, &iter)) {
	7508	+ udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
	7509	+ if (ignore)
	7510	+ continue;
6851	7511	if (max_depth < udev->upper_level)
6852	7512	max_depth = udev->upper_level;
6853	7513	}
..	..	@@ -6860,11 +7520,14 @@
6860	7520	struct net_device *ldev;
6861	7521	struct list_head *iter;
6862	7522	u8 max_depth = 0;
	7523	+ bool ignore;
6863	7524
6864	7525	for (iter = &dev->adj_list.lower,
6865		- ldev = netdev_next_lower_dev(dev, &iter);
	7526	+ ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
6866	7527	ldev;
6867		- ldev = netdev_next_lower_dev(dev, &iter)) {
	7528	+ ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
	7529	+ if (ignore)
	7530	+ continue;
6868	7531	if (max_depth < ldev->lower_level)
6869	7532	max_depth = ldev->lower_level;
6870	7533	}
..	..	@@ -6872,22 +7535,34 @@
6872	7535	return max_depth;
6873	7536	}
6874	7537
6875		-static int __netdev_update_upper_level(struct net_device dev, void data)
	7538	+static int __netdev_update_upper_level(struct net_device *dev,
	7539	+ struct netdev_nested_priv *__unused)
6876	7540	{
6877	7541	dev->upper_level = __netdev_upper_depth(dev) + 1;
6878	7542	return 0;
6879	7543	}
6880	7544
6881		-static int __netdev_update_lower_level(struct net_device dev, void data)
	7545	+static int __netdev_update_lower_level(struct net_device *dev,
	7546	+ struct netdev_nested_priv *priv)
6882	7547	{
6883	7548	dev->lower_level = __netdev_lower_depth(dev) + 1;
	7549	+
	7550	+#ifdef CONFIG_LOCKDEP
	7551	+ if (!priv)
	7552	+ return 0;
	7553	+
	7554	+ if (priv->flags & NESTED_SYNC_IMM)
	7555	+ dev->nested_level = dev->lower_level - 1;
	7556	+ if (priv->flags & NESTED_SYNC_TODO)
	7557	+ net_unlink_todo(dev);
	7558	+#endif
6884	7559	return 0;
6885	7560	}
6886	7561
6887	7562	int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
6888	7563	int (fn)(struct net_device dev,
6889		- void *data),
6890		- void *data)
	7564	+ struct netdev_nested_priv *priv),
	7565	+ struct netdev_nested_priv *priv)
6891	7566	{
6892	7567	struct net_device ldev, next, now, dev_stack[MAX_NEST_DEV + 1];
6893	7568	struct list_head niter, iter, *iter_stack[MAX_NEST_DEV + 1];
..	..	@@ -6898,7 +7573,7 @@
6898	7573
6899	7574	while (1) {
6900	7575	if (now != dev) {
6901		- ret = fn(now, data);
	7576	+ ret = fn(now, priv);
6902	7577	if (ret)
6903	7578	return ret;
6904	7579	}
..	..	@@ -7028,6 +7703,7 @@
7028	7703	adj->master = master;
7029	7704	adj->ref_nr = 1;
7030	7705	adj->private = private;
	7706	+ adj->ignore = false;
7031	7707	dev_hold(adj_dev);
7032	7708
7033	7709	pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
..	..	@@ -7157,6 +7833,7 @@
7157	7833	static int __netdev_upper_dev_link(struct net_device *dev,
7158	7834	struct net_device *upper_dev, bool master,
7159	7835	void upper_priv, void upper_info,
	7836	+ struct netdev_nested_priv *priv,
7160	7837	struct netlink_ext_ack *extack)
7161	7838	{
7162	7839	struct netdev_notifier_changeupper_info changeupper_info = {
..	..	@@ -7178,17 +7855,17 @@
7178	7855	return -EBUSY;
7179	7856
7180	7857	/* To prevent loops, check if dev is not upper device to upper_dev. */
7181		- if (netdev_has_upper_dev(upper_dev, dev))
	7858	+ if (__netdev_has_upper_dev(upper_dev, dev))
7182	7859	return -EBUSY;
7183	7860
7184	7861	if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
7185	7862	return -EMLINK;
7186	7863
7187	7864	if (!master) {
7188		- if (netdev_has_upper_dev(dev, upper_dev))
	7865	+ if (__netdev_has_upper_dev(dev, upper_dev))
7189	7866	return -EEXIST;
7190	7867	} else {
7191		- master_dev = netdev_master_upper_dev_get(dev);
	7868	+ master_dev = __netdev_master_upper_dev_get(dev);
7192	7869	if (master_dev)
7193	7870	return master_dev == upper_dev ? -EEXIST : -EBUSY;
7194	7871	}
..	..	@@ -7211,10 +7888,11 @@
7211	7888	goto rollback;
7212	7889
7213	7890	__netdev_update_upper_level(dev, NULL);
7214		- netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
	7891	+ __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7215	7892
7216		- __netdev_update_lower_level(upper_dev, NULL);
7217		- netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, NULL);
	7893	+ __netdev_update_lower_level(upper_dev, priv);
	7894	+ __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
	7895	+ priv);
7218	7896
7219	7897	return 0;
7220	7898
..	..	@@ -7239,8 +7917,13 @@
7239	7917	struct net_device *upper_dev,
7240	7918	struct netlink_ext_ack *extack)
7241	7919	{
	7920	+ struct netdev_nested_priv priv = {
	7921	+ .flags = NESTED_SYNC_IMM \| NESTED_SYNC_TODO,
	7922	+ .data = NULL,
	7923	+ };
	7924	+
7242	7925	return __netdev_upper_dev_link(dev, upper_dev, false,
7243		- NULL, NULL, extack);
	7926	+ NULL, NULL, &priv, extack);
7244	7927	}
7245	7928	EXPORT_SYMBOL(netdev_upper_dev_link);
7246	7929
..	..	@@ -7263,21 +7946,19 @@
7263	7946	void upper_priv, void upper_info,
7264	7947	struct netlink_ext_ack *extack)
7265	7948	{
	7949	+ struct netdev_nested_priv priv = {
	7950	+ .flags = NESTED_SYNC_IMM \| NESTED_SYNC_TODO,
	7951	+ .data = NULL,
	7952	+ };
	7953	+
7266	7954	return __netdev_upper_dev_link(dev, upper_dev, true,
7267		- upper_priv, upper_info, extack);
	7955	+ upper_priv, upper_info, &priv, extack);
7268	7956	}
7269	7957	EXPORT_SYMBOL(netdev_master_upper_dev_link);
7270	7958
7271		-/**
7272		- * netdev_upper_dev_unlink - Removes a link to upper device
7273		- * @dev: device
7274		- * @upper_dev: new upper device
7275		- *
7276		- * Removes a link to device which is upper to this one. The caller must hold
7277		- * the RTNL lock.
7278		- */
7279		-void netdev_upper_dev_unlink(struct net_device *dev,
7280		- struct net_device *upper_dev)
	7959	+static void __netdev_upper_dev_unlink(struct net_device *dev,
	7960	+ struct net_device *upper_dev,
	7961	+ struct netdev_nested_priv *priv)
7281	7962	{
7282	7963	struct netdev_notifier_changeupper_info changeupper_info = {
7283	7964	.info = {
..	..	@@ -7300,12 +7981,126 @@
7300	7981	&changeupper_info.info);
7301	7982
7302	7983	__netdev_update_upper_level(dev, NULL);
7303		- netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
	7984	+ __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7304	7985
7305		- __netdev_update_lower_level(upper_dev, NULL);
7306		- netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, NULL);
	7986	+ __netdev_update_lower_level(upper_dev, priv);
	7987	+ __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
	7988	+ priv);
	7989	+}
	7990	+
	7991	+/**
	7992	+ * netdev_upper_dev_unlink - Removes a link to upper device
	7993	+ * @dev: device
	7994	+ * @upper_dev: new upper device
	7995	+ *
	7996	+ * Removes a link to device which is upper to this one. The caller must hold
	7997	+ * the RTNL lock.
	7998	+ */
	7999	+void netdev_upper_dev_unlink(struct net_device *dev,
	8000	+ struct net_device *upper_dev)
	8001	+{
	8002	+ struct netdev_nested_priv priv = {
	8003	+ .flags = NESTED_SYNC_TODO,
	8004	+ .data = NULL,
	8005	+ };
	8006	+
	8007	+ __netdev_upper_dev_unlink(dev, upper_dev, &priv);
7307	8008	}
7308	8009	EXPORT_SYMBOL(netdev_upper_dev_unlink);
	8010	+
	8011	+static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
	8012	+ struct net_device *lower_dev,
	8013	+ bool val)
	8014	+{
	8015	+ struct netdev_adjacent *adj;
	8016	+
	8017	+ adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
	8018	+ if (adj)
	8019	+ adj->ignore = val;
	8020	+
	8021	+ adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
	8022	+ if (adj)
	8023	+ adj->ignore = val;
	8024	+}
	8025	+
	8026	+static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
	8027	+ struct net_device *lower_dev)
	8028	+{
	8029	+ __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
	8030	+}
	8031	+
	8032	+static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
	8033	+ struct net_device *lower_dev)
	8034	+{
	8035	+ __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
	8036	+}
	8037	+
	8038	+int netdev_adjacent_change_prepare(struct net_device *old_dev,
	8039	+ struct net_device *new_dev,
	8040	+ struct net_device *dev,
	8041	+ struct netlink_ext_ack *extack)
	8042	+{
	8043	+ struct netdev_nested_priv priv = {
	8044	+ .flags = 0,
	8045	+ .data = NULL,
	8046	+ };
	8047	+ int err;
	8048	+
	8049	+ if (!new_dev)
	8050	+ return 0;
	8051	+
	8052	+ if (old_dev && new_dev != old_dev)
	8053	+ netdev_adjacent_dev_disable(dev, old_dev);
	8054	+ err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
	8055	+ extack);
	8056	+ if (err) {
	8057	+ if (old_dev && new_dev != old_dev)
	8058	+ netdev_adjacent_dev_enable(dev, old_dev);
	8059	+ return err;
	8060	+ }
	8061	+
	8062	+ return 0;
	8063	+}
	8064	+EXPORT_SYMBOL(netdev_adjacent_change_prepare);
	8065	+
	8066	+void netdev_adjacent_change_commit(struct net_device *old_dev,
	8067	+ struct net_device *new_dev,
	8068	+ struct net_device *dev)
	8069	+{
	8070	+ struct netdev_nested_priv priv = {
	8071	+ .flags = NESTED_SYNC_IMM \| NESTED_SYNC_TODO,
	8072	+ .data = NULL,
	8073	+ };
	8074	+
	8075	+ if (!new_dev \|\| !old_dev)
	8076	+ return;
	8077	+
	8078	+ if (new_dev == old_dev)
	8079	+ return;
	8080	+
	8081	+ netdev_adjacent_dev_enable(dev, old_dev);
	8082	+ __netdev_upper_dev_unlink(old_dev, dev, &priv);
	8083	+}
	8084	+EXPORT_SYMBOL(netdev_adjacent_change_commit);
	8085	+
	8086	+void netdev_adjacent_change_abort(struct net_device *old_dev,
	8087	+ struct net_device *new_dev,
	8088	+ struct net_device *dev)
	8089	+{
	8090	+ struct netdev_nested_priv priv = {
	8091	+ .flags = 0,
	8092	+ .data = NULL,
	8093	+ };
	8094	+
	8095	+ if (!new_dev)
	8096	+ return;
	8097	+
	8098	+ if (old_dev && new_dev != old_dev)
	8099	+ netdev_adjacent_dev_enable(dev, old_dev);
	8100	+
	8101	+ __netdev_upper_dev_unlink(new_dev, dev, &priv);
	8102	+}
	8103	+EXPORT_SYMBOL(netdev_adjacent_change_abort);
7309	8104
7310	8105	/**
7311	8106	* netdev_bonding_info_change - Dispatch event about slave change
..	..	@@ -7328,6 +8123,29 @@
7328	8123	&info.info);
7329	8124	}
7330	8125	EXPORT_SYMBOL(netdev_bonding_info_change);
	8126	+
	8127	+/**
	8128	+ * netdev_get_xmit_slave - Get the xmit slave of master device
	8129	+ * @dev: device
	8130	+ * @skb: The packet
	8131	+ * @all_slaves: assume all the slaves are active
	8132	+ *
	8133	+ * The reference counters are not incremented so the caller must be
	8134	+ * careful with locks. The caller must hold RCU lock.
	8135	+ * %NULL is returned if no slave is found.
	8136	+ */
	8137	+
	8138	+struct net_device netdev_get_xmit_slave(struct net_device dev,
	8139	+ struct sk_buff *skb,
	8140	+ bool all_slaves)
	8141	+{
	8142	+ const struct net_device_ops *ops = dev->netdev_ops;
	8143	+
	8144	+ if (!ops->ndo_get_xmit_slave)
	8145	+ return NULL;
	8146	+ return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
	8147	+}
	8148	+EXPORT_SYMBOL(netdev_get_xmit_slave);
7331	8149
7332	8150	static void netdev_adjacent_add_links(struct net_device *dev)
7333	8151	{
..	..	@@ -7419,25 +8237,6 @@
7419	8237	}
7420	8238	EXPORT_SYMBOL(netdev_lower_dev_get_private);
7421	8239
7422		-
7423		-int dev_get_nest_level(struct net_device *dev)
7424		-{
7425		- struct net_device *lower = NULL;
7426		- struct list_head *iter;
7427		- int max_nest = -1;
7428		- int nest;
7429		-
7430		- ASSERT_RTNL();
7431		-
7432		- netdev_for_each_lower_dev(dev, lower, iter) {
7433		- nest = dev_get_nest_level(lower);
7434		- if (max_nest < nest)
7435		- max_nest = nest;
7436		- }
7437		-
7438		- return max_nest + 1;
7439		-}
7440		-EXPORT_SYMBOL(dev_get_nest_level);
7441	8240
7442	8241	/**
7443	8242	* netdev_lower_change - Dispatch event about lower device state change
..	..	@@ -7665,7 +8464,8 @@
7665	8464	}
7666	8465	EXPORT_SYMBOL(dev_get_flags);
7667	8466
7668		-int __dev_change_flags(struct net_device *dev, unsigned int flags)
	8467	+int __dev_change_flags(struct net_device *dev, unsigned int flags,
	8468	+ struct netlink_ext_ack *extack)
7669	8469	{
7670	8470	unsigned int old_flags = dev->flags;
7671	8471	int ret;
..	..	@@ -7702,7 +8502,7 @@
7702	8502	if (old_flags & IFF_UP)
7703	8503	__dev_close(dev);
7704	8504	else
7705		- ret = __dev_open(dev);
	8505	+ ret = __dev_open(dev, extack);
7706	8506	}
7707	8507
7708	8508	if ((flags ^ dev->gflags) & IFF_PROMISC) {
..	..	@@ -7762,16 +8562,18 @@
7762	8562	* dev_change_flags - change device settings
7763	8563	* @dev: device
7764	8564	* @flags: device state flags
	8565	+ * @extack: netlink extended ack
7765	8566	*
7766	8567	* Change settings on device based state flags. The flags are
7767	8568	* in the userspace exported format.
7768	8569	*/
7769		-int dev_change_flags(struct net_device *dev, unsigned int flags)
	8570	+int dev_change_flags(struct net_device *dev, unsigned int flags,
	8571	+ struct netlink_ext_ack *extack)
7770	8572	{
7771	8573	int ret;
7772	8574	unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
7773	8575
7774		- ret = __dev_change_flags(dev, flags);
	8576	+ ret = __dev_change_flags(dev, flags, extack);
7775	8577	if (ret < 0)
7776	8578	return ret;
7777	8579
..	..	@@ -7914,13 +8716,36 @@
7914	8716	EXPORT_SYMBOL(dev_set_group);
7915	8717
7916	8718	/**
	8719	+ * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
	8720	+ * @dev: device
	8721	+ * @addr: new address
	8722	+ * @extack: netlink extended ack
	8723	+ */
	8724	+int dev_pre_changeaddr_notify(struct net_device dev, const char addr,
	8725	+ struct netlink_ext_ack *extack)
	8726	+{
	8727	+ struct netdev_notifier_pre_changeaddr_info info = {
	8728	+ .info.dev = dev,
	8729	+ .info.extack = extack,
	8730	+ .dev_addr = addr,
	8731	+ };
	8732	+ int rc;
	8733	+
	8734	+ rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
	8735	+ return notifier_to_errno(rc);
	8736	+}
	8737	+EXPORT_SYMBOL(dev_pre_changeaddr_notify);
	8738	+
	8739	+/**
7917	8740	* dev_set_mac_address - Change Media Access Control Address
7918	8741	* @dev: device
7919	8742	* @sa: new address
	8743	+ * @extack: netlink extended ack
7920	8744	*
7921	8745	* Change the hardware (MAC) address of the device
7922	8746	*/
7923		-int dev_set_mac_address(struct net_device dev, struct sockaddr sa)
	8747	+int dev_set_mac_address(struct net_device dev, struct sockaddr sa,
	8748	+ struct netlink_ext_ack *extack)
7924	8749	{
7925	8750	const struct net_device_ops *ops = dev->netdev_ops;
7926	8751	int err;
..	..	@@ -7931,6 +8756,9 @@
7931	8756	return -EINVAL;
7932	8757	if (!netif_device_present(dev))
7933	8758	return -ENODEV;
	8759	+ err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
	8760	+ if (err)
	8761	+ return err;
7934	8762	err = ops->ndo_set_mac_address(dev, sa);
7935	8763	if (err)
7936	8764	return err;
..	..	@@ -7940,6 +8768,48 @@
7940	8768	return 0;
7941	8769	}
7942	8770	EXPORT_SYMBOL(dev_set_mac_address);
	8771	+
	8772	+static DECLARE_RWSEM(dev_addr_sem);
	8773	+
	8774	+int dev_set_mac_address_user(struct net_device dev, struct sockaddr sa,
	8775	+ struct netlink_ext_ack *extack)
	8776	+{
	8777	+ int ret;
	8778	+
	8779	+ down_write(&dev_addr_sem);
	8780	+ ret = dev_set_mac_address(dev, sa, extack);
	8781	+ up_write(&dev_addr_sem);
	8782	+ return ret;
	8783	+}
	8784	+EXPORT_SYMBOL(dev_set_mac_address_user);
	8785	+
	8786	+int dev_get_mac_address(struct sockaddr sa, struct net net, char *dev_name)
	8787	+{
	8788	+ size_t size = sizeof(sa->sa_data);
	8789	+ struct net_device *dev;
	8790	+ int ret = 0;
	8791	+
	8792	+ down_read(&dev_addr_sem);
	8793	+ rcu_read_lock();
	8794	+
	8795	+ dev = dev_get_by_name_rcu(net, dev_name);
	8796	+ if (!dev) {
	8797	+ ret = -ENODEV;
	8798	+ goto unlock;
	8799	+ }
	8800	+ if (!dev->addr_len)
	8801	+ memset(sa->sa_data, 0, size);
	8802	+ else
	8803	+ memcpy(sa->sa_data, dev->dev_addr,
	8804	+ min_t(size_t, size, dev->addr_len));
	8805	+ sa->sa_family = dev->type;
	8806	+
	8807	+unlock:
	8808	+ rcu_read_unlock();
	8809	+ up_read(&dev_addr_sem);
	8810	+ return ret;
	8811	+}
	8812	+EXPORT_SYMBOL(dev_get_mac_address);
7943	8813
7944	8814	/**
7945	8815	* dev_change_carrier - Change device carrier
..	..	@@ -7990,12 +8860,80 @@
7990	8860	char *name, size_t len)
7991	8861	{
7992	8862	const struct net_device_ops *ops = dev->netdev_ops;
	8863	+ int err;
7993	8864
7994		- if (!ops->ndo_get_phys_port_name)
7995		- return -EOPNOTSUPP;
7996		- return ops->ndo_get_phys_port_name(dev, name, len);
	8865	+ if (ops->ndo_get_phys_port_name) {
	8866	+ err = ops->ndo_get_phys_port_name(dev, name, len);
	8867	+ if (err != -EOPNOTSUPP)
	8868	+ return err;
	8869	+ }
	8870	+ return devlink_compat_phys_port_name_get(dev, name, len);
7997	8871	}
7998	8872	EXPORT_SYMBOL(dev_get_phys_port_name);
	8873	+
	8874	+/**
	8875	+ * dev_get_port_parent_id - Get the device's port parent identifier
	8876	+ * @dev: network device
	8877	+ * @ppid: pointer to a storage for the port's parent identifier
	8878	+ * @recurse: allow/disallow recursion to lower devices
	8879	+ *
	8880	+ * Get the devices's port parent identifier
	8881	+ */
	8882	+int dev_get_port_parent_id(struct net_device *dev,
	8883	+ struct netdev_phys_item_id *ppid,
	8884	+ bool recurse)
	8885	+{
	8886	+ const struct net_device_ops *ops = dev->netdev_ops;
	8887	+ struct netdev_phys_item_id first = { };
	8888	+ struct net_device *lower_dev;
	8889	+ struct list_head *iter;
	8890	+ int err;
	8891	+
	8892	+ if (ops->ndo_get_port_parent_id) {
	8893	+ err = ops->ndo_get_port_parent_id(dev, ppid);
	8894	+ if (err != -EOPNOTSUPP)
	8895	+ return err;
	8896	+ }
	8897	+
	8898	+ err = devlink_compat_switch_id_get(dev, ppid);
	8899	+ if (!err \|\| err != -EOPNOTSUPP)
	8900	+ return err;
	8901	+
	8902	+ if (!recurse)
	8903	+ return -EOPNOTSUPP;
	8904	+
	8905	+ netdev_for_each_lower_dev(dev, lower_dev, iter) {
	8906	+ err = dev_get_port_parent_id(lower_dev, ppid, recurse);
	8907	+ if (err)
	8908	+ break;
	8909	+ if (!first.id_len)
	8910	+ first = *ppid;
	8911	+ else if (memcmp(&first, ppid, sizeof(*ppid)))
	8912	+ return -EOPNOTSUPP;
	8913	+ }
	8914	+
	8915	+ return err;
	8916	+}
	8917	+EXPORT_SYMBOL(dev_get_port_parent_id);
	8918	+
	8919	+/**
	8920	+ * netdev_port_same_parent_id - Indicate if two network devices have
	8921	+ * the same port parent identifier
	8922	+ * @a: first network device
	8923	+ * @b: second network device
	8924	+ */
	8925	+bool netdev_port_same_parent_id(struct net_device a, struct net_device b)
	8926	+{
	8927	+ struct netdev_phys_item_id a_id = { };
	8928	+ struct netdev_phys_item_id b_id = { };
	8929	+
	8930	+ if (dev_get_port_parent_id(a, &a_id, true) \|\|
	8931	+ dev_get_port_parent_id(b, &b_id, true))
	8932	+ return false;
	8933	+
	8934	+ return netdev_phys_item_id_same(&a_id, &b_id);
	8935	+}
	8936	+EXPORT_SYMBOL(netdev_port_same_parent_id);
7999	8937
8000	8938	/**
8001	8939	* dev_change_proto_down - update protocol port state information
..	..	@@ -8017,67 +8955,495 @@
8017	8955	}
8018	8956	EXPORT_SYMBOL(dev_change_proto_down);
8019	8957
8020		-u32 __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op,
8021		- enum bpf_netdev_command cmd)
	8958	+/**
	8959	+ * dev_change_proto_down_generic - generic implementation for
	8960	+ * ndo_change_proto_down that sets carrier according to
	8961	+ * proto_down.
	8962	+ *
	8963	+ * @dev: device
	8964	+ * @proto_down: new value
	8965	+ */
	8966	+int dev_change_proto_down_generic(struct net_device *dev, bool proto_down)
8022	8967	{
8023		- struct netdev_bpf xdp;
	8968	+ if (proto_down)
	8969	+ netif_carrier_off(dev);
	8970	+ else
	8971	+ netif_carrier_on(dev);
	8972	+ dev->proto_down = proto_down;
	8973	+ return 0;
	8974	+}
	8975	+EXPORT_SYMBOL(dev_change_proto_down_generic);
8024	8976
8025		- if (!bpf_op)
8026		- return 0;
	8977	+/**
	8978	+ * dev_change_proto_down_reason - proto down reason
	8979	+ *
	8980	+ * @dev: device
	8981	+ * @mask: proto down mask
	8982	+ * @value: proto down value
	8983	+ */
	8984	+void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
	8985	+ u32 value)
	8986	+{
	8987	+ int b;
8027	8988
8028		- memset(&xdp, 0, sizeof(xdp));
8029		- xdp.command = cmd;
	8989	+ if (!mask) {
	8990	+ dev->proto_down_reason = value;
	8991	+ } else {
	8992	+ for_each_set_bit(b, &mask, 32) {
	8993	+ if (value & (1 << b))
	8994	+ dev->proto_down_reason \|= BIT(b);
	8995	+ else
	8996	+ dev->proto_down_reason &= ~BIT(b);
	8997	+ }
	8998	+ }
	8999	+}
	9000	+EXPORT_SYMBOL(dev_change_proto_down_reason);
8030	9001
8031		- /* Query must always succeed. */
8032		- WARN_ON(bpf_op(dev, &xdp) < 0 && cmd == XDP_QUERY_PROG);
	9002	+struct bpf_xdp_link {
	9003	+ struct bpf_link link;
	9004	+ struct net_device dev; / protected by rtnl_lock, no refcnt held */
	9005	+ int flags;
	9006	+};
8033	9007
8034		- return xdp.prog_id;
	9008	+static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
	9009	+{
	9010	+ if (flags & XDP_FLAGS_HW_MODE)
	9011	+ return XDP_MODE_HW;
	9012	+ if (flags & XDP_FLAGS_DRV_MODE)
	9013	+ return XDP_MODE_DRV;
	9014	+ if (flags & XDP_FLAGS_SKB_MODE)
	9015	+ return XDP_MODE_SKB;
	9016	+ return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
8035	9017	}
8036	9018
8037		-static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op,
8038		- struct netlink_ext_ack *extack, u32 flags,
8039		- struct bpf_prog *prog)
	9019	+static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
	9020	+{
	9021	+ switch (mode) {
	9022	+ case XDP_MODE_SKB:
	9023	+ return generic_xdp_install;
	9024	+ case XDP_MODE_DRV:
	9025	+ case XDP_MODE_HW:
	9026	+ return dev->netdev_ops->ndo_bpf;
	9027	+ default:
	9028	+ return NULL;
	9029	+ };
	9030	+}
	9031	+
	9032	+static struct bpf_xdp_link dev_xdp_link(struct net_device dev,
	9033	+ enum bpf_xdp_mode mode)
	9034	+{
	9035	+ return dev->xdp_state[mode].link;
	9036	+}
	9037	+
	9038	+static struct bpf_prog dev_xdp_prog(struct net_device dev,
	9039	+ enum bpf_xdp_mode mode)
	9040	+{
	9041	+ struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
	9042	+
	9043	+ if (link)
	9044	+ return link->link.prog;
	9045	+ return dev->xdp_state[mode].prog;
	9046	+}
	9047	+
	9048	+static u8 dev_xdp_prog_count(struct net_device *dev)
	9049	+{
	9050	+ u8 count = 0;
	9051	+ int i;
	9052	+
	9053	+ for (i = 0; i < __MAX_XDP_MODE; i++)
	9054	+ if (dev->xdp_state[i].prog \|\| dev->xdp_state[i].link)
	9055	+ count++;
	9056	+ return count;
	9057	+}
	9058	+
	9059	+u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
	9060	+{
	9061	+ struct bpf_prog *prog = dev_xdp_prog(dev, mode);
	9062	+
	9063	+ return prog ? prog->aux->id : 0;
	9064	+}
	9065	+
	9066	+static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
	9067	+ struct bpf_xdp_link *link)
	9068	+{
	9069	+ dev->xdp_state[mode].link = link;
	9070	+ dev->xdp_state[mode].prog = NULL;
	9071	+}
	9072	+
	9073	+static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
	9074	+ struct bpf_prog *prog)
	9075	+{
	9076	+ dev->xdp_state[mode].link = NULL;
	9077	+ dev->xdp_state[mode].prog = prog;
	9078	+}
	9079	+
	9080	+static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
	9081	+ bpf_op_t bpf_op, struct netlink_ext_ack *extack,
	9082	+ u32 flags, struct bpf_prog *prog)
8040	9083	{
8041	9084	struct netdev_bpf xdp;
	9085	+ int err;
8042	9086
8043	9087	memset(&xdp, 0, sizeof(xdp));
8044		- if (flags & XDP_FLAGS_HW_MODE)
8045		- xdp.command = XDP_SETUP_PROG_HW;
8046		- else
8047		- xdp.command = XDP_SETUP_PROG;
	9088	+ xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
8048	9089	xdp.extack = extack;
8049	9090	xdp.flags = flags;
8050	9091	xdp.prog = prog;
8051	9092
8052		- return bpf_op(dev, &xdp);
	9093	+ /* Drivers assume refcnt is already incremented (i.e, prog pointer is
	9094	+ * "moved" into driver), so they don't increment it on their own, but
	9095	+ * they do decrement refcnt when program is detached or replaced.
	9096	+ * Given net_device also owns link/prog, we need to bump refcnt here
	9097	+ * to prevent drivers from underflowing it.
	9098	+ */
	9099	+ if (prog)
	9100	+ bpf_prog_inc(prog);
	9101	+ err = bpf_op(dev, &xdp);
	9102	+ if (err) {
	9103	+ if (prog)
	9104	+ bpf_prog_put(prog);
	9105	+ return err;
	9106	+ }
	9107	+
	9108	+ if (mode != XDP_MODE_HW)
	9109	+ bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
	9110	+
	9111	+ return 0;
8053	9112	}
8054	9113
8055	9114	static void dev_xdp_uninstall(struct net_device *dev)
8056	9115	{
8057		- struct netdev_bpf xdp;
8058		- bpf_op_t ndo_bpf;
	9116	+ struct bpf_xdp_link *link;
	9117	+ struct bpf_prog *prog;
	9118	+ enum bpf_xdp_mode mode;
	9119	+ bpf_op_t bpf_op;
8059	9120
8060		- /* Remove generic XDP */
8061		- WARN_ON(dev_xdp_install(dev, generic_xdp_install, NULL, 0, NULL));
	9121	+ ASSERT_RTNL();
8062	9122
8063		- /* Remove from the driver */
8064		- ndo_bpf = dev->netdev_ops->ndo_bpf;
8065		- if (!ndo_bpf)
8066		- return;
	9123	+ for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
	9124	+ prog = dev_xdp_prog(dev, mode);
	9125	+ if (!prog)
	9126	+ continue;
8067	9127
8068		- memset(&xdp, 0, sizeof(xdp));
8069		- xdp.command = XDP_QUERY_PROG;
8070		- WARN_ON(ndo_bpf(dev, &xdp));
8071		- if (xdp.prog_id)
8072		- WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,
8073		- NULL));
	9128	+ bpf_op = dev_xdp_bpf_op(dev, mode);
	9129	+ if (!bpf_op)
	9130	+ continue;
8074	9131
8075		- /* Remove HW offload */
8076		- memset(&xdp, 0, sizeof(xdp));
8077		- xdp.command = XDP_QUERY_PROG_HW;
8078		- if (!ndo_bpf(dev, &xdp) && xdp.prog_id)
8079		- WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,
8080		- NULL));
	9132	+ WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
	9133	+
	9134	+ /* auto-detach link from net device */
	9135	+ link = dev_xdp_link(dev, mode);
	9136	+ if (link)
	9137	+ link->dev = NULL;
	9138	+ else
	9139	+ bpf_prog_put(prog);
	9140	+
	9141	+ dev_xdp_set_link(dev, mode, NULL);
	9142	+ }
	9143	+}
	9144	+
	9145	+static int dev_xdp_attach(struct net_device dev, struct netlink_ext_ack extack,
	9146	+ struct bpf_xdp_link link, struct bpf_prog new_prog,
	9147	+ struct bpf_prog *old_prog, u32 flags)
	9148	+{
	9149	+ unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
	9150	+ struct bpf_prog *cur_prog;
	9151	+ enum bpf_xdp_mode mode;
	9152	+ bpf_op_t bpf_op;
	9153	+ int err;
	9154	+
	9155	+ ASSERT_RTNL();
	9156	+
	9157	+ /* either link or prog attachment, never both */
	9158	+ if (link && (new_prog \|\| old_prog))
	9159	+ return -EINVAL;
	9160	+ /* link supports only XDP mode flags */
	9161	+ if (link && (flags & ~XDP_FLAGS_MODES)) {
	9162	+ NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
	9163	+ return -EINVAL;
	9164	+ }
	9165	+ /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
	9166	+ if (num_modes > 1) {
	9167	+ NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
	9168	+ return -EINVAL;
	9169	+ }
	9170	+ /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
	9171	+ if (!num_modes && dev_xdp_prog_count(dev) > 1) {
	9172	+ NL_SET_ERR_MSG(extack,
	9173	+ "More than one program loaded, unset mode is ambiguous");
	9174	+ return -EINVAL;
	9175	+ }
	9176	+ /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
	9177	+ if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
	9178	+ NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
	9179	+ return -EINVAL;
	9180	+ }
	9181	+
	9182	+ mode = dev_xdp_mode(dev, flags);
	9183	+ /* can't replace attached link */
	9184	+ if (dev_xdp_link(dev, mode)) {
	9185	+ NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
	9186	+ return -EBUSY;
	9187	+ }
	9188	+
	9189	+ cur_prog = dev_xdp_prog(dev, mode);
	9190	+ /* can't replace attached prog with link */
	9191	+ if (link && cur_prog) {
	9192	+ NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
	9193	+ return -EBUSY;
	9194	+ }
	9195	+ if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
	9196	+ NL_SET_ERR_MSG(extack, "Active program does not match expected");
	9197	+ return -EEXIST;
	9198	+ }
	9199	+
	9200	+ /* put effective new program into new_prog */
	9201	+ if (link)
	9202	+ new_prog = link->link.prog;
	9203	+
	9204	+ if (new_prog) {
	9205	+ bool offload = mode == XDP_MODE_HW;
	9206	+ enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
	9207	+ ? XDP_MODE_DRV : XDP_MODE_SKB;
	9208	+
	9209	+ if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
	9210	+ NL_SET_ERR_MSG(extack, "XDP program already attached");
	9211	+ return -EBUSY;
	9212	+ }
	9213	+ if (!offload && dev_xdp_prog(dev, other_mode)) {
	9214	+ NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
	9215	+ return -EEXIST;
	9216	+ }
	9217	+ if (!offload && bpf_prog_is_dev_bound(new_prog->aux)) {
	9218	+ NL_SET_ERR_MSG(extack, "Using device-bound program without HW_MODE flag is not supported");
	9219	+ return -EINVAL;
	9220	+ }
	9221	+ if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
	9222	+ NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
	9223	+ return -EINVAL;
	9224	+ }
	9225	+ if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
	9226	+ NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
	9227	+ return -EINVAL;
	9228	+ }
	9229	+ }
	9230	+
	9231	+ /* don't call drivers if the effective program didn't change */
	9232	+ if (new_prog != cur_prog) {
	9233	+ bpf_op = dev_xdp_bpf_op(dev, mode);
	9234	+ if (!bpf_op) {
	9235	+ NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
	9236	+ return -EOPNOTSUPP;
	9237	+ }
	9238	+
	9239	+ err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
	9240	+ if (err)
	9241	+ return err;
	9242	+ }
	9243	+
	9244	+ if (link)
	9245	+ dev_xdp_set_link(dev, mode, link);
	9246	+ else
	9247	+ dev_xdp_set_prog(dev, mode, new_prog);
	9248	+ if (cur_prog)
	9249	+ bpf_prog_put(cur_prog);
	9250	+
	9251	+ return 0;
	9252	+}
	9253	+
	9254	+static int dev_xdp_attach_link(struct net_device *dev,
	9255	+ struct netlink_ext_ack *extack,
	9256	+ struct bpf_xdp_link *link)
	9257	+{
	9258	+ return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
	9259	+}
	9260	+
	9261	+static int dev_xdp_detach_link(struct net_device *dev,
	9262	+ struct netlink_ext_ack *extack,
	9263	+ struct bpf_xdp_link *link)
	9264	+{
	9265	+ enum bpf_xdp_mode mode;
	9266	+ bpf_op_t bpf_op;
	9267	+
	9268	+ ASSERT_RTNL();
	9269	+
	9270	+ mode = dev_xdp_mode(dev, link->flags);
	9271	+ if (dev_xdp_link(dev, mode) != link)
	9272	+ return -EINVAL;
	9273	+
	9274	+ bpf_op = dev_xdp_bpf_op(dev, mode);
	9275	+ WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
	9276	+ dev_xdp_set_link(dev, mode, NULL);
	9277	+ return 0;
	9278	+}
	9279	+
	9280	+static void bpf_xdp_link_release(struct bpf_link *link)
	9281	+{
	9282	+ struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
	9283	+
	9284	+ rtnl_lock();
	9285	+
	9286	+ /* if racing with net_device's tear down, xdp_link->dev might be
	9287	+ * already NULL, in which case link was already auto-detached
	9288	+ */
	9289	+ if (xdp_link->dev) {
	9290	+ WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
	9291	+ xdp_link->dev = NULL;
	9292	+ }
	9293	+
	9294	+ rtnl_unlock();
	9295	+}
	9296	+
	9297	+static int bpf_xdp_link_detach(struct bpf_link *link)
	9298	+{
	9299	+ bpf_xdp_link_release(link);
	9300	+ return 0;
	9301	+}
	9302	+
	9303	+static void bpf_xdp_link_dealloc(struct bpf_link *link)
	9304	+{
	9305	+ struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
	9306	+
	9307	+ kfree(xdp_link);
	9308	+}
	9309	+
	9310	+static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
	9311	+ struct seq_file *seq)
	9312	+{
	9313	+ struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
	9314	+ u32 ifindex = 0;
	9315	+
	9316	+ rtnl_lock();
	9317	+ if (xdp_link->dev)
	9318	+ ifindex = xdp_link->dev->ifindex;
	9319	+ rtnl_unlock();
	9320	+
	9321	+ seq_printf(seq, "ifindex:\t%u\n", ifindex);
	9322	+}
	9323	+
	9324	+static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
	9325	+ struct bpf_link_info *info)
	9326	+{
	9327	+ struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
	9328	+ u32 ifindex = 0;
	9329	+
	9330	+ rtnl_lock();
	9331	+ if (xdp_link->dev)
	9332	+ ifindex = xdp_link->dev->ifindex;
	9333	+ rtnl_unlock();
	9334	+
	9335	+ info->xdp.ifindex = ifindex;
	9336	+ return 0;
	9337	+}
	9338	+
	9339	+static int bpf_xdp_link_update(struct bpf_link link, struct bpf_prog new_prog,
	9340	+ struct bpf_prog *old_prog)
	9341	+{
	9342	+ struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
	9343	+ enum bpf_xdp_mode mode;
	9344	+ bpf_op_t bpf_op;
	9345	+ int err = 0;
	9346	+
	9347	+ rtnl_lock();
	9348	+
	9349	+ /* link might have been auto-released already, so fail */
	9350	+ if (!xdp_link->dev) {
	9351	+ err = -ENOLINK;
	9352	+ goto out_unlock;
	9353	+ }
	9354	+
	9355	+ if (old_prog && link->prog != old_prog) {
	9356	+ err = -EPERM;
	9357	+ goto out_unlock;
	9358	+ }
	9359	+ old_prog = link->prog;
	9360	+ if (old_prog->type != new_prog->type \|\|
	9361	+ old_prog->expected_attach_type != new_prog->expected_attach_type) {
	9362	+ err = -EINVAL;
	9363	+ goto out_unlock;
	9364	+ }
	9365	+
	9366	+ if (old_prog == new_prog) {
	9367	+ /* no-op, don't disturb drivers */
	9368	+ bpf_prog_put(new_prog);
	9369	+ goto out_unlock;
	9370	+ }
	9371	+
	9372	+ mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
	9373	+ bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
	9374	+ err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
	9375	+ xdp_link->flags, new_prog);
	9376	+ if (err)
	9377	+ goto out_unlock;
	9378	+
	9379	+ old_prog = xchg(&link->prog, new_prog);
	9380	+ bpf_prog_put(old_prog);
	9381	+
	9382	+out_unlock:
	9383	+ rtnl_unlock();
	9384	+ return err;
	9385	+}
	9386	+
	9387	+static const struct bpf_link_ops bpf_xdp_link_lops = {
	9388	+ .release = bpf_xdp_link_release,
	9389	+ .dealloc = bpf_xdp_link_dealloc,
	9390	+ .detach = bpf_xdp_link_detach,
	9391	+ .show_fdinfo = bpf_xdp_link_show_fdinfo,
	9392	+ .fill_link_info = bpf_xdp_link_fill_link_info,
	9393	+ .update_prog = bpf_xdp_link_update,
	9394	+};
	9395	+
	9396	+int bpf_xdp_link_attach(const union bpf_attr attr, struct bpf_prog prog)
	9397	+{
	9398	+ struct net *net = current->nsproxy->net_ns;
	9399	+ struct bpf_link_primer link_primer;
	9400	+ struct bpf_xdp_link *link;
	9401	+ struct net_device *dev;
	9402	+ int err, fd;
	9403	+
	9404	+ rtnl_lock();
	9405	+ dev = dev_get_by_index(net, attr->link_create.target_ifindex);
	9406	+ if (!dev) {
	9407	+ rtnl_unlock();
	9408	+ return -EINVAL;
	9409	+ }
	9410	+
	9411	+ link = kzalloc(sizeof(*link), GFP_USER);
	9412	+ if (!link) {
	9413	+ err = -ENOMEM;
	9414	+ goto unlock;
	9415	+ }
	9416	+
	9417	+ bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
	9418	+ link->dev = dev;
	9419	+ link->flags = attr->link_create.flags;
	9420	+
	9421	+ err = bpf_link_prime(&link->link, &link_primer);
	9422	+ if (err) {
	9423	+ kfree(link);
	9424	+ goto unlock;
	9425	+ }
	9426	+
	9427	+ err = dev_xdp_attach_link(dev, NULL, link);
	9428	+ rtnl_unlock();
	9429	+
	9430	+ if (err) {
	9431	+ link->dev = NULL;
	9432	+ bpf_link_cleanup(&link_primer);
	9433	+ goto out_put_dev;
	9434	+ }
	9435	+
	9436	+ fd = bpf_link_settle(&link_primer);
	9437	+ /* link itself doesn't hold dev's refcnt to not complicate shutdown */
	9438	+ dev_put(dev);
	9439	+ return fd;
	9440	+
	9441	+unlock:
	9442	+ rtnl_unlock();
	9443	+
	9444	+out_put_dev:
	9445	+ dev_put(dev);
	9446	+ return err;
8081	9447	}
8082	9448
8083	9449	/**
..	..	@@ -8085,56 +9451,44 @@
8085	9451	* @dev: device
8086	9452	* @extack: netlink extended ack
8087	9453	* @fd: new program fd or negative value to clear
	9454	+ * @expected_fd: old program fd that userspace expects to replace or clear
8088	9455	* @flags: xdp-related flags
8089	9456	*
8090	9457	* Set or clear a bpf program for a device
8091	9458	*/
8092	9459	int dev_change_xdp_fd(struct net_device dev, struct netlink_ext_ack extack,
8093		- int fd, u32 flags)
	9460	+ int fd, int expected_fd, u32 flags)
8094	9461	{
8095		- const struct net_device_ops *ops = dev->netdev_ops;
8096		- enum bpf_netdev_command query;
8097		- struct bpf_prog *prog = NULL;
8098		- bpf_op_t bpf_op, bpf_chk;
	9462	+ enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
	9463	+ struct bpf_prog new_prog = NULL, old_prog = NULL;
8099	9464	int err;
8100	9465
8101	9466	ASSERT_RTNL();
8102	9467
8103		- query = flags & XDP_FLAGS_HW_MODE ? XDP_QUERY_PROG_HW : XDP_QUERY_PROG;
8104		-
8105		- bpf_op = bpf_chk = ops->ndo_bpf;
8106		- if (!bpf_op && (flags & (XDP_FLAGS_DRV_MODE \| XDP_FLAGS_HW_MODE)))
8107		- return -EOPNOTSUPP;
8108		- if (!bpf_op \|\| (flags & XDP_FLAGS_SKB_MODE))
8109		- bpf_op = generic_xdp_install;
8110		- if (bpf_op == bpf_chk)
8111		- bpf_chk = generic_xdp_install;
8112		-
8113	9468	if (fd >= 0) {
8114		- if (__dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG) \|\|
8115		- __dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG_HW))
8116		- return -EEXIST;
8117		- if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
8118		- __dev_xdp_query(dev, bpf_op, query))
8119		- return -EBUSY;
	9469	+ new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
	9470	+ mode != XDP_MODE_SKB);
	9471	+ if (IS_ERR(new_prog))
	9472	+ return PTR_ERR(new_prog);
	9473	+ }
8120	9474
8121		- prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
8122		- bpf_op == ops->ndo_bpf);
8123		- if (IS_ERR(prog))
8124		- return PTR_ERR(prog);
8125		-
8126		- if (!(flags & XDP_FLAGS_HW_MODE) &&
8127		- bpf_prog_is_dev_bound(prog->aux)) {
8128		- NL_SET_ERR_MSG(extack, "using device-bound program without HW_MODE flag is not supported");
8129		- bpf_prog_put(prog);
8130		- return -EINVAL;
	9475	+ if (expected_fd >= 0) {
	9476	+ old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
	9477	+ mode != XDP_MODE_SKB);
	9478	+ if (IS_ERR(old_prog)) {
	9479	+ err = PTR_ERR(old_prog);
	9480	+ old_prog = NULL;
	9481	+ goto err_out;
8131	9482	}
8132	9483	}
8133	9484
8134		- err = dev_xdp_install(dev, bpf_op, extack, flags, prog);
8135		- if (err < 0 && prog)
8136		- bpf_prog_put(prog);
	9485	+ err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
8137	9486
	9487	+err_out:
	9488	+ if (err && new_prog)
	9489	+ bpf_prog_put(new_prog);
	9490	+ if (old_prog)
	9491	+ bpf_prog_put(old_prog);
8138	9492	return err;
8139	9493	}
8140	9494
..	..	@@ -8166,103 +9520,6 @@
8166	9520	{
8167	9521	list_add_tail(&dev->todo_list, &net_todo_list);
8168	9522	dev_net(dev)->dev_unreg_count++;
8169		-}
8170		-
8171		-static void rollback_registered_many(struct list_head *head)
8172		-{
8173		- struct net_device dev, tmp;
8174		- LIST_HEAD(close_head);
8175		-
8176		- BUG_ON(dev_boot_phase);
8177		- ASSERT_RTNL();
8178		-
8179		- list_for_each_entry_safe(dev, tmp, head, unreg_list) {
8180		- /* Some devices call without registering
8181		- * for initialization unwind. Remove those
8182		- * devices and proceed with the remaining.
8183		- */
8184		- if (dev->reg_state == NETREG_UNINITIALIZED) {
8185		- pr_debug("unregister_netdevice: device %s/%p never was registered\n",
8186		- dev->name, dev);
8187		-
8188		- WARN_ON(1);
8189		- list_del(&dev->unreg_list);
8190		- continue;
8191		- }
8192		- dev->dismantle = true;
8193		- BUG_ON(dev->reg_state != NETREG_REGISTERED);
8194		- }
8195		-
8196		- /* If device is running, close it first. */
8197		- list_for_each_entry(dev, head, unreg_list)
8198		- list_add_tail(&dev->close_list, &close_head);
8199		- dev_close_many(&close_head, true);
8200		-
8201		- list_for_each_entry(dev, head, unreg_list) {
8202		- /* And unlink it from device chain. */
8203		- unlist_netdevice(dev);
8204		-
8205		- dev->reg_state = NETREG_UNREGISTERING;
8206		- }
8207		- flush_all_backlogs();
8208		-
8209		- synchronize_net();
8210		-
8211		- list_for_each_entry(dev, head, unreg_list) {
8212		- struct sk_buff *skb = NULL;
8213		-
8214		- /* Shutdown queueing discipline. */
8215		- dev_shutdown(dev);
8216		-
8217		- dev_xdp_uninstall(dev);
8218		-
8219		- /* Notify protocols, that we are about to destroy
8220		- * this device. They should clean all the things.
8221		- */
8222		- call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
8223		-
8224		- if (!dev->rtnl_link_ops \|\|
8225		- dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
8226		- skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
8227		- GFP_KERNEL, NULL, 0);
8228		-
8229		- /*
8230		- * Flush the unicast and multicast chains
8231		- */
8232		- dev_uc_flush(dev);
8233		- dev_mc_flush(dev);
8234		-
8235		- if (dev->netdev_ops->ndo_uninit)
8236		- dev->netdev_ops->ndo_uninit(dev);
8237		-
8238		- if (skb)
8239		- rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
8240		-
8241		- /* Notifier chain MUST detach us all upper devices. */
8242		- WARN_ON(netdev_has_any_upper_dev(dev));
8243		- WARN_ON(netdev_has_any_lower_dev(dev));
8244		-
8245		- /* Remove entries from kobject tree */
8246		- netdev_unregister_kobject(dev);
8247		-#ifdef CONFIG_XPS
8248		- /* Remove XPS queueing entries */
8249		- netif_reset_xps_queues_gt(dev, 0);
8250		-#endif
8251		- }
8252		-
8253		- synchronize_net();
8254		-
8255		- list_for_each_entry(dev, head, unreg_list)
8256		- dev_put(dev);
8257		-}
8258		-
8259		-static void rollback_registered(struct net_device *dev)
8260		-{
8261		- LIST_HEAD(single);
8262		-
8263		- list_add(&dev->unreg_list, &single);
8264		- rollback_registered_many(&single);
8265		- list_del(&single);
8266	9523	}
8267	9524
8268	9525	static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
..	..	@@ -8410,7 +9667,7 @@
8410	9667	/* driver might be less strict about feature dependencies */
8411	9668	features = netdev_fix_features(dev, features);
8412	9669
8413		- /* some features can't be enabled if they're off an an upper device */
	9670	+ /* some features can't be enabled if they're off on an upper device */
8414	9671	netdev_for_each_upper_dev_rcu(dev, upper, iter)
8415	9672	features = netdev_sync_upper_features(dev, upper, features);
8416	9673
..	..	@@ -8533,6 +9790,11 @@
8533	9790	netif_dormant_on(dev);
8534	9791	else
8535	9792	netif_dormant_off(dev);
	9793	+
	9794	+ if (rootdev->operstate == IF_OPER_TESTING)
	9795	+ netif_testing_on(dev);
	9796	+ else
	9797	+ netif_testing_off(dev);
8536	9798
8537	9799	if (netif_carrier_ok(rootdev))
8538	9800	netif_carrier_on(dev);
..	..	@@ -8674,11 +9936,20 @@
8674	9936	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
8675	9937	BUG_ON(!net);
8676	9938
	9939	+ ret = ethtool_check_ops(dev->ethtool_ops);
	9940	+ if (ret)
	9941	+ return ret;
	9942	+
8677	9943	spin_lock_init(&dev->addr_list_lock);
8678	9944	netdev_set_addr_lockdep_class(dev);
8679	9945
8680	9946	ret = dev_get_valid_name(net, dev, dev->name);
8681	9947	if (ret < 0)
	9948	+ goto out;
	9949	+
	9950	+ ret = -ENOMEM;
	9951	+ dev->name_node = netdev_name_node_head_alloc(dev);
	9952	+ if (!dev->name_node)
8682	9953	goto out;
8683	9954
8684	9955	/* Init, if this function is available */
..	..	@@ -8687,7 +9958,7 @@
8687	9958	if (ret) {
8688	9959	if (ret > 0)
8689	9960	ret = -EIO;
8690		- goto out;
	9961	+ goto err_free_name;
8691	9962	}
8692	9963	}
8693	9964
..	..	@@ -8709,7 +9980,7 @@
8709	9980	/* Transfer changeable features to wanted_features and enable
8710	9981	* software offloads (GSO and GRO).
8711	9982	*/
8712		- dev->hw_features \|= NETIF_F_SOFT_FEATURES;
	9983	+ dev->hw_features \|= (NETIF_F_SOFT_FEATURES \| NETIF_F_SOFT_FEATURES_OFF);
8713	9984	dev->features \|= NETIF_F_SOFT_FEATURES;
8714	9985
8715	9986	if (dev->netdev_ops->ndo_udp_tunnel_add) {
..	..	@@ -8787,17 +10058,10 @@
8787	10058	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
8788	10059	ret = notifier_to_errno(ret);
8789	10060	if (ret) {
8790		- rollback_registered(dev);
8791		- rcu_barrier();
8792		-
8793		- dev->reg_state = NETREG_UNREGISTERED;
8794		- /* We should put the kobject that hold in
8795		- * netdev_unregister_kobject(), otherwise
8796		- * the net device cannot be freed when
8797		- * driver calls free_netdev(), because the
8798		- * kobject is being hold.
8799		- */
8800		- kobject_put(&dev->dev.kobj);
	10061	+ /* Expect explicit free_netdev() on failure */
	10062	+ dev->needs_free_netdev = false;
	10063	+ unregister_netdevice_queue(dev, NULL);
	10064	+ goto out;
8801	10065	}
8802	10066	/*
8803	10067	* Prevent userspace races by waiting until the network
..	..	@@ -8815,6 +10079,8 @@
8815	10079	dev->netdev_ops->ndo_uninit(dev);
8816	10080	if (dev->priv_destructor)
8817	10081	dev->priv_destructor(dev);
	10082	+err_free_name:
	10083	+ netdev_name_node_free(dev->name_node);
8818	10084	goto out;
8819	10085	}
8820	10086	EXPORT_SYMBOL(register_netdevice);
..	..	@@ -8898,6 +10164,8 @@
8898	10164	}
8899	10165	EXPORT_SYMBOL(netdev_refcnt_read);
8900	10166
	10167	+#define WAIT_REFS_MIN_MSECS 1
	10168	+#define WAIT_REFS_MAX_MSECS 250
8901	10169	/**
8902	10170	* netdev_wait_allrefs - wait until all references are gone.
8903	10171	* @dev: target net_device
..	..	@@ -8913,7 +10181,7 @@
8913	10181	static void netdev_wait_allrefs(struct net_device *dev)
8914	10182	{
8915	10183	unsigned long rebroadcast_time, warning_time;
8916		- int refcnt;
	10184	+ int wait = 0, refcnt;
8917	10185
8918	10186	linkwatch_forget_dev(dev);
8919	10187
..	..	@@ -8947,7 +10215,13 @@
8947	10215	rebroadcast_time = jiffies;
8948	10216	}
8949	10217
8950		- msleep(250);
	10218	+ if (!wait) {
	10219	+ rcu_barrier();
	10220	+ wait = WAIT_REFS_MIN_MSECS;
	10221	+ } else {
	10222	+ msleep(wait);
	10223	+ wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
	10224	+ }
8951	10225
8952	10226	refcnt = netdev_refcnt_read(dev);
8953	10227
..	..	@@ -8986,6 +10260,19 @@
8986	10260	void netdev_run_todo(void)
8987	10261	{
8988	10262	struct list_head list;
	10263	+#ifdef CONFIG_LOCKDEP
	10264	+ struct list_head unlink_list;
	10265	+
	10266	+ list_replace_init(&net_unlink_list, &unlink_list);
	10267	+
	10268	+ while (!list_empty(&unlink_list)) {
	10269	+ struct net_device *dev = list_first_entry(&unlink_list,
	10270	+ struct net_device,
	10271	+ unlink_list);
	10272	+ list_del_init(&dev->unlink_list);
	10273	+ dev->nested_level = dev->lower_level - 1;
	10274	+ }
	10275	+#endif
8989	10276
8990	10277	/* Snapshot list, allow later requests */
8991	10278	list_replace_init(&net_todo_list, &list);
..	..	@@ -9097,6 +10384,40 @@
9097	10384	}
9098	10385	EXPORT_SYMBOL(dev_get_stats);
9099	10386
	10387	+/**
	10388	+ * dev_fetch_sw_netstats - get per-cpu network device statistics
	10389	+ * @s: place to store stats
	10390	+ * @netstats: per-cpu network stats to read from
	10391	+ *
	10392	+ * Read per-cpu network statistics and populate the related fields in @s.
	10393	+ */
	10394	+void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
	10395	+ const struct pcpu_sw_netstats __percpu *netstats)
	10396	+{
	10397	+ int cpu;
	10398	+
	10399	+ for_each_possible_cpu(cpu) {
	10400	+ const struct pcpu_sw_netstats *stats;
	10401	+ struct pcpu_sw_netstats tmp;
	10402	+ unsigned int start;
	10403	+
	10404	+ stats = per_cpu_ptr(netstats, cpu);
	10405	+ do {
	10406	+ start = u64_stats_fetch_begin_irq(&stats->syncp);
	10407	+ tmp.rx_packets = stats->rx_packets;
	10408	+ tmp.rx_bytes = stats->rx_bytes;
	10409	+ tmp.tx_packets = stats->tx_packets;
	10410	+ tmp.tx_bytes = stats->tx_bytes;
	10411	+ } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
	10412	+
	10413	+ s->rx_packets += tmp.rx_packets;
	10414	+ s->rx_bytes += tmp.rx_bytes;
	10415	+ s->tx_packets += tmp.tx_packets;
	10416	+ s->tx_bytes += tmp.tx_bytes;
	10417	+ }
	10418	+}
	10419	+EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
	10420	+
9100	10421	struct netdev_queue dev_ingress_queue_create(struct net_device dev)
9101	10422	{
9102	10423	struct netdev_queue *queue = dev_ingress_queue(dev);
..	..	@@ -9198,6 +10519,10 @@
9198	10519	dev->gso_max_segs = GSO_MAX_SEGS;
9199	10520	dev->upper_level = 1;
9200	10521	dev->lower_level = 1;
	10522	+#ifdef CONFIG_LOCKDEP
	10523	+ dev->nested_level = 0;
	10524	+ INIT_LIST_HEAD(&dev->unlink_list);
	10525	+#endif
9201	10526
9202	10527	INIT_LIST_HEAD(&dev->napi_list);
9203	10528	INIT_LIST_HEAD(&dev->unreg_list);
..	..	@@ -9207,6 +10532,7 @@
9207	10532	INIT_LIST_HEAD(&dev->adj_list.lower);
9208	10533	INIT_LIST_HEAD(&dev->ptype_all);
9209	10534	INIT_LIST_HEAD(&dev->ptype_specific);
	10535	+ INIT_LIST_HEAD(&dev->net_notifier_list);
9210	10536	#ifdef CONFIG_NET_SCHED
9211	10537	hash_init(dev->qdisc_hash);
9212	10538	#endif
..	..	@@ -9264,6 +10590,17 @@
9264	10590	struct napi_struct p, n;
9265	10591
9266	10592	might_sleep();
	10593	+
	10594	+ /* When called immediately after register_netdevice() failed the unwind
	10595	+ * handling may still be dismantling the device. Handle that case by
	10596	+ * deferring the free.
	10597	+ */
	10598	+ if (dev->reg_state == NETREG_UNREGISTERING) {
	10599	+ ASSERT_RTNL();
	10600	+ dev->needs_free_netdev = true;
	10601	+ return;
	10602	+ }
	10603	+
9267	10604	netif_free_tx_queues(dev);
9268	10605	netif_free_rx_queues(dev);
9269	10606
..	..	@@ -9277,6 +10614,8 @@
9277	10614
9278	10615	free_percpu(dev->pcpu_refcnt);
9279	10616	dev->pcpu_refcnt = NULL;
	10617	+ free_percpu(dev->xdp_bulkq);
	10618	+ dev->xdp_bulkq = NULL;
9280	10619
9281	10620	/* Compatibility with error handling in drivers */
9282	10621	if (dev->reg_state == NETREG_UNINITIALIZED) {
..	..	@@ -9328,9 +10667,10 @@
9328	10667	if (head) {
9329	10668	list_move_tail(&dev->unreg_list, head);
9330	10669	} else {
9331		- rollback_registered(dev);
9332		- /* Finish processing unregister after unlock */
9333		- net_set_todo(dev);
	10670	+ LIST_HEAD(single);
	10671	+
	10672	+ list_add(&dev->unreg_list, &single);
	10673	+ unregister_netdevice_many(&single);
9334	10674	}
9335	10675	}
9336	10676	EXPORT_SYMBOL(unregister_netdevice_queue);
..	..	@@ -9344,14 +10684,100 @@
9344	10684	*/
9345	10685	void unregister_netdevice_many(struct list_head *head)
9346	10686	{
9347		- struct net_device *dev;
	10687	+ struct net_device dev, tmp;
	10688	+ LIST_HEAD(close_head);
9348	10689
9349		- if (!list_empty(head)) {
9350		- rollback_registered_many(head);
9351		- list_for_each_entry(dev, head, unreg_list)
9352		- net_set_todo(dev);
9353		- list_del(head);
	10690	+ BUG_ON(dev_boot_phase);
	10691	+ ASSERT_RTNL();
	10692	+
	10693	+ if (list_empty(head))
	10694	+ return;
	10695	+
	10696	+ list_for_each_entry_safe(dev, tmp, head, unreg_list) {
	10697	+ /* Some devices call without registering
	10698	+ * for initialization unwind. Remove those
	10699	+ * devices and proceed with the remaining.
	10700	+ */
	10701	+ if (dev->reg_state == NETREG_UNINITIALIZED) {
	10702	+ pr_debug("unregister_netdevice: device %s/%p never was registered\n",
	10703	+ dev->name, dev);
	10704	+
	10705	+ WARN_ON(1);
	10706	+ list_del(&dev->unreg_list);
	10707	+ continue;
	10708	+ }
	10709	+ dev->dismantle = true;
	10710	+ BUG_ON(dev->reg_state != NETREG_REGISTERED);
9354	10711	}
	10712	+
	10713	+ /* If device is running, close it first. */
	10714	+ list_for_each_entry(dev, head, unreg_list)
	10715	+ list_add_tail(&dev->close_list, &close_head);
	10716	+ dev_close_many(&close_head, true);
	10717	+
	10718	+ list_for_each_entry(dev, head, unreg_list) {
	10719	+ /* And unlink it from device chain. */
	10720	+ unlist_netdevice(dev);
	10721	+
	10722	+ dev->reg_state = NETREG_UNREGISTERING;
	10723	+ }
	10724	+ flush_all_backlogs();
	10725	+
	10726	+ synchronize_net();
	10727	+
	10728	+ list_for_each_entry(dev, head, unreg_list) {
	10729	+ struct sk_buff *skb = NULL;
	10730	+
	10731	+ /* Shutdown queueing discipline. */
	10732	+ dev_shutdown(dev);
	10733	+
	10734	+ dev_xdp_uninstall(dev);
	10735	+
	10736	+ /* Notify protocols, that we are about to destroy
	10737	+ * this device. They should clean all the things.
	10738	+ */
	10739	+ call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
	10740	+
	10741	+ if (!dev->rtnl_link_ops \|\|
	10742	+ dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
	10743	+ skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
	10744	+ GFP_KERNEL, NULL, 0);
	10745	+
	10746	+ /*
	10747	+ * Flush the unicast and multicast chains
	10748	+ */
	10749	+ dev_uc_flush(dev);
	10750	+ dev_mc_flush(dev);
	10751	+
	10752	+ netdev_name_node_alt_flush(dev);
	10753	+ netdev_name_node_free(dev->name_node);
	10754	+
	10755	+ if (dev->netdev_ops->ndo_uninit)
	10756	+ dev->netdev_ops->ndo_uninit(dev);
	10757	+
	10758	+ if (skb)
	10759	+ rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
	10760	+
	10761	+ /* Notifier chain MUST detach us all upper devices. */
	10762	+ WARN_ON(netdev_has_any_upper_dev(dev));
	10763	+ WARN_ON(netdev_has_any_lower_dev(dev));
	10764	+
	10765	+ /* Remove entries from kobject tree */
	10766	+ netdev_unregister_kobject(dev);
	10767	+#ifdef CONFIG_XPS
	10768	+ /* Remove XPS queueing entries */
	10769	+ netif_reset_xps_queues_gt(dev, 0);
	10770	+#endif
	10771	+ }
	10772	+
	10773	+ synchronize_net();
	10774	+
	10775	+ list_for_each_entry(dev, head, unreg_list) {
	10776	+ dev_put(dev);
	10777	+ net_set_todo(dev);
	10778	+ }
	10779	+
	10780	+ list_del(head);
9355	10781	}
9356	10782	EXPORT_SYMBOL(unregister_netdevice_many);
9357	10783
..	..	@@ -9390,6 +10816,7 @@
9390	10816
9391	10817	int dev_change_net_namespace(struct net_device dev, struct net net, const char *pat)
9392	10818	{
	10819	+ struct net *net_old = dev_net(dev);
9393	10820	int err, new_nsid, new_ifindex;
9394	10821
9395	10822	ASSERT_RTNL();
..	..	@@ -9405,7 +10832,7 @@
9405	10832
9406	10833	/* Get out if there is nothing todo */
9407	10834	err = 0;
9408		- if (net_eq(dev_net(dev), net))
	10835	+ if (net_eq(net_old, net))
9409	10836	goto out;
9410	10837
9411	10838	/* Pick the destination device name, and ensure
..	..	@@ -9466,6 +10893,9 @@
9466	10893	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
9467	10894	netdev_adjacent_del_links(dev);
9468	10895
	10896	+ /* Move per-net netdevice notifiers that are following the netdevice */
	10897	+ move_netdevice_notifiers_dev_net(dev, net);
	10898	+
9469	10899	/* Actually switch the network namespace */
9470	10900	dev_net_set(dev, net);
9471	10901	dev->ifindex = new_ifindex;
..	..	@@ -9476,6 +10906,12 @@
9476	10906
9477	10907	/* Fixup kobjects */
9478	10908	err = device_rename(&dev->dev, dev->name);
	10909	+ WARN_ON(err);
	10910	+
	10911	+ /* Adapt owner in case owning user namespace of target network
	10912	+ * namespace is different from the original one.
	10913	+ */
	10914	+ err = netdev_change_owner(dev, net_old, net);
9479	10915	WARN_ON(err);
9480	10916
9481	10917	/* Add the device back in the hashes */
..	..	@@ -9542,6 +10978,7 @@
9542	10978
9543	10979	raise_softirq_irqoff(NET_TX_SOFTIRQ);
9544	10980	local_irq_enable();
	10981	+ preempt_check_resched_rt();
9545	10982
9546	10983	#ifdef CONFIG_RPS
9547	10984	remsd = oldsd->rps_ipi_list;
..	..	@@ -9555,7 +10992,7 @@
9555	10992	netif_rx_ni(skb);
9556	10993	input_queue_head_incr(oldsd);
9557	10994	}
9558		- while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
	10995	+ while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
9559	10996	netif_rx_ni(skb);
9560	10997	input_queue_head_incr(oldsd);
9561	10998	}
..	..	@@ -9608,7 +11045,7 @@
9608	11045	static int __net_init netdev_init(struct net *net)
9609	11046	{
9610	11047	BUILD_BUG_ON(GRO_HASH_BUCKETS >
9611		- 8 * FIELD_SIZEOF(struct napi_struct, gro_bitmask));
	11048	+ 8 * sizeof_field(struct napi_struct, gro_bitmask));
9612	11049
9613	11050	if (net != &init_net)
9614	11051	INIT_LIST_HEAD(&net->dev_base_head);
..	..	@@ -9620,6 +11057,8 @@
9620	11057	net->dev_index_head = netdev_create_hash();
9621	11058	if (net->dev_index_head == NULL)
9622	11059	goto err_idx;
	11060	+
	11061	+ RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
9623	11062
9624	11063	return 0;
9625	11064
..	..	@@ -9742,7 +11181,7 @@
9742	11181	continue;
9743	11182
9744	11183	/* Leave virtual devices for the generic cleanup */
9745		- if (dev->rtnl_link_ops)
	11184	+ if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
9746	11185	continue;
9747	11186
9748	11187	/* Push remaining network devices to init_net */
..	..	@@ -9869,7 +11308,7 @@
9869	11308
9870	11309	INIT_WORK(flush, flush_backlog);
9871	11310
9872		- skb_queue_head_init(&sd->input_pkt_queue);
	11311	+ skb_queue_head_init_raw(&sd->input_pkt_queue);
9873	11312	skb_queue_head_init(&sd->process_queue);
9874	11313	#ifdef CONFIG_XFRM_OFFLOAD
9875	11314	skb_queue_head_init(&sd->xfrm_backlog);