~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,17 +1,12 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* net/sched/cls_api.c Packet classifier API.
3		- *
4		- * This program is free software; you can redistribute it and/or
5		- * modify it under the terms of the GNU General Public License
6		- * as published by the Free Software Foundation; either version
7		- * 2 of the License, or (at your option) any later version.
8	4	*
9	5	* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10	6	*
11	7	* Changes:
12	8	*
13	9	* Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
14		- *
15	10	*/
16	11
17	12	#include <linux/module.h>
..	..	@@ -25,11 +20,26 @@
25	20	#include <linux/kmod.h>
26	21	#include <linux/slab.h>
27	22	#include <linux/idr.h>
	23	+#include <linux/jhash.h>
	24	+#include <linux/rculist.h>
28	25	#include <net/net_namespace.h>
29	26	#include <net/sock.h>
30	27	#include <net/netlink.h>
31	28	#include <net/pkt_sched.h>
32	29	#include <net/pkt_cls.h>
	30	+#include <net/tc_act/tc_pedit.h>
	31	+#include <net/tc_act/tc_mirred.h>
	32	+#include <net/tc_act/tc_vlan.h>
	33	+#include <net/tc_act/tc_tunnel_key.h>
	34	+#include <net/tc_act/tc_csum.h>
	35	+#include <net/tc_act/tc_gact.h>
	36	+#include <net/tc_act/tc_police.h>
	37	+#include <net/tc_act/tc_sample.h>
	38	+#include <net/tc_act/tc_skbedit.h>
	39	+#include <net/tc_act/tc_ct.h>
	40	+#include <net/tc_act/tc_mpls.h>
	41	+#include <net/tc_act/tc_gate.h>
	42	+#include <net/flow_offload.h>
33	43
34	44	extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
35	45
..	..	@@ -38,6 +48,62 @@
38	48
39	49	/* Protects list of registered TC modules. It is pure SMP lock. */
40	50	static DEFINE_RWLOCK(cls_mod_lock);
	51	+
	52	+static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
	53	+{
	54	+ return jhash_3words(tp->chain->index, tp->prio,
	55	+ (__force __u32)tp->protocol, 0);
	56	+}
	57	+
	58	+static void tcf_proto_signal_destroying(struct tcf_chain *chain,
	59	+ struct tcf_proto *tp)
	60	+{
	61	+ struct tcf_block *block = chain->block;
	62	+
	63	+ mutex_lock(&block->proto_destroy_lock);
	64	+ hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
	65	+ destroy_obj_hashfn(tp));
	66	+ mutex_unlock(&block->proto_destroy_lock);
	67	+}
	68	+
	69	+static bool tcf_proto_cmp(const struct tcf_proto *tp1,
	70	+ const struct tcf_proto *tp2)
	71	+{
	72	+ return tp1->chain->index == tp2->chain->index &&
	73	+ tp1->prio == tp2->prio &&
	74	+ tp1->protocol == tp2->protocol;
	75	+}
	76	+
	77	+static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
	78	+ struct tcf_proto *tp)
	79	+{
	80	+ u32 hash = destroy_obj_hashfn(tp);
	81	+ struct tcf_proto *iter;
	82	+ bool found = false;
	83	+
	84	+ rcu_read_lock();
	85	+ hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
	86	+ destroy_ht_node, hash) {
	87	+ if (tcf_proto_cmp(tp, iter)) {
	88	+ found = true;
	89	+ break;
	90	+ }
	91	+ }
	92	+ rcu_read_unlock();
	93	+
	94	+ return found;
	95	+}
	96	+
	97	+static void
	98	+tcf_proto_signal_destroyed(struct tcf_chain chain, struct tcf_proto tp)
	99	+{
	100	+ struct tcf_block *block = chain->block;
	101	+
	102	+ mutex_lock(&block->proto_destroy_lock);
	103	+ if (hash_hashed(&tp->destroy_ht_node))
	104	+ hash_del_rcu(&tp->destroy_ht_node);
	105	+ mutex_unlock(&block->proto_destroy_lock);
	106	+}
41	107
42	108	/* Find classifier type by string name */
43	109
..	..	@@ -60,7 +126,8 @@
60	126	}
61	127
62	128	static const struct tcf_proto_ops *
63		-tcf_proto_lookup_ops(const char kind, struct netlink_ext_ack extack)
	129	+tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
	130	+ struct netlink_ext_ack *extack)
64	131	{
65	132	const struct tcf_proto_ops *ops;
66	133
..	..	@@ -68,9 +135,11 @@
68	135	if (ops)
69	136	return ops;
70	137	#ifdef CONFIG_MODULES
71		- rtnl_unlock();
	138	+ if (rtnl_held)
	139	+ rtnl_unlock();
72	140	request_module("cls_%s", kind);
73		- rtnl_lock();
	141	+ if (rtnl_held)
	142	+ rtnl_lock();
74	143	ops = __tcf_proto_lookup_ops(kind);
75	144	/* We dropped the RTNL semaphore in order to perform
76	145	* the module load. So, even if we succeeded in loading
..	..	@@ -151,8 +220,37 @@
151	220	return TC_H_MAJ(first);
152	221	}
153	222
	223	+static bool tcf_proto_check_kind(struct nlattr kind, char name)
	224	+{
	225	+ if (kind)
	226	+ return nla_strlcpy(name, kind, IFNAMSIZ) >= IFNAMSIZ;
	227	+ memset(name, 0, IFNAMSIZ);
	228	+ return false;
	229	+}
	230	+
	231	+static bool tcf_proto_is_unlocked(const char *kind)
	232	+{
	233	+ const struct tcf_proto_ops *ops;
	234	+ bool ret;
	235	+
	236	+ if (strlen(kind) == 0)
	237	+ return false;
	238	+
	239	+ ops = tcf_proto_lookup_ops(kind, false, NULL);
	240	+ /* On error return false to take rtnl lock. Proto lookup/create
	241	+ * functions will perform lookup again and properly handle errors.
	242	+ */
	243	+ if (IS_ERR(ops))
	244	+ return false;
	245	+
	246	+ ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
	247	+ module_put(ops->owner);
	248	+ return ret;
	249	+}
	250	+
154	251	static struct tcf_proto tcf_proto_create(const char kind, u32 protocol,
155	252	u32 prio, struct tcf_chain *chain,
	253	+ bool rtnl_held,
156	254	struct netlink_ext_ack *extack)
157	255	{
158	256	struct tcf_proto *tp;
..	..	@@ -162,7 +260,7 @@
162	260	if (!tp)
163	261	return ERR_PTR(-ENOBUFS);
164	262
165		- tp->ops = tcf_proto_lookup_ops(kind, extack);
	263	+ tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
166	264	if (IS_ERR(tp->ops)) {
167	265	err = PTR_ERR(tp->ops);
168	266	goto errout;
..	..	@@ -171,6 +269,8 @@
171	269	tp->protocol = protocol;
172	270	tp->prio = prio;
173	271	tp->chain = chain;
	272	+ spin_lock_init(&tp->lock);
	273	+ refcount_set(&tp->refcnt, 1);
174	274
175	275	err = tp->ops->init(tp);
176	276	if (err) {
..	..	@@ -184,13 +284,60 @@
184	284	return ERR_PTR(err);
185	285	}
186	286
187		-static void tcf_proto_destroy(struct tcf_proto *tp,
188		- struct netlink_ext_ack *extack)
	287	+static void tcf_proto_get(struct tcf_proto *tp)
189	288	{
190		- tp->ops->destroy(tp, extack);
	289	+ refcount_inc(&tp->refcnt);
	290	+}
	291	+
	292	+static void tcf_chain_put(struct tcf_chain *chain);
	293	+
	294	+static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
	295	+ bool sig_destroy, struct netlink_ext_ack *extack)
	296	+{
	297	+ tp->ops->destroy(tp, rtnl_held, extack);
	298	+ if (sig_destroy)
	299	+ tcf_proto_signal_destroyed(tp->chain, tp);
	300	+ tcf_chain_put(tp->chain);
191	301	module_put(tp->ops->owner);
192	302	kfree_rcu(tp, rcu);
193	303	}
	304	+
	305	+static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
	306	+ struct netlink_ext_ack *extack)
	307	+{
	308	+ if (refcount_dec_and_test(&tp->refcnt))
	309	+ tcf_proto_destroy(tp, rtnl_held, true, extack);
	310	+}
	311	+
	312	+static bool tcf_proto_check_delete(struct tcf_proto *tp)
	313	+{
	314	+ if (tp->ops->delete_empty)
	315	+ return tp->ops->delete_empty(tp);
	316	+
	317	+ tp->deleting = true;
	318	+ return tp->deleting;
	319	+}
	320	+
	321	+static void tcf_proto_mark_delete(struct tcf_proto *tp)
	322	+{
	323	+ spin_lock(&tp->lock);
	324	+ tp->deleting = true;
	325	+ spin_unlock(&tp->lock);
	326	+}
	327	+
	328	+static bool tcf_proto_is_deleting(struct tcf_proto *tp)
	329	+{
	330	+ bool deleting;
	331	+
	332	+ spin_lock(&tp->lock);
	333	+ deleting = tp->deleting;
	334	+ spin_unlock(&tp->lock);
	335	+
	336	+ return deleting;
	337	+}
	338	+
	339	+#define ASSERT_BLOCK_LOCKED(block) \
	340	+ lockdep_assert_held(&(block)->lock)
194	341
195	342	struct tcf_filter_chain_list_item {
196	343	struct list_head list;
..	..	@@ -203,10 +350,13 @@
203	350	{
204	351	struct tcf_chain *chain;
205	352
	353	+ ASSERT_BLOCK_LOCKED(block);
	354	+
206	355	chain = kzalloc(sizeof(*chain), GFP_KERNEL);
207	356	if (!chain)
208	357	return NULL;
209		- list_add_tail(&chain->list, &block->chain_list);
	358	+ list_add_tail_rcu(&chain->list, &block->chain_list);
	359	+ mutex_init(&chain->filter_chain_lock);
210	360	chain->block = block;
211	361	chain->index = chain_index;
212	362	chain->refcnt = 1;
..	..	@@ -230,29 +380,60 @@
230	380
231	381	if (chain->index)
232	382	return;
	383	+
	384	+ mutex_lock(&block->lock);
233	385	list_for_each_entry(item, &block->chain0.filter_chain_list, list)
234	386	tcf_chain_head_change_item(item, tp_head);
	387	+ mutex_unlock(&block->lock);
235	388	}
236	389
237		-static void tcf_chain_destroy(struct tcf_chain *chain)
	390	+/* Returns true if block can be safely freed. */
	391	+
	392	+static bool tcf_chain_detach(struct tcf_chain *chain)
238	393	{
239	394	struct tcf_block *block = chain->block;
240	395
241		- list_del(&chain->list);
	396	+ ASSERT_BLOCK_LOCKED(block);
	397	+
	398	+ list_del_rcu(&chain->list);
242	399	if (!chain->index)
243	400	block->chain0.chain = NULL;
244		- kfree(chain);
245		- if (list_empty(&block->chain_list) && block->refcnt == 0)
246		- kfree(block);
	401	+
	402	+ if (list_empty(&block->chain_list) &&
	403	+ refcount_read(&block->refcnt) == 0)
	404	+ return true;
	405	+
	406	+ return false;
	407	+}
	408	+
	409	+static void tcf_block_destroy(struct tcf_block *block)
	410	+{
	411	+ mutex_destroy(&block->lock);
	412	+ mutex_destroy(&block->proto_destroy_lock);
	413	+ kfree_rcu(block, rcu);
	414	+}
	415	+
	416	+static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
	417	+{
	418	+ struct tcf_block *block = chain->block;
	419	+
	420	+ mutex_destroy(&chain->filter_chain_lock);
	421	+ kfree_rcu(chain, rcu);
	422	+ if (free_block)
	423	+ tcf_block_destroy(block);
247	424	}
248	425
249	426	static void tcf_chain_hold(struct tcf_chain *chain)
250	427	{
	428	+ ASSERT_BLOCK_LOCKED(chain->block);
	429	+
251	430	++chain->refcnt;
252	431	}
253	432
254	433	static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
255	434	{
	435	+ ASSERT_BLOCK_LOCKED(chain->block);
	436	+
256	437	/* In case all the references are action references, this
257	438	* chain should not be shown to the user.
258	439	*/
..	..	@@ -264,12 +445,28 @@
264	445	{
265	446	struct tcf_chain *chain;
266	447
	448	+ ASSERT_BLOCK_LOCKED(block);
	449	+
267	450	list_for_each_entry(chain, &block->chain_list, list) {
268	451	if (chain->index == chain_index)
269	452	return chain;
270	453	}
271	454	return NULL;
272	455	}
	456	+
	457	+#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
	458	+static struct tcf_chain tcf_chain_lookup_rcu(const struct tcf_block block,
	459	+ u32 chain_index)
	460	+{
	461	+ struct tcf_chain *chain;
	462	+
	463	+ list_for_each_entry_rcu(chain, &block->chain_list, list) {
	464	+ if (chain->index == chain_index)
	465	+ return chain;
	466	+ }
	467	+ return NULL;
	468	+}
	469	+#endif
273	470
274	471	static int tc_chain_notify(struct tcf_chain chain, struct sk_buff oskb,
275	472	u32 seq, u16 flags, int event, bool unicast);
..	..	@@ -278,30 +475,39 @@
278	475	u32 chain_index, bool create,
279	476	bool by_act)
280	477	{
281		- struct tcf_chain *chain = tcf_chain_lookup(block, chain_index);
	478	+ struct tcf_chain *chain = NULL;
	479	+ bool is_first_reference;
282	480
	481	+ mutex_lock(&block->lock);
	482	+ chain = tcf_chain_lookup(block, chain_index);
283	483	if (chain) {
284	484	tcf_chain_hold(chain);
285	485	} else {
286	486	if (!create)
287		- return NULL;
	487	+ goto errout;
288	488	chain = tcf_chain_create(block, chain_index);
289	489	if (!chain)
290		- return NULL;
	490	+ goto errout;
291	491	}
292	492
293	493	if (by_act)
294	494	++chain->action_refcnt;
	495	+ is_first_reference = chain->refcnt - chain->action_refcnt == 1;
	496	+ mutex_unlock(&block->lock);
295	497
296	498	/* Send notification only in case we got the first
297	499	* non-action reference. Until then, the chain acts only as
298	500	* a placeholder for actions pointing to it and user ought
299	501	* not know about them.
300	502	*/
301		- if (chain->refcnt - chain->action_refcnt == 1 && !by_act)
	503	+ if (is_first_reference && !by_act)
302	504	tc_chain_notify(chain, NULL, 0, NLM_F_CREATE \| NLM_F_EXCL,
303	505	RTM_NEWCHAIN, false);
304	506
	507	+ return chain;
	508	+
	509	+errout:
	510	+ mutex_unlock(&block->lock);
305	511	return chain;
306	512	}
307	513
..	..	@@ -317,72 +523,180 @@
317	523	}
318	524	EXPORT_SYMBOL(tcf_chain_get_by_act);
319	525
320		-static void tc_chain_tmplt_del(struct tcf_chain *chain);
	526	+static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
	527	+ void *tmplt_priv);
	528	+static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
	529	+ void *tmplt_priv, u32 chain_index,
	530	+ struct tcf_block block, struct sk_buff oskb,
	531	+ u32 seq, u16 flags, bool unicast);
321	532
322		-static void __tcf_chain_put(struct tcf_chain *chain, bool by_act)
	533	+static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
	534	+ bool explicitly_created)
323	535	{
	536	+ struct tcf_block *block = chain->block;
	537	+ const struct tcf_proto_ops *tmplt_ops;
	538	+ bool free_block = false;
	539	+ unsigned int refcnt;
	540	+ void *tmplt_priv;
	541	+
	542	+ mutex_lock(&block->lock);
	543	+ if (explicitly_created) {
	544	+ if (!chain->explicitly_created) {
	545	+ mutex_unlock(&block->lock);
	546	+ return;
	547	+ }
	548	+ chain->explicitly_created = false;
	549	+ }
	550	+
324	551	if (by_act)
325	552	chain->action_refcnt--;
326		- chain->refcnt--;
	553	+
	554	+ /* tc_chain_notify_delete can't be called while holding block lock.
	555	+ * However, when block is unlocked chain can be changed concurrently, so
	556	+ * save these to temporary variables.
	557	+ */
	558	+ refcnt = --chain->refcnt;
	559	+ tmplt_ops = chain->tmplt_ops;
	560	+ tmplt_priv = chain->tmplt_priv;
327	561
328	562	/* The last dropped non-action reference will trigger notification. */
329		- if (chain->refcnt - chain->action_refcnt == 0 && !by_act)
330		- tc_chain_notify(chain, NULL, 0, 0, RTM_DELCHAIN, false);
	563	+ if (refcnt - chain->action_refcnt == 0 && !by_act) {
	564	+ tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index,
	565	+ block, NULL, 0, 0, false);
	566	+ /* Last reference to chain, no need to lock. */
	567	+ chain->flushing = false;
	568	+ }
331	569
332		- if (chain->refcnt == 0) {
333		- tc_chain_tmplt_del(chain);
334		- tcf_chain_destroy(chain);
	570	+ if (refcnt == 0)
	571	+ free_block = tcf_chain_detach(chain);
	572	+ mutex_unlock(&block->lock);
	573	+
	574	+ if (refcnt == 0) {
	575	+ tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
	576	+ tcf_chain_destroy(chain, free_block);
335	577	}
336	578	}
337	579
338	580	static void tcf_chain_put(struct tcf_chain *chain)
339	581	{
340		- __tcf_chain_put(chain, false);
	582	+ __tcf_chain_put(chain, false, false);
341	583	}
342	584
343	585	void tcf_chain_put_by_act(struct tcf_chain *chain)
344	586	{
345		- __tcf_chain_put(chain, true);
	587	+ __tcf_chain_put(chain, true, false);
346	588	}
347	589	EXPORT_SYMBOL(tcf_chain_put_by_act);
348	590
349	591	static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
350	592	{
351		- if (chain->explicitly_created)
352		- tcf_chain_put(chain);
	593	+ __tcf_chain_put(chain, false, true);
353	594	}
354	595
355		-static void tcf_chain_flush(struct tcf_chain *chain)
	596	+static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
356	597	{
357		- struct tcf_proto *tp = rtnl_dereference(chain->filter_chain);
	598	+ struct tcf_proto tp, tp_next;
358	599
359		- tcf_chain0_head_change(chain, NULL);
	600	+ mutex_lock(&chain->filter_chain_lock);
	601	+ tp = tcf_chain_dereference(chain->filter_chain, chain);
360	602	while (tp) {
361		- RCU_INIT_POINTER(chain->filter_chain, tp->next);
362		- tcf_proto_destroy(tp, NULL);
363		- tp = rtnl_dereference(chain->filter_chain);
364		- tcf_chain_put(chain);
	603	+ tp_next = rcu_dereference_protected(tp->next, 1);
	604	+ tcf_proto_signal_destroying(chain, tp);
	605	+ tp = tp_next;
365	606	}
	607	+ tp = tcf_chain_dereference(chain->filter_chain, chain);
	608	+ RCU_INIT_POINTER(chain->filter_chain, NULL);
	609	+ tcf_chain0_head_change(chain, NULL);
	610	+ chain->flushing = true;
	611	+ mutex_unlock(&chain->filter_chain_lock);
	612	+
	613	+ while (tp) {
	614	+ tp_next = rcu_dereference_protected(tp->next, 1);
	615	+ tcf_proto_put(tp, rtnl_held, NULL);
	616	+ tp = tp_next;
	617	+ }
	618	+}
	619	+
	620	+static int tcf_block_setup(struct tcf_block *block,
	621	+ struct flow_block_offload *bo);
	622	+
	623	+static void tcf_block_offload_init(struct flow_block_offload *bo,
	624	+ struct net_device dev, struct Qdisc sch,
	625	+ enum flow_block_command command,
	626	+ enum flow_block_binder_type binder_type,
	627	+ struct flow_block *flow_block,
	628	+ bool shared, struct netlink_ext_ack *extack)
	629	+{
	630	+ bo->net = dev_net(dev);
	631	+ bo->command = command;
	632	+ bo->binder_type = binder_type;
	633	+ bo->block = flow_block;
	634	+ bo->block_shared = shared;
	635	+ bo->extack = extack;
	636	+ bo->sch = sch;
	637	+ bo->cb_list_head = &flow_block->cb_list;
	638	+ INIT_LIST_HEAD(&bo->cb_list);
	639	+}
	640	+
	641	+static void tcf_block_unbind(struct tcf_block *block,
	642	+ struct flow_block_offload *bo);
	643	+
	644	+static void tc_block_indr_cleanup(struct flow_block_cb *block_cb)
	645	+{
	646	+ struct tcf_block *block = block_cb->indr.data;
	647	+ struct net_device *dev = block_cb->indr.dev;
	648	+ struct Qdisc *sch = block_cb->indr.sch;
	649	+ struct netlink_ext_ack extack = {};
	650	+ struct flow_block_offload bo = {};
	651	+
	652	+ tcf_block_offload_init(&bo, dev, sch, FLOW_BLOCK_UNBIND,
	653	+ block_cb->indr.binder_type,
	654	+ &block->flow_block, tcf_block_shared(block),
	655	+ &extack);
	656	+ rtnl_lock();
	657	+ down_write(&block->cb_lock);
	658	+ list_del(&block_cb->driver_list);
	659	+ list_move(&block_cb->list, &bo.cb_list);
	660	+ tcf_block_unbind(block, &bo);
	661	+ up_write(&block->cb_lock);
	662	+ rtnl_unlock();
366	663	}
367	664
368	665	static bool tcf_block_offload_in_use(struct tcf_block *block)
369	666	{
370		- return block->offloadcnt;
	667	+ return atomic_read(&block->offloadcnt);
371	668	}
372	669
373	670	static int tcf_block_offload_cmd(struct tcf_block *block,
374		- struct net_device *dev,
	671	+ struct net_device dev, struct Qdisc sch,
375	672	struct tcf_block_ext_info *ei,
376		- enum tc_block_command command,
	673	+ enum flow_block_command command,
377	674	struct netlink_ext_ack *extack)
378	675	{
379		- struct tc_block_offload bo = {};
	676	+ struct flow_block_offload bo = {};
380	677
381		- bo.command = command;
382		- bo.binder_type = ei->binder_type;
383		- bo.block = block;
384		- bo.extack = extack;
385		- return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	678	+ tcf_block_offload_init(&bo, dev, sch, command, ei->binder_type,
	679	+ &block->flow_block, tcf_block_shared(block),
	680	+ extack);
	681	+
	682	+ if (dev->netdev_ops->ndo_setup_tc) {
	683	+ int err;
	684	+
	685	+ err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	686	+ if (err < 0) {
	687	+ if (err != -EOPNOTSUPP)
	688	+ NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed");
	689	+ return err;
	690	+ }
	691	+
	692	+ return tcf_block_setup(block, &bo);
	693	+ }
	694	+
	695	+ flow_indr_dev_setup_offload(dev, sch, TC_SETUP_BLOCK, block, &bo,
	696	+ tc_block_indr_cleanup);
	697	+ tcf_block_setup(block, &bo);
	698	+
	699	+ return -EOPNOTSUPP;
386	700	}
387	701
388	702	static int tcf_block_offload_bind(struct tcf_block block, struct Qdisc q,
..	..	@@ -392,27 +706,37 @@
392	706	struct net_device *dev = q->dev_queue->dev;
393	707	int err;
394	708
395		- if (!dev->netdev_ops->ndo_setup_tc)
396		- goto no_offload_dev_inc;
	709	+ down_write(&block->cb_lock);
397	710
398	711	/* If tc offload feature is disabled and the block we try to bind
399	712	* to already has some offloaded filters, forbid to bind.
400	713	*/
401		- if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) {
	714	+ if (dev->netdev_ops->ndo_setup_tc &&
	715	+ !tc_can_offload(dev) &&
	716	+ tcf_block_offload_in_use(block)) {
402	717	NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
403		- return -EOPNOTSUPP;
	718	+ err = -EOPNOTSUPP;
	719	+ goto err_unlock;
404	720	}
405	721
406		- err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_BIND, extack);
	722	+ err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_BIND, extack);
407	723	if (err == -EOPNOTSUPP)
408	724	goto no_offload_dev_inc;
409		- return err;
	725	+ if (err)
	726	+ goto err_unlock;
	727	+
	728	+ up_write(&block->cb_lock);
	729	+ return 0;
410	730
411	731	no_offload_dev_inc:
412	732	if (tcf_block_offload_in_use(block))
413		- return -EOPNOTSUPP;
	733	+ goto err_unlock;
	734	+
	735	+ err = 0;
414	736	block->nooffloaddevcnt++;
415		- return 0;
	737	+err_unlock:
	738	+ up_write(&block->cb_lock);
	739	+ return err;
416	740	}
417	741
418	742	static void tcf_block_offload_unbind(struct tcf_block block, struct Qdisc q,
..	..	@@ -421,15 +745,16 @@
421	745	struct net_device *dev = q->dev_queue->dev;
422	746	int err;
423	747
424		- if (!dev->netdev_ops->ndo_setup_tc)
425		- goto no_offload_dev_dec;
426		- err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_UNBIND, NULL);
	748	+ down_write(&block->cb_lock);
	749	+ err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_UNBIND, NULL);
427	750	if (err == -EOPNOTSUPP)
428	751	goto no_offload_dev_dec;
	752	+ up_write(&block->cb_lock);
429	753	return;
430	754
431	755	no_offload_dev_dec:
432	756	WARN_ON(block->nooffloaddevcnt-- == 0);
	757	+ up_write(&block->cb_lock);
433	758	}
434	759
435	760	static int
..	..	@@ -437,8 +762,8 @@
437	762	struct tcf_block_ext_info *ei,
438	763	struct netlink_ext_ack *extack)
439	764	{
440		- struct tcf_chain *chain0 = block->chain0.chain;
441	765	struct tcf_filter_chain_list_item *item;
	766	+ struct tcf_chain *chain0;
442	767
443	768	item = kmalloc(sizeof(*item), GFP_KERNEL);
444	769	if (!item) {
..	..	@@ -447,9 +772,32 @@
447	772	}
448	773	item->chain_head_change = ei->chain_head_change;
449	774	item->chain_head_change_priv = ei->chain_head_change_priv;
450		- if (chain0 && chain0->filter_chain)
451		- tcf_chain_head_change_item(item, chain0->filter_chain);
452		- list_add(&item->list, &block->chain0.filter_chain_list);
	775	+
	776	+ mutex_lock(&block->lock);
	777	+ chain0 = block->chain0.chain;
	778	+ if (chain0)
	779	+ tcf_chain_hold(chain0);
	780	+ else
	781	+ list_add(&item->list, &block->chain0.filter_chain_list);
	782	+ mutex_unlock(&block->lock);
	783	+
	784	+ if (chain0) {
	785	+ struct tcf_proto *tp_head;
	786	+
	787	+ mutex_lock(&chain0->filter_chain_lock);
	788	+
	789	+ tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
	790	+ if (tp_head)
	791	+ tcf_chain_head_change_item(item, tp_head);
	792	+
	793	+ mutex_lock(&block->lock);
	794	+ list_add(&item->list, &block->chain0.filter_chain_list);
	795	+ mutex_unlock(&block->lock);
	796	+
	797	+ mutex_unlock(&chain0->filter_chain_lock);
	798	+ tcf_chain_put(chain0);
	799	+ }
	800	+
453	801	return 0;
454	802	}
455	803
..	..	@@ -457,24 +805,28 @@
457	805	tcf_chain0_head_change_cb_del(struct tcf_block *block,
458	806	struct tcf_block_ext_info *ei)
459	807	{
460		- struct tcf_chain *chain0 = block->chain0.chain;
461	808	struct tcf_filter_chain_list_item *item;
462	809
	810	+ mutex_lock(&block->lock);
463	811	list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
464	812	if ((!ei->chain_head_change && !ei->chain_head_change_priv) \|\|
465	813	(item->chain_head_change == ei->chain_head_change &&
466	814	item->chain_head_change_priv == ei->chain_head_change_priv)) {
467		- if (chain0)
	815	+ if (block->chain0.chain)
468	816	tcf_chain_head_change_item(item, NULL);
469	817	list_del(&item->list);
	818	+ mutex_unlock(&block->lock);
	819	+
470	820	kfree(item);
471	821	return;
472	822	}
473	823	}
	824	+ mutex_unlock(&block->lock);
474	825	WARN_ON(1);
475	826	}
476	827
477	828	struct tcf_net {
	829	+ spinlock_t idr_lock; /* Protects idr */
478	830	struct idr idr;
479	831	};
480	832
..	..	@@ -484,16 +836,25 @@
484	836	struct netlink_ext_ack *extack)
485	837	{
486	838	struct tcf_net *tn = net_generic(net, tcf_net_id);
	839	+ int err;
487	840
488		- return idr_alloc_u32(&tn->idr, block, &block->index, block->index,
489		- GFP_KERNEL);
	841	+ idr_preload(GFP_KERNEL);
	842	+ spin_lock(&tn->idr_lock);
	843	+ err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
	844	+ GFP_NOWAIT);
	845	+ spin_unlock(&tn->idr_lock);
	846	+ idr_preload_end();
	847	+
	848	+ return err;
490	849	}
491	850
492	851	static void tcf_block_remove(struct tcf_block block, struct net net)
493	852	{
494	853	struct tcf_net *tn = net_generic(net, tcf_net_id);
495	854
	855	+ spin_lock(&tn->idr_lock);
496	856	idr_remove(&tn->idr, block->index);
	857	+ spin_unlock(&tn->idr_lock);
497	858	}
498	859
499	860	static struct tcf_block tcf_block_create(struct net net, struct Qdisc *q,
..	..	@@ -507,12 +868,15 @@
507	868	NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
508	869	return ERR_PTR(-ENOMEM);
509	870	}
	871	+ mutex_init(&block->lock);
	872	+ mutex_init(&block->proto_destroy_lock);
	873	+ init_rwsem(&block->cb_lock);
	874	+ flow_block_init(&block->flow_block);
510	875	INIT_LIST_HEAD(&block->chain_list);
511		- INIT_LIST_HEAD(&block->cb_list);
512	876	INIT_LIST_HEAD(&block->owner_list);
513	877	INIT_LIST_HEAD(&block->chain0.filter_chain_list);
514	878
515		- block->refcnt = 1;
	879	+ refcount_set(&block->refcnt, 1);
516	880	block->net = net;
517	881	block->index = block_index;
518	882
..	..	@@ -529,6 +893,301 @@
529	893	return idr_find(&tn->idr, block_index);
530	894	}
531	895
	896	+static struct tcf_block tcf_block_refcnt_get(struct net net, u32 block_index)
	897	+{
	898	+ struct tcf_block *block;
	899	+
	900	+ rcu_read_lock();
	901	+ block = tcf_block_lookup(net, block_index);
	902	+ if (block && !refcount_inc_not_zero(&block->refcnt))
	903	+ block = NULL;
	904	+ rcu_read_unlock();
	905	+
	906	+ return block;
	907	+}
	908	+
	909	+static struct tcf_chain *
	910	+__tcf_get_next_chain(struct tcf_block block, struct tcf_chain chain)
	911	+{
	912	+ mutex_lock(&block->lock);
	913	+ if (chain)
	914	+ chain = list_is_last(&chain->list, &block->chain_list) ?
	915	+ NULL : list_next_entry(chain, list);
	916	+ else
	917	+ chain = list_first_entry_or_null(&block->chain_list,
	918	+ struct tcf_chain, list);
	919	+
	920	+ /* skip all action-only chains */
	921	+ while (chain && tcf_chain_held_by_acts_only(chain))
	922	+ chain = list_is_last(&chain->list, &block->chain_list) ?
	923	+ NULL : list_next_entry(chain, list);
	924	+
	925	+ if (chain)
	926	+ tcf_chain_hold(chain);
	927	+ mutex_unlock(&block->lock);
	928	+
	929	+ return chain;
	930	+}
	931	+
	932	+/* Function to be used by all clients that want to iterate over all chains on
	933	+ * block. It properly obtains block->lock and takes reference to chain before
	934	+ * returning it. Users of this function must be tolerant to concurrent chain
	935	+ * insertion/deletion or ensure that no concurrent chain modification is
	936	+ * possible. Note that all netlink dump callbacks cannot guarantee to provide
	937	+ * consistent dump because rtnl lock is released each time skb is filled with
	938	+ * data and sent to user-space.
	939	+ */
	940	+
	941	+struct tcf_chain *
	942	+tcf_get_next_chain(struct tcf_block block, struct tcf_chain chain)
	943	+{
	944	+ struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
	945	+
	946	+ if (chain)
	947	+ tcf_chain_put(chain);
	948	+
	949	+ return chain_next;
	950	+}
	951	+EXPORT_SYMBOL(tcf_get_next_chain);
	952	+
	953	+static struct tcf_proto *
	954	+__tcf_get_next_proto(struct tcf_chain chain, struct tcf_proto tp)
	955	+{
	956	+ u32 prio = 0;
	957	+
	958	+ ASSERT_RTNL();
	959	+ mutex_lock(&chain->filter_chain_lock);
	960	+
	961	+ if (!tp) {
	962	+ tp = tcf_chain_dereference(chain->filter_chain, chain);
	963	+ } else if (tcf_proto_is_deleting(tp)) {
	964	+ /* 'deleting' flag is set and chain->filter_chain_lock was
	965	+ * unlocked, which means next pointer could be invalid. Restart
	966	+ * search.
	967	+ */
	968	+ prio = tp->prio + 1;
	969	+ tp = tcf_chain_dereference(chain->filter_chain, chain);
	970	+
	971	+ for (; tp; tp = tcf_chain_dereference(tp->next, chain))
	972	+ if (!tp->deleting && tp->prio >= prio)
	973	+ break;
	974	+ } else {
	975	+ tp = tcf_chain_dereference(tp->next, chain);
	976	+ }
	977	+
	978	+ if (tp)
	979	+ tcf_proto_get(tp);
	980	+
	981	+ mutex_unlock(&chain->filter_chain_lock);
	982	+
	983	+ return tp;
	984	+}
	985	+
	986	+/* Function to be used by all clients that want to iterate over all tp's on
	987	+ * chain. Users of this function must be tolerant to concurrent tp
	988	+ * insertion/deletion or ensure that no concurrent chain modification is
	989	+ * possible. Note that all netlink dump callbacks cannot guarantee to provide
	990	+ * consistent dump because rtnl lock is released each time skb is filled with
	991	+ * data and sent to user-space.
	992	+ */
	993	+
	994	+struct tcf_proto *
	995	+tcf_get_next_proto(struct tcf_chain chain, struct tcf_proto tp,
	996	+ bool rtnl_held)
	997	+{
	998	+ struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
	999	+
	1000	+ if (tp)
	1001	+ tcf_proto_put(tp, rtnl_held, NULL);
	1002	+
	1003	+ return tp_next;
	1004	+}
	1005	+EXPORT_SYMBOL(tcf_get_next_proto);
	1006	+
	1007	+static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
	1008	+{
	1009	+ struct tcf_chain *chain;
	1010	+
	1011	+ /* Last reference to block. At this point chains cannot be added or
	1012	+ * removed concurrently.
	1013	+ */
	1014	+ for (chain = tcf_get_next_chain(block, NULL);
	1015	+ chain;
	1016	+ chain = tcf_get_next_chain(block, chain)) {
	1017	+ tcf_chain_put_explicitly_created(chain);
	1018	+ tcf_chain_flush(chain, rtnl_held);
	1019	+ }
	1020	+}
	1021	+
	1022	+/* Lookup Qdisc and increments its reference counter.
	1023	+ * Set parent, if necessary.
	1024	+ */
	1025	+
	1026	+static int __tcf_qdisc_find(struct net net, struct Qdisc *q,
	1027	+ u32 *parent, int ifindex, bool rtnl_held,
	1028	+ struct netlink_ext_ack *extack)
	1029	+{
	1030	+ const struct Qdisc_class_ops *cops;
	1031	+ struct net_device *dev;
	1032	+ int err = 0;
	1033	+
	1034	+ if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
	1035	+ return 0;
	1036	+
	1037	+ rcu_read_lock();
	1038	+
	1039	+ /* Find link */
	1040	+ dev = dev_get_by_index_rcu(net, ifindex);
	1041	+ if (!dev) {
	1042	+ rcu_read_unlock();
	1043	+ return -ENODEV;
	1044	+ }
	1045	+
	1046	+ /* Find qdisc */
	1047	+ if (!*parent) {
	1048	+ *q = rcu_dereference(dev->qdisc);
	1049	+ parent = (q)->handle;
	1050	+ } else {
	1051	+ q = qdisc_lookup_rcu(dev, TC_H_MAJ(parent));
	1052	+ if (!*q) {
	1053	+ NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
	1054	+ err = -EINVAL;
	1055	+ goto errout_rcu;
	1056	+ }
	1057	+ }
	1058	+
	1059	+ q = qdisc_refcount_inc_nz(q);
	1060	+ if (!*q) {
	1061	+ NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
	1062	+ err = -EINVAL;
	1063	+ goto errout_rcu;
	1064	+ }
	1065	+
	1066	+ /* Is it classful? */
	1067	+ cops = (*q)->ops->cl_ops;
	1068	+ if (!cops) {
	1069	+ NL_SET_ERR_MSG(extack, "Qdisc not classful");
	1070	+ err = -EINVAL;
	1071	+ goto errout_qdisc;
	1072	+ }
	1073	+
	1074	+ if (!cops->tcf_block) {
	1075	+ NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
	1076	+ err = -EOPNOTSUPP;
	1077	+ goto errout_qdisc;
	1078	+ }
	1079	+
	1080	+errout_rcu:
	1081	+ /* At this point we know that qdisc is not noop_qdisc,
	1082	+ * which means that qdisc holds a reference to net_device
	1083	+ * and we hold a reference to qdisc, so it is safe to release
	1084	+ * rcu read lock.
	1085	+ */
	1086	+ rcu_read_unlock();
	1087	+ return err;
	1088	+
	1089	+errout_qdisc:
	1090	+ rcu_read_unlock();
	1091	+
	1092	+ if (rtnl_held)
	1093	+ qdisc_put(*q);
	1094	+ else
	1095	+ qdisc_put_unlocked(*q);
	1096	+ *q = NULL;
	1097	+
	1098	+ return err;
	1099	+}
	1100	+
	1101	+static int __tcf_qdisc_cl_find(struct Qdisc q, u32 parent, unsigned long cl,
	1102	+ int ifindex, struct netlink_ext_ack *extack)
	1103	+{
	1104	+ if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
	1105	+ return 0;
	1106	+
	1107	+ /* Do we search for filter, attached to class? */
	1108	+ if (TC_H_MIN(parent)) {
	1109	+ const struct Qdisc_class_ops *cops = q->ops->cl_ops;
	1110	+
	1111	+ *cl = cops->find(q, parent);
	1112	+ if (*cl == 0) {
	1113	+ NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
	1114	+ return -ENOENT;
	1115	+ }
	1116	+ }
	1117	+
	1118	+ return 0;
	1119	+}
	1120	+
	1121	+static struct tcf_block __tcf_block_find(struct net net, struct Qdisc *q,
	1122	+ unsigned long cl, int ifindex,
	1123	+ u32 block_index,
	1124	+ struct netlink_ext_ack *extack)
	1125	+{
	1126	+ struct tcf_block *block;
	1127	+
	1128	+ if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
	1129	+ block = tcf_block_refcnt_get(net, block_index);
	1130	+ if (!block) {
	1131	+ NL_SET_ERR_MSG(extack, "Block of given index was not found");
	1132	+ return ERR_PTR(-EINVAL);
	1133	+ }
	1134	+ } else {
	1135	+ const struct Qdisc_class_ops *cops = q->ops->cl_ops;
	1136	+
	1137	+ block = cops->tcf_block(q, cl, extack);
	1138	+ if (!block)
	1139	+ return ERR_PTR(-EINVAL);
	1140	+
	1141	+ if (tcf_block_shared(block)) {
	1142	+ NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
	1143	+ return ERR_PTR(-EOPNOTSUPP);
	1144	+ }
	1145	+
	1146	+ /* Always take reference to block in order to support execution
	1147	+ * of rules update path of cls API without rtnl lock. Caller
	1148	+ * must release block when it is finished using it. 'if' block
	1149	+ * of this conditional obtain reference to block by calling
	1150	+ * tcf_block_refcnt_get().
	1151	+ */
	1152	+ refcount_inc(&block->refcnt);
	1153	+ }
	1154	+
	1155	+ return block;
	1156	+}
	1157	+
	1158	+static void __tcf_block_put(struct tcf_block block, struct Qdisc q,
	1159	+ struct tcf_block_ext_info *ei, bool rtnl_held)
	1160	+{
	1161	+ if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
	1162	+ /* Flushing/putting all chains will cause the block to be
	1163	+ * deallocated when last chain is freed. However, if chain_list
	1164	+ * is empty, block has to be manually deallocated. After block
	1165	+ * reference counter reached 0, it is no longer possible to
	1166	+ * increment it or add new chains to block.
	1167	+ */
	1168	+ bool free_block = list_empty(&block->chain_list);
	1169	+
	1170	+ mutex_unlock(&block->lock);
	1171	+ if (tcf_block_shared(block))
	1172	+ tcf_block_remove(block, block->net);
	1173	+
	1174	+ if (q)
	1175	+ tcf_block_offload_unbind(block, q, ei);
	1176	+
	1177	+ if (free_block)
	1178	+ tcf_block_destroy(block);
	1179	+ else
	1180	+ tcf_block_flush_all_chains(block, rtnl_held);
	1181	+ } else if (q) {
	1182	+ tcf_block_offload_unbind(block, q, ei);
	1183	+ }
	1184	+}
	1185	+
	1186	+static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
	1187	+{
	1188	+ __tcf_block_put(block, NULL, NULL, rtnl_held);
	1189	+}
	1190	+
532	1191	/* Find tcf block.
533	1192	* Set q, parent, cl when appropriate.
534	1193	*/
..	..	@@ -541,121 +1200,60 @@
541	1200	struct tcf_block *block;
542	1201	int err = 0;
543	1202
544		- if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
545		- block = tcf_block_lookup(net, block_index);
546		- if (!block) {
547		- NL_SET_ERR_MSG(extack, "Block of given index was not found");
548		- return ERR_PTR(-EINVAL);
549		- }
550		- } else {
551		- const struct Qdisc_class_ops *cops;
552		- struct net_device *dev;
	1203	+ ASSERT_RTNL();
553	1204
554		- rcu_read_lock();
	1205	+ err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
	1206	+ if (err)
	1207	+ goto errout;
555	1208
556		- /* Find link */
557		- dev = dev_get_by_index_rcu(net, ifindex);
558		- if (!dev) {
559		- rcu_read_unlock();
560		- return ERR_PTR(-ENODEV);
561		- }
	1209	+ err = __tcf_qdisc_cl_find(q, parent, cl, ifindex, extack);
	1210	+ if (err)
	1211	+ goto errout_qdisc;
562	1212
563		- /* Find qdisc */
564		- if (!*parent) {
565		- *q = dev->qdisc;
566		- parent = (q)->handle;
567		- } else {
568		- q = qdisc_lookup_rcu(dev, TC_H_MAJ(parent));
569		- if (!*q) {
570		- NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
571		- err = -EINVAL;
572		- goto errout_rcu;
573		- }
574		- }
575		-
576		- q = qdisc_refcount_inc_nz(q);
577		- if (!*q) {
578		- NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
579		- err = -EINVAL;
580		- goto errout_rcu;
581		- }
582		-
583		- /* Is it classful? */
584		- cops = (*q)->ops->cl_ops;
585		- if (!cops) {
586		- NL_SET_ERR_MSG(extack, "Qdisc not classful");
587		- err = -EINVAL;
588		- goto errout_rcu;
589		- }
590		-
591		- if (!cops->tcf_block) {
592		- NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
593		- err = -EOPNOTSUPP;
594		- goto errout_rcu;
595		- }
596		-
597		- /* At this point we know that qdisc is not noop_qdisc,
598		- * which means that qdisc holds a reference to net_device
599		- * and we hold a reference to qdisc, so it is safe to release
600		- * rcu read lock.
601		- */
602		- rcu_read_unlock();
603		-
604		- /* Do we search for filter, attached to class? */
605		- if (TC_H_MIN(*parent)) {
606		- cl = cops->find(q, *parent);
607		- if (*cl == 0) {
608		- NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
609		- err = -ENOENT;
610		- goto errout_qdisc;
611		- }
612		- }
613		-
614		- /* And the last stroke */
615		- block = cops->tcf_block(q, cl, extack);
616		- if (!block) {
617		- err = -EINVAL;
618		- goto errout_qdisc;
619		- }
620		- if (tcf_block_shared(block)) {
621		- NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
622		- err = -EOPNOTSUPP;
623		- goto errout_qdisc;
624		- }
	1213	+ block = __tcf_block_find(net, q, cl, ifindex, block_index, extack);
	1214	+ if (IS_ERR(block)) {
	1215	+ err = PTR_ERR(block);
	1216	+ goto errout_qdisc;
625	1217	}
626	1218
627	1219	return block;
628	1220
629		-errout_rcu:
630		- rcu_read_unlock();
631	1221	errout_qdisc:
632		- if (*q) {
	1222	+ if (*q)
633	1223	qdisc_put(*q);
634		- *q = NULL;
635		- }
	1224	+errout:
	1225	+ *q = NULL;
636	1226	return ERR_PTR(err);
637	1227	}
638	1228
639		-static void tcf_block_release(struct Qdisc q, struct tcf_block block)
	1229	+static void tcf_block_release(struct Qdisc q, struct tcf_block block,
	1230	+ bool rtnl_held)
640	1231	{
641		- if (q)
642		- qdisc_put(q);
	1232	+ if (!IS_ERR_OR_NULL(block))
	1233	+ tcf_block_refcnt_put(block, rtnl_held);
	1234	+
	1235	+ if (q) {
	1236	+ if (rtnl_held)
	1237	+ qdisc_put(q);
	1238	+ else
	1239	+ qdisc_put_unlocked(q);
	1240	+ }
643	1241	}
644	1242
645	1243	struct tcf_block_owner_item {
646	1244	struct list_head list;
647	1245	struct Qdisc *q;
648		- enum tcf_block_binder_type binder_type;
	1246	+ enum flow_block_binder_type binder_type;
649	1247	};
650	1248
651	1249	static void
652	1250	tcf_block_owner_netif_keep_dst(struct tcf_block *block,
653	1251	struct Qdisc *q,
654		- enum tcf_block_binder_type binder_type)
	1252	+ enum flow_block_binder_type binder_type)
655	1253	{
656	1254	if (block->keep_dst &&
657		- binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
658		- binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
	1255	+ binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
	1256	+ binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
659	1257	netif_keep_dst(qdisc_dev(q));
660	1258	}
661	1259
..	..	@@ -672,7 +1270,7 @@
672	1270
673	1271	static int tcf_block_owner_add(struct tcf_block *block,
674	1272	struct Qdisc *q,
675		- enum tcf_block_binder_type binder_type)
	1273	+ enum flow_block_binder_type binder_type)
676	1274	{
677	1275	struct tcf_block_owner_item *item;
678	1276
..	..	@@ -687,7 +1285,7 @@
687	1285
688	1286	static void tcf_block_owner_del(struct tcf_block *block,
689	1287	struct Qdisc *q,
690		- enum tcf_block_binder_type binder_type)
	1288	+ enum flow_block_binder_type binder_type)
691	1289	{
692	1290	struct tcf_block_owner_item *item;
693	1291
..	..	@@ -707,21 +1305,16 @@
707	1305	{
708	1306	struct net *net = qdisc_net(q);
709	1307	struct tcf_block *block = NULL;
710		- bool created = false;
711	1308	int err;
712	1309
713		- if (ei->block_index) {
	1310	+ if (ei->block_index)
714	1311	/* block_index not 0 means the shared block is requested */
715		- block = tcf_block_lookup(net, ei->block_index);
716		- if (block)
717		- block->refcnt++;
718		- }
	1312	+ block = tcf_block_refcnt_get(net, ei->block_index);
719	1313
720	1314	if (!block) {
721	1315	block = tcf_block_create(net, q, ei->block_index, extack);
722	1316	if (IS_ERR(block))
723	1317	return PTR_ERR(block);
724		- created = true;
725	1318	if (tcf_block_shared(block)) {
726	1319	err = tcf_block_insert(block, net, extack);
727	1320	if (err)
..	..	@@ -751,14 +1344,8 @@
751	1344	err_chain0_head_change_cb_add:
752	1345	tcf_block_owner_del(block, q, ei->binder_type);
753	1346	err_block_owner_add:
754		- if (created) {
755		- if (tcf_block_shared(block))
756		- tcf_block_remove(block, net);
757	1347	err_block_insert:
758		- kfree(block);
759		- } else {
760		- block->refcnt--;
761		- }
	1348	+ tcf_block_refcnt_put(block, true);
762	1349	return err;
763	1350	}
764	1351	EXPORT_SYMBOL(tcf_block_get_ext);
..	..	@@ -790,42 +1377,12 @@
790	1377	void tcf_block_put_ext(struct tcf_block block, struct Qdisc q,
791	1378	struct tcf_block_ext_info *ei)
792	1379	{
793		- struct tcf_chain chain, tmp;
794		-
795	1380	if (!block)
796	1381	return;
797	1382	tcf_chain0_head_change_cb_del(block, ei);
798	1383	tcf_block_owner_del(block, q, ei->binder_type);
799	1384
800		- if (block->refcnt == 1) {
801		- if (tcf_block_shared(block))
802		- tcf_block_remove(block, block->net);
803		-
804		- /* Hold a refcnt for all chains, so that they don't disappear
805		- * while we are iterating.
806		- */
807		- list_for_each_entry(chain, &block->chain_list, list)
808		- tcf_chain_hold(chain);
809		-
810		- list_for_each_entry(chain, &block->chain_list, list)
811		- tcf_chain_flush(chain);
812		- }
813		-
814		- tcf_block_offload_unbind(block, q, ei);
815		-
816		- if (block->refcnt == 1) {
817		- /* At this point, all the chains should have refcnt >= 1. */
818		- list_for_each_entry_safe(chain, tmp, &block->chain_list, list) {
819		- tcf_chain_put_explicitly_created(chain);
820		- tcf_chain_put(chain);
821		- }
822		-
823		- block->refcnt--;
824		- if (list_empty(&block->chain_list))
825		- kfree(block);
826		- } else {
827		- block->refcnt--;
828		- }
	1385	+ __tcf_block_put(block, q, ei, true);
829	1386	}
830	1387	EXPORT_SYMBOL(tcf_block_put_ext);
831	1388
..	..	@@ -840,55 +1397,26 @@
840	1397
841	1398	EXPORT_SYMBOL(tcf_block_put);
842	1399
843		-struct tcf_block_cb {
844		- struct list_head list;
845		- tc_setup_cb_t *cb;
846		- void *cb_ident;
847		- void *cb_priv;
848		- unsigned int refcnt;
849		-};
850		-
851		-void tcf_block_cb_priv(struct tcf_block_cb block_cb)
852		-{
853		- return block_cb->cb_priv;
854		-}
855		-EXPORT_SYMBOL(tcf_block_cb_priv);
856		-
857		-struct tcf_block_cb tcf_block_cb_lookup(struct tcf_block block,
858		- tc_setup_cb_t cb, void cb_ident)
859		-{ struct tcf_block_cb *block_cb;
860		-
861		- list_for_each_entry(block_cb, &block->cb_list, list)
862		- if (block_cb->cb == cb && block_cb->cb_ident == cb_ident)
863		- return block_cb;
864		- return NULL;
865		-}
866		-EXPORT_SYMBOL(tcf_block_cb_lookup);
867		-
868		-void tcf_block_cb_incref(struct tcf_block_cb *block_cb)
869		-{
870		- block_cb->refcnt++;
871		-}
872		-EXPORT_SYMBOL(tcf_block_cb_incref);
873		-
874		-unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb)
875		-{
876		- return --block_cb->refcnt;
877		-}
878		-EXPORT_SYMBOL(tcf_block_cb_decref);
879		-
880	1400	static int
881		-tcf_block_playback_offloads(struct tcf_block block, tc_setup_cb_t cb,
	1401	+tcf_block_playback_offloads(struct tcf_block block, flow_setup_cb_t cb,
882	1402	void *cb_priv, bool add, bool offload_in_use,
883	1403	struct netlink_ext_ack *extack)
884	1404	{
885		- struct tcf_chain *chain;
886		- struct tcf_proto *tp;
	1405	+ struct tcf_chain chain, chain_prev;
	1406	+ struct tcf_proto tp, tp_prev;
887	1407	int err;
888	1408
889		- list_for_each_entry(chain, &block->chain_list, list) {
890		- for (tp = rtnl_dereference(chain->filter_chain); tp;
891		- tp = rtnl_dereference(tp->next)) {
	1409	+ lockdep_assert_held(&block->cb_lock);
	1410	+
	1411	+ for (chain = __tcf_get_next_chain(block, NULL);
	1412	+ chain;
	1413	+ chain_prev = chain,
	1414	+ chain = __tcf_get_next_chain(block, chain),
	1415	+ tcf_chain_put(chain_prev)) {
	1416	+ for (tp = __tcf_get_next_proto(chain, NULL); tp;
	1417	+ tp_prev = tp,
	1418	+ tp = __tcf_get_next_proto(chain, tp),
	1419	+ tcf_proto_put(tp_prev, true, NULL)) {
892	1420	if (tp->ops->reoffload) {
893	1421	err = tp->ops->reoffload(tp, add, cb, cb_priv,
894	1422	extack);
..	..	@@ -905,105 +1433,107 @@
905	1433	return 0;
906	1434
907	1435	err_playback_remove:
	1436	+ tcf_proto_put(tp, true, NULL);
	1437	+ tcf_chain_put(chain);
908	1438	tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
909	1439	extack);
910	1440	return err;
911	1441	}
912	1442
913		-struct tcf_block_cb __tcf_block_cb_register(struct tcf_block block,
914		- tc_setup_cb_t cb, void cb_ident,
915		- void *cb_priv,
916		- struct netlink_ext_ack *extack)
	1443	+static int tcf_block_bind(struct tcf_block *block,
	1444	+ struct flow_block_offload *bo)
917	1445	{
918		- struct tcf_block_cb *block_cb;
919		- int err;
	1446	+ struct flow_block_cb block_cb, next;
	1447	+ int err, i = 0;
920	1448
921		- /* Replay any already present rules */
922		- err = tcf_block_playback_offloads(block, cb, cb_priv, true,
923		- tcf_block_offload_in_use(block),
924		- extack);
925		- if (err)
926		- return ERR_PTR(err);
	1449	+ lockdep_assert_held(&block->cb_lock);
927	1450
928		- block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL);
929		- if (!block_cb)
930		- return ERR_PTR(-ENOMEM);
931		- block_cb->cb = cb;
932		- block_cb->cb_ident = cb_ident;
933		- block_cb->cb_priv = cb_priv;
934		- list_add(&block_cb->list, &block->cb_list);
935		- return block_cb;
936		-}
937		-EXPORT_SYMBOL(__tcf_block_cb_register);
	1451	+ list_for_each_entry(block_cb, &bo->cb_list, list) {
	1452	+ err = tcf_block_playback_offloads(block, block_cb->cb,
	1453	+ block_cb->cb_priv, true,
	1454	+ tcf_block_offload_in_use(block),
	1455	+ bo->extack);
	1456	+ if (err)
	1457	+ goto err_unroll;
	1458	+ if (!bo->unlocked_driver_cb)
	1459	+ block->lockeddevcnt++;
938	1460
939		-int tcf_block_cb_register(struct tcf_block *block,
940		- tc_setup_cb_t cb, void cb_ident,
941		- void cb_priv, struct netlink_ext_ack extack)
942		-{
943		- struct tcf_block_cb *block_cb;
944		-
945		- block_cb = __tcf_block_cb_register(block, cb, cb_ident, cb_priv,
946		- extack);
947		- return PTR_ERR_OR_ZERO(block_cb);
948		-}
949		-EXPORT_SYMBOL(tcf_block_cb_register);
950		-
951		-void __tcf_block_cb_unregister(struct tcf_block *block,
952		- struct tcf_block_cb *block_cb)
953		-{
954		- tcf_block_playback_offloads(block, block_cb->cb, block_cb->cb_priv,
955		- false, tcf_block_offload_in_use(block),
956		- NULL);
957		- list_del(&block_cb->list);
958		- kfree(block_cb);
959		-}
960		-EXPORT_SYMBOL(__tcf_block_cb_unregister);
961		-
962		-void tcf_block_cb_unregister(struct tcf_block *block,
963		- tc_setup_cb_t cb, void cb_ident)
964		-{
965		- struct tcf_block_cb *block_cb;
966		-
967		- block_cb = tcf_block_cb_lookup(block, cb, cb_ident);
968		- if (!block_cb)
969		- return;
970		- __tcf_block_cb_unregister(block, block_cb);
971		-}
972		-EXPORT_SYMBOL(tcf_block_cb_unregister);
973		-
974		-static int tcf_block_cb_call(struct tcf_block *block, enum tc_setup_type type,
975		- void *type_data, bool err_stop)
976		-{
977		- struct tcf_block_cb *block_cb;
978		- int ok_count = 0;
979		- int err;
980		-
981		- /* Make sure all netdevs sharing this block are offload-capable. */
982		- if (block->nooffloaddevcnt && err_stop)
983		- return -EOPNOTSUPP;
984		-
985		- list_for_each_entry(block_cb, &block->cb_list, list) {
986		- err = block_cb->cb(type, type_data, block_cb->cb_priv);
987		- if (err) {
988		- if (err_stop)
989		- return err;
990		- } else {
991		- ok_count++;
992		- }
	1461	+ i++;
993	1462	}
994		- return ok_count;
	1463	+ list_splice(&bo->cb_list, &block->flow_block.cb_list);
	1464	+
	1465	+ return 0;
	1466	+
	1467	+err_unroll:
	1468	+ list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
	1469	+ if (i-- > 0) {
	1470	+ list_del(&block_cb->list);
	1471	+ tcf_block_playback_offloads(block, block_cb->cb,
	1472	+ block_cb->cb_priv, false,
	1473	+ tcf_block_offload_in_use(block),
	1474	+ NULL);
	1475	+ if (!bo->unlocked_driver_cb)
	1476	+ block->lockeddevcnt--;
	1477	+ }
	1478	+ flow_block_cb_free(block_cb);
	1479	+ }
	1480	+
	1481	+ return err;
	1482	+}
	1483	+
	1484	+static void tcf_block_unbind(struct tcf_block *block,
	1485	+ struct flow_block_offload *bo)
	1486	+{
	1487	+ struct flow_block_cb block_cb, next;
	1488	+
	1489	+ lockdep_assert_held(&block->cb_lock);
	1490	+
	1491	+ list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
	1492	+ tcf_block_playback_offloads(block, block_cb->cb,
	1493	+ block_cb->cb_priv, false,
	1494	+ tcf_block_offload_in_use(block),
	1495	+ NULL);
	1496	+ list_del(&block_cb->list);
	1497	+ flow_block_cb_free(block_cb);
	1498	+ if (!bo->unlocked_driver_cb)
	1499	+ block->lockeddevcnt--;
	1500	+ }
	1501	+}
	1502	+
	1503	+static int tcf_block_setup(struct tcf_block *block,
	1504	+ struct flow_block_offload *bo)
	1505	+{
	1506	+ int err;
	1507	+
	1508	+ switch (bo->command) {
	1509	+ case FLOW_BLOCK_BIND:
	1510	+ err = tcf_block_bind(block, bo);
	1511	+ break;
	1512	+ case FLOW_BLOCK_UNBIND:
	1513	+ err = 0;
	1514	+ tcf_block_unbind(block, bo);
	1515	+ break;
	1516	+ default:
	1517	+ WARN_ON_ONCE(1);
	1518	+ err = -EOPNOTSUPP;
	1519	+ }
	1520	+
	1521	+ return err;
995	1522	}
996	1523
997	1524	/* Main classifier routine: scans classifier chain attached
998	1525	* to this qdisc, (optionally) tests for protocol and asks
999	1526	* specific classifiers.
1000	1527	*/
1001		-int tcf_classify(struct sk_buff skb, const struct tcf_proto tp,
1002		- struct tcf_result *res, bool compat_mode)
	1528	+static inline int __tcf_classify(struct sk_buff *skb,
	1529	+ const struct tcf_proto *tp,
	1530	+ const struct tcf_proto *orig_tp,
	1531	+ struct tcf_result *res,
	1532	+ bool compat_mode,
	1533	+ u32 *last_executed_chain)
1003	1534	{
1004	1535	#ifdef CONFIG_NET_CLS_ACT
1005		- const int max_reclassify_loop = 4;
1006		- const struct tcf_proto *orig_tp = tp;
	1536	+ const int max_reclassify_loop = 16;
1007	1537	const struct tcf_proto *first_tp;
1008	1538	int limit = 0;
1009	1539
..	..	@@ -1021,9 +1551,11 @@
1021	1551	#ifdef CONFIG_NET_CLS_ACT
1022	1552	if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1023	1553	first_tp = orig_tp;
	1554	+ *last_executed_chain = first_tp->chain->index;
1024	1555	goto reset;
1025	1556	} else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1026	1557	first_tp = res->goto_tp;
	1558	+ *last_executed_chain = err & TC_ACT_EXT_VAL_MASK;
1027	1559	goto reset;
1028	1560	}
1029	1561	#endif
..	..	@@ -1046,39 +1578,188 @@
1046	1578	goto reclassify;
1047	1579	#endif
1048	1580	}
	1581	+
	1582	+int tcf_classify(struct sk_buff skb, const struct tcf_proto tp,
	1583	+ struct tcf_result *res, bool compat_mode)
	1584	+{
	1585	+ u32 last_executed_chain = 0;
	1586	+
	1587	+ return __tcf_classify(skb, tp, tp, res, compat_mode,
	1588	+ &last_executed_chain);
	1589	+}
1049	1590	EXPORT_SYMBOL(tcf_classify);
	1591	+
	1592	+int tcf_classify_ingress(struct sk_buff *skb,
	1593	+ const struct tcf_block *ingress_block,
	1594	+ const struct tcf_proto *tp,
	1595	+ struct tcf_result *res, bool compat_mode)
	1596	+{
	1597	+#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
	1598	+ u32 last_executed_chain = 0;
	1599	+
	1600	+ return __tcf_classify(skb, tp, tp, res, compat_mode,
	1601	+ &last_executed_chain);
	1602	+#else
	1603	+ u32 last_executed_chain = tp ? tp->chain->index : 0;
	1604	+ const struct tcf_proto *orig_tp = tp;
	1605	+ struct tc_skb_ext *ext;
	1606	+ int ret;
	1607	+
	1608	+ ext = skb_ext_find(skb, TC_SKB_EXT);
	1609	+
	1610	+ if (ext && ext->chain) {
	1611	+ struct tcf_chain *fchain;
	1612	+
	1613	+ fchain = tcf_chain_lookup_rcu(ingress_block, ext->chain);
	1614	+ if (!fchain)
	1615	+ return TC_ACT_SHOT;
	1616	+
	1617	+ /* Consume, so cloned/redirect skbs won't inherit ext */
	1618	+ skb_ext_del(skb, TC_SKB_EXT);
	1619	+
	1620	+ tp = rcu_dereference_bh(fchain->filter_chain);
	1621	+ last_executed_chain = fchain->index;
	1622	+ }
	1623	+
	1624	+ ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode,
	1625	+ &last_executed_chain);
	1626	+
	1627	+ /* If we missed on some chain */
	1628	+ if (ret == TC_ACT_UNSPEC && last_executed_chain) {
	1629	+ ext = tc_skb_ext_alloc(skb);
	1630	+ if (WARN_ON_ONCE(!ext))
	1631	+ return TC_ACT_SHOT;
	1632	+ ext->chain = last_executed_chain;
	1633	+ ext->mru = qdisc_skb_cb(skb)->mru;
	1634	+ }
	1635	+
	1636	+ return ret;
	1637	+#endif
	1638	+}
	1639	+EXPORT_SYMBOL(tcf_classify_ingress);
1050	1640
1051	1641	struct tcf_chain_info {
1052	1642	struct tcf_proto __rcu **pprev;
1053	1643	struct tcf_proto __rcu *next;
1054	1644	};
1055	1645
1056		-static struct tcf_proto tcf_chain_tp_prev(struct tcf_chain_info chain_info)
	1646	+static struct tcf_proto tcf_chain_tp_prev(struct tcf_chain chain,
	1647	+ struct tcf_chain_info *chain_info)
1057	1648	{
1058		- return rtnl_dereference(*chain_info->pprev);
	1649	+ return tcf_chain_dereference(*chain_info->pprev, chain);
1059	1650	}
1060	1651
1061		-static void tcf_chain_tp_insert(struct tcf_chain *chain,
1062		- struct tcf_chain_info *chain_info,
1063		- struct tcf_proto *tp)
	1652	+static int tcf_chain_tp_insert(struct tcf_chain *chain,
	1653	+ struct tcf_chain_info *chain_info,
	1654	+ struct tcf_proto *tp)
1064	1655	{
	1656	+ if (chain->flushing)
	1657	+ return -EAGAIN;
	1658	+
	1659	+ RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1065	1660	if (*chain_info->pprev == chain->filter_chain)
1066	1661	tcf_chain0_head_change(chain, tp);
1067		- RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain_info));
	1662	+ tcf_proto_get(tp);
1068	1663	rcu_assign_pointer(*chain_info->pprev, tp);
1069		- tcf_chain_hold(chain);
	1664	+
	1665	+ return 0;
1070	1666	}
1071	1667
1072	1668	static void tcf_chain_tp_remove(struct tcf_chain *chain,
1073	1669	struct tcf_chain_info *chain_info,
1074	1670	struct tcf_proto *tp)
1075	1671	{
1076		- struct tcf_proto *next = rtnl_dereference(chain_info->next);
	1672	+ struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
1077	1673
	1674	+ tcf_proto_mark_delete(tp);
1078	1675	if (tp == chain->filter_chain)
1079	1676	tcf_chain0_head_change(chain, next);
1080	1677	RCU_INIT_POINTER(*chain_info->pprev, next);
1081		- tcf_chain_put(chain);
	1678	+}
	1679	+
	1680	+static struct tcf_proto tcf_chain_tp_find(struct tcf_chain chain,
	1681	+ struct tcf_chain_info *chain_info,
	1682	+ u32 protocol, u32 prio,
	1683	+ bool prio_allocate);
	1684	+
	1685	+/* Try to insert new proto.
	1686	+ * If proto with specified priority already exists, free new proto
	1687	+ * and return existing one.
	1688	+ */
	1689	+
	1690	+static struct tcf_proto tcf_chain_tp_insert_unique(struct tcf_chain chain,
	1691	+ struct tcf_proto *tp_new,
	1692	+ u32 protocol, u32 prio,
	1693	+ bool rtnl_held)
	1694	+{
	1695	+ struct tcf_chain_info chain_info;
	1696	+ struct tcf_proto *tp;
	1697	+ int err = 0;
	1698	+
	1699	+ mutex_lock(&chain->filter_chain_lock);
	1700	+
	1701	+ if (tcf_proto_exists_destroying(chain, tp_new)) {
	1702	+ mutex_unlock(&chain->filter_chain_lock);
	1703	+ tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
	1704	+ return ERR_PTR(-EAGAIN);
	1705	+ }
	1706	+
	1707	+ tp = tcf_chain_tp_find(chain, &chain_info,
	1708	+ protocol, prio, false);
	1709	+ if (!tp)
	1710	+ err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
	1711	+ mutex_unlock(&chain->filter_chain_lock);
	1712	+
	1713	+ if (tp) {
	1714	+ tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
	1715	+ tp_new = tp;
	1716	+ } else if (err) {
	1717	+ tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
	1718	+ tp_new = ERR_PTR(err);
	1719	+ }
	1720	+
	1721	+ return tp_new;
	1722	+}
	1723	+
	1724	+static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
	1725	+ struct tcf_proto *tp, bool rtnl_held,
	1726	+ struct netlink_ext_ack *extack)
	1727	+{
	1728	+ struct tcf_chain_info chain_info;
	1729	+ struct tcf_proto *tp_iter;
	1730	+ struct tcf_proto **pprev;
	1731	+ struct tcf_proto *next;
	1732	+
	1733	+ mutex_lock(&chain->filter_chain_lock);
	1734	+
	1735	+ /* Atomically find and remove tp from chain. */
	1736	+ for (pprev = &chain->filter_chain;
	1737	+ (tp_iter = tcf_chain_dereference(*pprev, chain));
	1738	+ pprev = &tp_iter->next) {
	1739	+ if (tp_iter == tp) {
	1740	+ chain_info.pprev = pprev;
	1741	+ chain_info.next = tp_iter->next;
	1742	+ WARN_ON(tp_iter->deleting);
	1743	+ break;
	1744	+ }
	1745	+ }
	1746	+ /* Verify that tp still exists and no new filters were inserted
	1747	+ * concurrently.
	1748	+ * Mark tp for deletion if it is empty.
	1749	+ */
	1750	+ if (!tp_iter \|\| !tcf_proto_check_delete(tp)) {
	1751	+ mutex_unlock(&chain->filter_chain_lock);
	1752	+ return;
	1753	+ }
	1754	+
	1755	+ tcf_proto_signal_destroying(chain, tp);
	1756	+ next = tcf_chain_dereference(chain_info.next, chain);
	1757	+ if (tp == chain->filter_chain)
	1758	+ tcf_chain0_head_change(chain, next);
	1759	+ RCU_INIT_POINTER(*chain_info.pprev, next);
	1760	+ mutex_unlock(&chain->filter_chain_lock);
	1761	+
	1762	+ tcf_proto_put(tp, rtnl_held, extack);
1082	1763	}
1083	1764
1084	1765	static struct tcf_proto tcf_chain_tp_find(struct tcf_chain chain,
..	..	@@ -1091,7 +1772,8 @@
1091	1772
1092	1773	/* Check the chain for existence of proto-tcf with this priority */
1093	1774	for (pprev = &chain->filter_chain;
1094		- (tp = rtnl_dereference(*pprev)); pprev = &tp->next) {
	1775	+ (tp = tcf_chain_dereference(*pprev, chain));
	1776	+ pprev = &tp->next) {
1095	1777	if (tp->prio >= prio) {
1096	1778	if (tp->prio == prio) {
1097	1779	if (prio_allocate \|\|
..	..	@@ -1104,14 +1786,20 @@
1104	1786	}
1105	1787	}
1106	1788	chain_info->pprev = pprev;
1107		- chain_info->next = tp ? tp->next : NULL;
	1789	+ if (tp) {
	1790	+ chain_info->next = tp->next;
	1791	+ tcf_proto_get(tp);
	1792	+ } else {
	1793	+ chain_info->next = NULL;
	1794	+ }
1108	1795	return tp;
1109	1796	}
1110	1797
1111	1798	static int tcf_fill_node(struct net net, struct sk_buff skb,
1112	1799	struct tcf_proto tp, struct tcf_block block,
1113	1800	struct Qdisc q, u32 parent, void fh,
1114		- u32 portid, u32 seq, u16 flags, int event)
	1801	+ u32 portid, u32 seq, u16 flags, int event,
	1802	+ bool terse_dump, bool rtnl_held)
1115	1803	{
1116	1804	struct tcmsg *tcm;
1117	1805	struct nlmsghdr *nlh;
..	..	@@ -1138,8 +1826,17 @@
1138	1826	goto nla_put_failure;
1139	1827	if (!fh) {
1140	1828	tcm->tcm_handle = 0;
	1829	+ } else if (terse_dump) {
	1830	+ if (tp->ops->terse_dump) {
	1831	+ if (tp->ops->terse_dump(net, tp, fh, skb, tcm,
	1832	+ rtnl_held) < 0)
	1833	+ goto nla_put_failure;
	1834	+ } else {
	1835	+ goto cls_op_not_supp;
	1836	+ }
1141	1837	} else {
1142		- if (tp->ops->dump && tp->ops->dump(net, tp, fh, skb, tcm) < 0)
	1838	+ if (tp->ops->dump &&
	1839	+ tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
1143	1840	goto nla_put_failure;
1144	1841	}
1145	1842	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
..	..	@@ -1147,6 +1844,7 @@
1147	1844
1148	1845	out_nlmsg_trim:
1149	1846	nla_put_failure:
	1847	+cls_op_not_supp:
1150	1848	nlmsg_trim(skb, b);
1151	1849	return -1;
1152	1850	}
..	..	@@ -1154,33 +1852,40 @@
1154	1852	static int tfilter_notify(struct net net, struct sk_buff oskb,
1155	1853	struct nlmsghdr n, struct tcf_proto tp,
1156	1854	struct tcf_block block, struct Qdisc q,
1157		- u32 parent, void *fh, int event, bool unicast)
	1855	+ u32 parent, void *fh, int event, bool unicast,
	1856	+ bool rtnl_held)
1158	1857	{
1159	1858	struct sk_buff *skb;
1160	1859	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
	1860	+ int err = 0;
1161	1861
1162	1862	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1163	1863	if (!skb)
1164	1864	return -ENOBUFS;
1165	1865
1166	1866	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1167		- n->nlmsg_seq, n->nlmsg_flags, event) <= 0) {
	1867	+ n->nlmsg_seq, n->nlmsg_flags, event,
	1868	+ false, rtnl_held) <= 0) {
1168	1869	kfree_skb(skb);
1169	1870	return -EINVAL;
1170	1871	}
1171	1872
1172	1873	if (unicast)
1173		- return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
	1874	+ err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
	1875	+ else
	1876	+ err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
	1877	+ n->nlmsg_flags & NLM_F_ECHO);
1174	1878
1175		- return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1176		- n->nlmsg_flags & NLM_F_ECHO);
	1879	+ if (err > 0)
	1880	+ err = 0;
	1881	+ return err;
1177	1882	}
1178	1883
1179	1884	static int tfilter_del_notify(struct net net, struct sk_buff oskb,
1180	1885	struct nlmsghdr n, struct tcf_proto tp,
1181	1886	struct tcf_block block, struct Qdisc q,
1182	1887	u32 parent, void fh, bool unicast, bool last,
1183		- struct netlink_ext_ack *extack)
	1888	+ bool rtnl_held, struct netlink_ext_ack *extack)
1184	1889	{
1185	1890	struct sk_buff *skb;
1186	1891	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
..	..	@@ -1191,39 +1896,50 @@
1191	1896	return -ENOBUFS;
1192	1897
1193	1898	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1194		- n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER) <= 0) {
	1899	+ n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER,
	1900	+ false, rtnl_held) <= 0) {
1195	1901	NL_SET_ERR_MSG(extack, "Failed to build del event notification");
1196	1902	kfree_skb(skb);
1197	1903	return -EINVAL;
1198	1904	}
1199	1905
1200		- err = tp->ops->delete(tp, fh, last, extack);
	1906	+ err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
1201	1907	if (err) {
1202	1908	kfree_skb(skb);
1203	1909	return err;
1204	1910	}
1205	1911
1206	1912	if (unicast)
1207		- return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1208		-
1209		- err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1210		- n->nlmsg_flags & NLM_F_ECHO);
	1913	+ err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
	1914	+ else
	1915	+ err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
	1916	+ n->nlmsg_flags & NLM_F_ECHO);
1211	1917	if (err < 0)
1212	1918	NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
	1919	+
	1920	+ if (err > 0)
	1921	+ err = 0;
1213	1922	return err;
1214	1923	}
1215	1924
1216	1925	static void tfilter_notify_chain(struct net net, struct sk_buff oskb,
1217	1926	struct tcf_block block, struct Qdisc q,
1218	1927	u32 parent, struct nlmsghdr *n,
1219		- struct tcf_chain *chain, int event)
	1928	+ struct tcf_chain *chain, int event,
	1929	+ bool rtnl_held)
1220	1930	{
1221	1931	struct tcf_proto *tp;
1222	1932
1223		- for (tp = rtnl_dereference(chain->filter_chain);
1224		- tp; tp = rtnl_dereference(tp->next))
	1933	+ for (tp = tcf_get_next_proto(chain, NULL, rtnl_held);
	1934	+ tp; tp = tcf_get_next_proto(chain, tp, rtnl_held))
1225	1935	tfilter_notify(net, oskb, n, tp, block,
1226		- q, parent, NULL, event, false);
	1936	+ q, parent, NULL, event, false, rtnl_held);
	1937	+}
	1938	+
	1939	+static void tfilter_put(struct tcf_proto tp, void fh)
	1940	+{
	1941	+ if (tp->ops->put && fh)
	1942	+ tp->ops->put(tp, fh);
1227	1943	}
1228	1944
1229	1945	static int tc_new_tfilter(struct sk_buff skb, struct nlmsghdr n,
..	..	@@ -1231,21 +1947,23 @@
1231	1947	{
1232	1948	struct net *net = sock_net(skb->sk);
1233	1949	struct nlattr *tca[TCA_MAX + 1];
	1950	+ char name[IFNAMSIZ];
1234	1951	struct tcmsg *t;
1235	1952	u32 protocol;
1236	1953	u32 prio;
1237	1954	bool prio_allocate;
1238	1955	u32 parent;
1239	1956	u32 chain_index;
1240		- struct Qdisc *q = NULL;
	1957	+ struct Qdisc *q;
1241	1958	struct tcf_chain_info chain_info;
1242		- struct tcf_chain *chain = NULL;
	1959	+ struct tcf_chain *chain;
1243	1960	struct tcf_block *block;
1244	1961	struct tcf_proto *tp;
1245	1962	unsigned long cl;
1246	1963	void *fh;
1247	1964	int err;
1248	1965	int tp_created;
	1966	+ bool rtnl_held = false;
1249	1967
1250	1968	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1251	1969	return -EPERM;
..	..	@@ -1253,7 +1971,8 @@
1253	1971	replay:
1254	1972	tp_created = 0;
1255	1973
1256		- err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
	1974	+ err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
	1975	+ rtm_tca_policy, extack);
1257	1976	if (err < 0)
1258	1977	return err;
1259	1978
..	..	@@ -1262,7 +1981,11 @@
1262	1981	prio = TC_H_MAJ(t->tcm_info);
1263	1982	prio_allocate = false;
1264	1983	parent = t->tcm_parent;
	1984	+ tp = NULL;
1265	1985	cl = 0;
	1986	+ block = NULL;
	1987	+ q = NULL;
	1988	+ chain = NULL;
1266	1989
1267	1990	if (prio == 0) {
1268	1991	/* If no priority is provided by the user,
..	..	@@ -1279,12 +2002,38 @@
1279	2002
1280	2003	/* Find head of filter chain. */
1281	2004
1282		- block = tcf_block_find(net, &q, &parent, &cl,
1283		- t->tcm_ifindex, t->tcm_block_index, extack);
	2005	+ err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
	2006	+ if (err)
	2007	+ return err;
	2008	+
	2009	+ if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
	2010	+ NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
	2011	+ err = -EINVAL;
	2012	+ goto errout;
	2013	+ }
	2014	+
	2015	+ /* Take rtnl mutex if rtnl_held was set to true on previous iteration,
	2016	+ * block is shared (no qdisc found), qdisc is not unlocked, classifier
	2017	+ * type is not specified, classifier is not unlocked.
	2018	+ */
	2019	+ if (rtnl_held \|\|
	2020	+ (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) \|\|
	2021	+ !tcf_proto_is_unlocked(name)) {
	2022	+ rtnl_held = true;
	2023	+ rtnl_lock();
	2024	+ }
	2025	+
	2026	+ err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
	2027	+ if (err)
	2028	+ goto errout;
	2029	+
	2030	+ block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
	2031	+ extack);
1284	2032	if (IS_ERR(block)) {
1285	2033	err = PTR_ERR(block);
1286	2034	goto errout;
1287	2035	}
	2036	+ block->classid = parent;
1288	2037
1289	2038	chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1290	2039	if (chain_index > TC_ACT_EXT_VAL_MASK) {
..	..	@@ -1299,40 +2048,61 @@
1299	2048	goto errout;
1300	2049	}
1301	2050
	2051	+ mutex_lock(&chain->filter_chain_lock);
1302	2052	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1303	2053	prio, prio_allocate);
1304	2054	if (IS_ERR(tp)) {
1305	2055	NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1306	2056	err = PTR_ERR(tp);
1307		- goto errout;
	2057	+ goto errout_locked;
1308	2058	}
1309	2059
1310	2060	if (tp == NULL) {
	2061	+ struct tcf_proto *tp_new = NULL;
	2062	+
	2063	+ if (chain->flushing) {
	2064	+ err = -EAGAIN;
	2065	+ goto errout_locked;
	2066	+ }
	2067	+
1311	2068	/* Proto-tcf does not exist, create new one */
1312	2069
1313	2070	if (tca[TCA_KIND] == NULL \|\| !protocol) {
1314	2071	NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
1315	2072	err = -EINVAL;
1316		- goto errout;
	2073	+ goto errout_locked;
1317	2074	}
1318	2075
1319	2076	if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1320	2077	NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1321	2078	err = -ENOENT;
1322		- goto errout;
	2079	+ goto errout_locked;
1323	2080	}
1324	2081
1325	2082	if (prio_allocate)
1326		- prio = tcf_auto_prio(tcf_chain_tp_prev(&chain_info));
	2083	+ prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
	2084	+ &chain_info));
1327	2085
1328		- tp = tcf_proto_create(nla_data(tca[TCA_KIND]),
1329		- protocol, prio, chain, extack);
	2086	+ mutex_unlock(&chain->filter_chain_lock);
	2087	+ tp_new = tcf_proto_create(name, protocol, prio, chain,
	2088	+ rtnl_held, extack);
	2089	+ if (IS_ERR(tp_new)) {
	2090	+ err = PTR_ERR(tp_new);
	2091	+ goto errout_tp;
	2092	+ }
	2093	+
	2094	+ tp_created = 1;
	2095	+ tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
	2096	+ rtnl_held);
1330	2097	if (IS_ERR(tp)) {
1331	2098	err = PTR_ERR(tp);
1332		- goto errout;
	2099	+ goto errout_tp;
1333	2100	}
1334		- tp_created = 1;
1335		- } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
	2101	+ } else {
	2102	+ mutex_unlock(&chain->filter_chain_lock);
	2103	+ }
	2104	+
	2105	+ if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1336	2106	NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1337	2107	err = -EINVAL;
1338	2108	goto errout;
..	..	@@ -1347,12 +2117,14 @@
1347	2117	goto errout;
1348	2118	}
1349	2119	} else if (n->nlmsg_flags & NLM_F_EXCL) {
	2120	+ tfilter_put(tp, fh);
1350	2121	NL_SET_ERR_MSG(extack, "Filter already exists");
1351	2122	err = -EEXIST;
1352	2123	goto errout;
1353	2124	}
1354	2125
1355	2126	if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
	2127	+ tfilter_put(tp, fh);
1356	2128	NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
1357	2129	err = -EINVAL;
1358	2130	goto errout;
..	..	@@ -1360,28 +2132,44 @@
1360	2132
1361	2133	err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
1362	2134	n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
1363		- extack);
	2135	+ rtnl_held, extack);
1364	2136	if (err == 0) {
1365		- if (tp_created)
1366		- tcf_chain_tp_insert(chain, &chain_info, tp);
1367	2137	tfilter_notify(net, skb, n, tp, block, q, parent, fh,
1368		- RTM_NEWTFILTER, false);
	2138	+ RTM_NEWTFILTER, false, rtnl_held);
	2139	+ tfilter_put(tp, fh);
1369	2140	/* q pointer is NULL for shared blocks */
1370	2141	if (q)
1371	2142	q->flags &= ~TCQ_F_CAN_BYPASS;
1372		- } else {
1373		- if (tp_created)
1374		- tcf_proto_destroy(tp, NULL);
1375	2143	}
1376	2144
1377	2145	errout:
1378		- if (chain)
1379		- tcf_chain_put(chain);
1380		- tcf_block_release(q, block);
1381		- if (err == -EAGAIN)
	2146	+ if (err && tp_created)
	2147	+ tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
	2148	+errout_tp:
	2149	+ if (chain) {
	2150	+ if (tp && !IS_ERR(tp))
	2151	+ tcf_proto_put(tp, rtnl_held, NULL);
	2152	+ if (!tp_created)
	2153	+ tcf_chain_put(chain);
	2154	+ }
	2155	+ tcf_block_release(q, block, rtnl_held);
	2156	+
	2157	+ if (rtnl_held)
	2158	+ rtnl_unlock();
	2159	+
	2160	+ if (err == -EAGAIN) {
	2161	+ /* Take rtnl lock in case EAGAIN is caused by concurrent flush
	2162	+ * of target chain.
	2163	+ */
	2164	+ rtnl_held = true;
1382	2165	/* Replay the request. */
1383	2166	goto replay;
	2167	+ }
1384	2168	return err;
	2169	+
	2170	+errout_locked:
	2171	+ mutex_unlock(&chain->filter_chain_lock);
	2172	+ goto errout;
1385	2173	}
1386	2174
1387	2175	static int tc_del_tfilter(struct sk_buff skb, struct nlmsghdr n,
..	..	@@ -1389,6 +2177,7 @@
1389	2177	{
1390	2178	struct net *net = sock_net(skb->sk);
1391	2179	struct nlattr *tca[TCA_MAX + 1];
	2180	+ char name[IFNAMSIZ];
1392	2181	struct tcmsg *t;
1393	2182	u32 protocol;
1394	2183	u32 prio;
..	..	@@ -1397,16 +2186,18 @@
1397	2186	struct Qdisc *q = NULL;
1398	2187	struct tcf_chain_info chain_info;
1399	2188	struct tcf_chain *chain = NULL;
1400		- struct tcf_block *block;
	2189	+ struct tcf_block *block = NULL;
1401	2190	struct tcf_proto *tp = NULL;
1402	2191	unsigned long cl = 0;
1403	2192	void *fh = NULL;
1404	2193	int err;
	2194	+ bool rtnl_held = false;
1405	2195
1406	2196	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1407	2197	return -EPERM;
1408	2198
1409		- err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
	2199	+ err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
	2200	+ rtm_tca_policy, extack);
1410	2201	if (err < 0)
1411	2202	return err;
1412	2203
..	..	@@ -1422,8 +2213,32 @@
1422	2213
1423	2214	/* Find head of filter chain. */
1424	2215
1425		- block = tcf_block_find(net, &q, &parent, &cl,
1426		- t->tcm_ifindex, t->tcm_block_index, extack);
	2216	+ err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
	2217	+ if (err)
	2218	+ return err;
	2219	+
	2220	+ if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
	2221	+ NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
	2222	+ err = -EINVAL;
	2223	+ goto errout;
	2224	+ }
	2225	+ /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
	2226	+ * found), qdisc is not unlocked, classifier type is not specified,
	2227	+ * classifier is not unlocked.
	2228	+ */
	2229	+ if (!prio \|\|
	2230	+ (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) \|\|
	2231	+ !tcf_proto_is_unlocked(name)) {
	2232	+ rtnl_held = true;
	2233	+ rtnl_lock();
	2234	+ }
	2235	+
	2236	+ err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
	2237	+ if (err)
	2238	+ goto errout;
	2239	+
	2240	+ block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
	2241	+ extack);
1427	2242	if (IS_ERR(block)) {
1428	2243	err = PTR_ERR(block);
1429	2244	goto errout;
..	..	@@ -1451,56 +2266,70 @@
1451	2266
1452	2267	if (prio == 0) {
1453	2268	tfilter_notify_chain(net, skb, block, q, parent, n,
1454		- chain, RTM_DELTFILTER);
1455		- tcf_chain_flush(chain);
	2269	+ chain, RTM_DELTFILTER, rtnl_held);
	2270	+ tcf_chain_flush(chain, rtnl_held);
1456	2271	err = 0;
1457	2272	goto errout;
1458	2273	}
1459	2274
	2275	+ mutex_lock(&chain->filter_chain_lock);
1460	2276	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1461	2277	prio, false);
1462	2278	if (!tp \|\| IS_ERR(tp)) {
1463	2279	NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1464	2280	err = tp ? PTR_ERR(tp) : -ENOENT;
1465		- goto errout;
	2281	+ goto errout_locked;
1466	2282	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1467	2283	NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1468	2284	err = -EINVAL;
	2285	+ goto errout_locked;
	2286	+ } else if (t->tcm_handle == 0) {
	2287	+ tcf_proto_signal_destroying(chain, tp);
	2288	+ tcf_chain_tp_remove(chain, &chain_info, tp);
	2289	+ mutex_unlock(&chain->filter_chain_lock);
	2290	+
	2291	+ tcf_proto_put(tp, rtnl_held, NULL);
	2292	+ tfilter_notify(net, skb, n, tp, block, q, parent, fh,
	2293	+ RTM_DELTFILTER, false, rtnl_held);
	2294	+ err = 0;
1469	2295	goto errout;
1470	2296	}
	2297	+ mutex_unlock(&chain->filter_chain_lock);
1471	2298
1472	2299	fh = tp->ops->get(tp, t->tcm_handle);
1473	2300
1474	2301	if (!fh) {
1475		- if (t->tcm_handle == 0) {
1476		- tcf_chain_tp_remove(chain, &chain_info, tp);
1477		- tfilter_notify(net, skb, n, tp, block, q, parent, fh,
1478		- RTM_DELTFILTER, false);
1479		- tcf_proto_destroy(tp, extack);
1480		- err = 0;
1481		- } else {
1482		- NL_SET_ERR_MSG(extack, "Specified filter handle not found");
1483		- err = -ENOENT;
1484		- }
	2302	+ NL_SET_ERR_MSG(extack, "Specified filter handle not found");
	2303	+ err = -ENOENT;
1485	2304	} else {
1486	2305	bool last;
1487	2306
1488	2307	err = tfilter_del_notify(net, skb, n, tp, block,
1489	2308	q, parent, fh, false, &last,
1490		- extack);
	2309	+ rtnl_held, extack);
	2310	+
1491	2311	if (err)
1492	2312	goto errout;
1493		- if (last) {
1494		- tcf_chain_tp_remove(chain, &chain_info, tp);
1495		- tcf_proto_destroy(tp, extack);
1496		- }
	2313	+ if (last)
	2314	+ tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
1497	2315	}
1498	2316
1499	2317	errout:
1500		- if (chain)
	2318	+ if (chain) {
	2319	+ if (tp && !IS_ERR(tp))
	2320	+ tcf_proto_put(tp, rtnl_held, NULL);
1501	2321	tcf_chain_put(chain);
1502		- tcf_block_release(q, block);
	2322	+ }
	2323	+ tcf_block_release(q, block, rtnl_held);
	2324	+
	2325	+ if (rtnl_held)
	2326	+ rtnl_unlock();
	2327	+
1503	2328	return err;
	2329	+
	2330	+errout_locked:
	2331	+ mutex_unlock(&chain->filter_chain_lock);
	2332	+ goto errout;
1504	2333	}
1505	2334
1506	2335	static int tc_get_tfilter(struct sk_buff skb, struct nlmsghdr n,
..	..	@@ -1508,6 +2337,7 @@
1508	2337	{
1509	2338	struct net *net = sock_net(skb->sk);
1510	2339	struct nlattr *tca[TCA_MAX + 1];
	2340	+ char name[IFNAMSIZ];
1511	2341	struct tcmsg *t;
1512	2342	u32 protocol;
1513	2343	u32 prio;
..	..	@@ -1516,13 +2346,15 @@
1516	2346	struct Qdisc *q = NULL;
1517	2347	struct tcf_chain_info chain_info;
1518	2348	struct tcf_chain *chain = NULL;
1519		- struct tcf_block *block;
	2349	+ struct tcf_block *block = NULL;
1520	2350	struct tcf_proto *tp = NULL;
1521	2351	unsigned long cl = 0;
1522	2352	void *fh = NULL;
1523	2353	int err;
	2354	+ bool rtnl_held = false;
1524	2355
1525		- err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
	2356	+ err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
	2357	+ rtm_tca_policy, extack);
1526	2358	if (err < 0)
1527	2359	return err;
1528	2360
..	..	@@ -1538,8 +2370,31 @@
1538	2370
1539	2371	/* Find head of filter chain. */
1540	2372
1541		- block = tcf_block_find(net, &q, &parent, &cl,
1542		- t->tcm_ifindex, t->tcm_block_index, extack);
	2373	+ err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
	2374	+ if (err)
	2375	+ return err;
	2376	+
	2377	+ if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
	2378	+ NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
	2379	+ err = -EINVAL;
	2380	+ goto errout;
	2381	+ }
	2382	+ /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
	2383	+ * unlocked, classifier type is not specified, classifier is not
	2384	+ * unlocked.
	2385	+ */
	2386	+ if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) \|\|
	2387	+ !tcf_proto_is_unlocked(name)) {
	2388	+ rtnl_held = true;
	2389	+ rtnl_lock();
	2390	+ }
	2391	+
	2392	+ err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
	2393	+ if (err)
	2394	+ goto errout;
	2395	+
	2396	+ block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
	2397	+ extack);
1543	2398	if (IS_ERR(block)) {
1544	2399	err = PTR_ERR(block);
1545	2400	goto errout;
..	..	@@ -1558,8 +2413,10 @@
1558	2413	goto errout;
1559	2414	}
1560	2415
	2416	+ mutex_lock(&chain->filter_chain_lock);
1561	2417	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1562	2418	prio, false);
	2419	+ mutex_unlock(&chain->filter_chain_lock);
1563	2420	if (!tp \|\| IS_ERR(tp)) {
1564	2421	NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1565	2422	err = tp ? PTR_ERR(tp) : -ENOENT;
..	..	@@ -1577,15 +2434,23 @@
1577	2434	err = -ENOENT;
1578	2435	} else {
1579	2436	err = tfilter_notify(net, skb, n, tp, block, q, parent,
1580		- fh, RTM_NEWTFILTER, true);
	2437	+ fh, RTM_NEWTFILTER, true, rtnl_held);
1581	2438	if (err < 0)
1582	2439	NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
1583	2440	}
1584	2441
	2442	+ tfilter_put(tp, fh);
1585	2443	errout:
1586		- if (chain)
	2444	+ if (chain) {
	2445	+ if (tp && !IS_ERR(tp))
	2446	+ tcf_proto_put(tp, rtnl_held, NULL);
1587	2447	tcf_chain_put(chain);
1588		- tcf_block_release(q, block);
	2448	+ }
	2449	+ tcf_block_release(q, block, rtnl_held);
	2450	+
	2451	+ if (rtnl_held)
	2452	+ rtnl_unlock();
	2453	+
1589	2454	return err;
1590	2455	}
1591	2456
..	..	@@ -1596,6 +2461,7 @@
1596	2461	struct tcf_block *block;
1597	2462	struct Qdisc *q;
1598	2463	u32 parent;
	2464	+ bool terse_dump;
1599	2465	};
1600	2466
1601	2467	static int tcf_node_dump(struct tcf_proto tp, void n, struct tcf_walker *arg)
..	..	@@ -1606,21 +2472,25 @@
1606	2472	return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
1607	2473	n, NETLINK_CB(a->cb->skb).portid,
1608	2474	a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
1609		- RTM_NEWTFILTER);
	2475	+ RTM_NEWTFILTER, a->terse_dump, true);
1610	2476	}
1611	2477
1612	2478	static bool tcf_chain_dump(struct tcf_chain chain, struct Qdisc q, u32 parent,
1613	2479	struct sk_buff skb, struct netlink_callback cb,
1614		- long index_start, long *p_index)
	2480	+ long index_start, long *p_index, bool terse)
1615	2481	{
1616	2482	struct net *net = sock_net(skb->sk);
1617	2483	struct tcf_block *block = chain->block;
1618	2484	struct tcmsg *tcm = nlmsg_data(cb->nlh);
	2485	+ struct tcf_proto tp, tp_prev;
1619	2486	struct tcf_dump_args arg;
1620		- struct tcf_proto *tp;
1621	2487
1622		- for (tp = rtnl_dereference(chain->filter_chain);
1623		- tp; tp = rtnl_dereference(tp->next), (*p_index)++) {
	2488	+ for (tp = __tcf_get_next_proto(chain, NULL);
	2489	+ tp;
	2490	+ tp_prev = tp,
	2491	+ tp = __tcf_get_next_proto(chain, tp),
	2492	+ tcf_proto_put(tp_prev, true, NULL),
	2493	+ (*p_index)++) {
1624	2494	if (*p_index < index_start)
1625	2495	continue;
1626	2496	if (TC_H_MAJ(tcm->tcm_info) &&
..	..	@@ -1636,9 +2506,8 @@
1636	2506	if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
1637	2507	NETLINK_CB(cb->skb).portid,
1638	2508	cb->nlh->nlmsg_seq, NLM_F_MULTI,
1639		- RTM_NEWTFILTER) <= 0)
1640		- return false;
1641		-
	2509	+ RTM_NEWTFILTER, false, true) <= 0)
	2510	+ goto errout;
1642	2511	cb->args[1] = 1;
1643	2512	}
1644	2513	if (!tp->ops->walk)
..	..	@@ -1653,24 +2522,34 @@
1653	2522	arg.w.skip = cb->args[1] - 1;
1654	2523	arg.w.count = 0;
1655	2524	arg.w.cookie = cb->args[2];
1656		- tp->ops->walk(tp, &arg.w);
	2525	+ arg.terse_dump = terse;
	2526	+ tp->ops->walk(tp, &arg.w, true);
1657	2527	cb->args[2] = arg.w.cookie;
1658	2528	cb->args[1] = arg.w.count + 1;
1659	2529	if (arg.w.stop)
1660		- return false;
	2530	+ goto errout;
1661	2531	}
1662	2532	return true;
	2533	+
	2534	+errout:
	2535	+ tcf_proto_put(tp, true, NULL);
	2536	+ return false;
1663	2537	}
	2538	+
	2539	+static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = {
	2540	+ [TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE),
	2541	+};
1664	2542
1665	2543	/* called with RTNL */
1666	2544	static int tc_dump_tfilter(struct sk_buff skb, struct netlink_callback cb)
1667	2545	{
	2546	+ struct tcf_chain chain, chain_prev;
1668	2547	struct net *net = sock_net(skb->sk);
1669	2548	struct nlattr *tca[TCA_MAX + 1];
1670	2549	struct Qdisc *q = NULL;
1671	2550	struct tcf_block *block;
1672		- struct tcf_chain *chain;
1673	2551	struct tcmsg *tcm = nlmsg_data(cb->nlh);
	2552	+ bool terse_dump = false;
1674	2553	long index_start;
1675	2554	long index;
1676	2555	u32 parent;
..	..	@@ -1679,12 +2558,20 @@
1679	2558	if (nlmsg_len(cb->nlh) < sizeof(*tcm))
1680	2559	return skb->len;
1681	2560
1682		- err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, NULL, NULL);
	2561	+ err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
	2562	+ tcf_tfilter_dump_policy, cb->extack);
1683	2563	if (err)
1684	2564	return err;
1685	2565
	2566	+ if (tca[TCA_DUMP_FLAGS]) {
	2567	+ struct nla_bitfield32 flags =
	2568	+ nla_get_bitfield32(tca[TCA_DUMP_FLAGS]);
	2569	+
	2570	+ terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE;
	2571	+ }
	2572	+
1686	2573	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1687		- block = tcf_block_lookup(net, tcm->tcm_block_index);
	2574	+ block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
1688	2575	if (!block)
1689	2576	goto out;
1690	2577	/* If we work with block index, q is NULL and parent value
..	..	@@ -1704,12 +2591,10 @@
1704	2591	return skb->len;
1705	2592
1706	2593	parent = tcm->tcm_parent;
1707		- if (!parent) {
1708		- q = dev->qdisc;
1709		- parent = q->handle;
1710		- } else {
	2594	+ if (!parent)
	2595	+ q = rtnl_dereference(dev->qdisc);
	2596	+ else
1711	2597	q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
1712		- }
1713	2598	if (!q)
1714	2599	goto out;
1715	2600	cops = q->ops->cl_ops;
..	..	@@ -1725,6 +2610,7 @@
1725	2610	block = cops->tcf_block(q, cl, NULL);
1726	2611	if (!block)
1727	2612	goto out;
	2613	+ parent = block->classid;
1728	2614	if (tcf_block_shared(block))
1729	2615	q = NULL;
1730	2616	}
..	..	@@ -1732,17 +2618,24 @@
1732	2618	index_start = cb->args[0];
1733	2619	index = 0;
1734	2620
1735		- list_for_each_entry(chain, &block->chain_list, list) {
	2621	+ for (chain = __tcf_get_next_chain(block, NULL);
	2622	+ chain;
	2623	+ chain_prev = chain,
	2624	+ chain = __tcf_get_next_chain(block, chain),
	2625	+ tcf_chain_put(chain_prev)) {
1736	2626	if (tca[TCA_CHAIN] &&
1737	2627	nla_get_u32(tca[TCA_CHAIN]) != chain->index)
1738	2628	continue;
1739	2629	if (!tcf_chain_dump(chain, q, parent, skb, cb,
1740		- index_start, &index)) {
	2630	+ index_start, &index, terse_dump)) {
	2631	+ tcf_chain_put(chain);
1741	2632	err = -EMSGSIZE;
1742	2633	break;
1743	2634	}
1744	2635	}
1745	2636
	2637	+ if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
	2638	+ tcf_block_refcnt_put(block, true);
1746	2639	cb->args[0] = index;
1747	2640
1748	2641	out:
..	..	@@ -1752,8 +2645,10 @@
1752	2645	return skb->len;
1753	2646	}
1754	2647
1755		-static int tc_chain_fill_node(struct tcf_chain chain, struct net net,
1756		- struct sk_buff skb, struct tcf_block block,
	2648	+static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
	2649	+ void *tmplt_priv, u32 chain_index,
	2650	+ struct net net, struct sk_buff skb,
	2651	+ struct tcf_block *block,
1757	2652	u32 portid, u32 seq, u16 flags, int event)
1758	2653	{
1759	2654	unsigned char *b = skb_tail_pointer(skb);
..	..	@@ -1762,8 +2657,8 @@
1762	2657	struct tcmsg *tcm;
1763	2658	void *priv;
1764	2659
1765		- ops = chain->tmplt_ops;
1766		- priv = chain->tmplt_priv;
	2660	+ ops = tmplt_ops;
	2661	+ priv = tmplt_priv;
1767	2662
1768	2663	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1769	2664	if (!nlh)
..	..	@@ -1781,7 +2676,7 @@
1781	2676	tcm->tcm_block_index = block->index;
1782	2677	}
1783	2678
1784		- if (nla_put_u32(skb, TCA_CHAIN, chain->index))
	2679	+ if (nla_put_u32(skb, TCA_CHAIN, chain_index))
1785	2680	goto nla_put_failure;
1786	2681
1787	2682	if (ops) {
..	..	@@ -1807,13 +2702,45 @@
1807	2702	struct tcf_block *block = chain->block;
1808	2703	struct net *net = block->net;
1809	2704	struct sk_buff *skb;
	2705	+ int err = 0;
1810	2706
1811	2707	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1812	2708	if (!skb)
1813	2709	return -ENOBUFS;
1814	2710
1815		- if (tc_chain_fill_node(chain, net, skb, block, portid,
	2711	+ if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
	2712	+ chain->index, net, skb, block, portid,
1816	2713	seq, flags, event) <= 0) {
	2714	+ kfree_skb(skb);
	2715	+ return -EINVAL;
	2716	+ }
	2717	+
	2718	+ if (unicast)
	2719	+ err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
	2720	+ else
	2721	+ err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
	2722	+ flags & NLM_F_ECHO);
	2723	+
	2724	+ if (err > 0)
	2725	+ err = 0;
	2726	+ return err;
	2727	+}
	2728	+
	2729	+static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
	2730	+ void *tmplt_priv, u32 chain_index,
	2731	+ struct tcf_block block, struct sk_buff oskb,
	2732	+ u32 seq, u16 flags, bool unicast)
	2733	+{
	2734	+ u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
	2735	+ struct net *net = block->net;
	2736	+ struct sk_buff *skb;
	2737	+
	2738	+ skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
	2739	+ if (!skb)
	2740	+ return -ENOBUFS;
	2741	+
	2742	+ if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
	2743	+ block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
1817	2744	kfree_skb(skb);
1818	2745	return -EINVAL;
1819	2746	}
..	..	@@ -1829,13 +2756,19 @@
1829	2756	struct netlink_ext_ack *extack)
1830	2757	{
1831	2758	const struct tcf_proto_ops *ops;
	2759	+ char name[IFNAMSIZ];
1832	2760	void *tmplt_priv;
1833	2761
1834	2762	/* If kind is not set, user did not specify template. */
1835	2763	if (!tca[TCA_KIND])
1836	2764	return 0;
1837	2765
1838		- ops = tcf_proto_lookup_ops(nla_data(tca[TCA_KIND]), extack);
	2766	+ if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
	2767	+ NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
	2768	+ return -EINVAL;
	2769	+ }
	2770	+
	2771	+ ops = tcf_proto_lookup_ops(name, true, extack);
1839	2772	if (IS_ERR(ops))
1840	2773	return PTR_ERR(ops);
1841	2774	if (!ops->tmplt_create \|\| !ops->tmplt_destroy \|\| !ops->tmplt_dump) {
..	..	@@ -1853,16 +2786,15 @@
1853	2786	return 0;
1854	2787	}
1855	2788
1856		-static void tc_chain_tmplt_del(struct tcf_chain *chain)
	2789	+static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
	2790	+ void *tmplt_priv)
1857	2791	{
1858		- const struct tcf_proto_ops *ops = chain->tmplt_ops;
1859		-
1860	2792	/* If template ops are set, no work to do for us. */
1861		- if (!ops)
	2793	+ if (!tmplt_ops)
1862	2794	return;
1863	2795
1864		- ops->tmplt_destroy(chain->tmplt_priv);
1865		- module_put(ops->owner);
	2796	+ tmplt_ops->tmplt_destroy(tmplt_priv);
	2797	+ module_put(tmplt_ops->owner);
1866	2798	}
1867	2799
1868	2800	/* Add/delete/get a chain */
..	..	@@ -1875,8 +2807,8 @@
1875	2807	struct tcmsg *t;
1876	2808	u32 parent;
1877	2809	u32 chain_index;
1878		- struct Qdisc *q = NULL;
1879		- struct tcf_chain *chain = NULL;
	2810	+ struct Qdisc *q;
	2811	+ struct tcf_chain *chain;
1880	2812	struct tcf_block *block;
1881	2813	unsigned long cl;
1882	2814	int err;
..	..	@@ -1886,7 +2818,9 @@
1886	2818	return -EPERM;
1887	2819
1888	2820	replay:
1889		- err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
	2821	+ q = NULL;
	2822	+ err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
	2823	+ rtm_tca_policy, extack);
1890	2824	if (err < 0)
1891	2825	return err;
1892	2826
..	..	@@ -1905,6 +2839,8 @@
1905	2839	err = -EINVAL;
1906	2840	goto errout_block;
1907	2841	}
	2842	+
	2843	+ mutex_lock(&block->lock);
1908	2844	chain = tcf_chain_lookup(block, chain_index);
1909	2845	if (n->nlmsg_type == RTM_NEWCHAIN) {
1910	2846	if (chain) {
..	..	@@ -1916,54 +2852,61 @@
1916	2852	} else {
1917	2853	NL_SET_ERR_MSG(extack, "Filter chain already exists");
1918	2854	err = -EEXIST;
1919		- goto errout_block;
	2855	+ goto errout_block_locked;
1920	2856	}
1921	2857	} else {
1922	2858	if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1923	2859	NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
1924	2860	err = -ENOENT;
1925		- goto errout_block;
	2861	+ goto errout_block_locked;
1926	2862	}
1927	2863	chain = tcf_chain_create(block, chain_index);
1928	2864	if (!chain) {
1929	2865	NL_SET_ERR_MSG(extack, "Failed to create filter chain");
1930	2866	err = -ENOMEM;
1931		- goto errout_block;
	2867	+ goto errout_block_locked;
1932	2868	}
1933	2869	}
1934	2870	} else {
1935	2871	if (!chain \|\| tcf_chain_held_by_acts_only(chain)) {
1936	2872	NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
1937	2873	err = -EINVAL;
1938		- goto errout_block;
	2874	+ goto errout_block_locked;
1939	2875	}
1940	2876	tcf_chain_hold(chain);
1941	2877	}
1942	2878
1943		- switch (n->nlmsg_type) {
1944		- case RTM_NEWCHAIN:
1945		- err = tc_chain_tmplt_add(chain, net, tca, extack);
1946		- if (err)
1947		- goto errout;
1948		- /* In case the chain was successfully added, take a reference
1949		- * to the chain. This ensures that an empty chain
1950		- * does not disappear at the end of this function.
	2879	+ if (n->nlmsg_type == RTM_NEWCHAIN) {
	2880	+ /* Modifying chain requires holding parent block lock. In case
	2881	+ * the chain was successfully added, take a reference to the
	2882	+ * chain. This ensures that an empty chain does not disappear at
	2883	+ * the end of this function.
1951	2884	*/
1952	2885	tcf_chain_hold(chain);
1953	2886	chain->explicitly_created = true;
	2887	+ }
	2888	+ mutex_unlock(&block->lock);
	2889	+
	2890	+ switch (n->nlmsg_type) {
	2891	+ case RTM_NEWCHAIN:
	2892	+ err = tc_chain_tmplt_add(chain, net, tca, extack);
	2893	+ if (err) {
	2894	+ tcf_chain_put_explicitly_created(chain);
	2895	+ goto errout;
	2896	+ }
	2897	+
1954	2898	tc_chain_notify(chain, NULL, 0, NLM_F_CREATE \| NLM_F_EXCL,
1955	2899	RTM_NEWCHAIN, false);
1956	2900	break;
1957	2901	case RTM_DELCHAIN:
1958	2902	tfilter_notify_chain(net, skb, block, q, parent, n,
1959		- chain, RTM_DELTFILTER);
	2903	+ chain, RTM_DELTFILTER, true);
1960	2904	/* Flush the chain first as the user requested chain removal. */
1961		- tcf_chain_flush(chain);
	2905	+ tcf_chain_flush(chain, true);
1962	2906	/* In case the chain was successfully deleted, put a reference
1963	2907	* to the chain previously taken during addition.
1964	2908	*/
1965	2909	tcf_chain_put_explicitly_created(chain);
1966		- chain->explicitly_created = false;
1967	2910	break;
1968	2911	case RTM_GETCHAIN:
1969	2912	err = tc_chain_notify(chain, skb, n->nlmsg_seq,
..	..	@@ -1980,11 +2923,15 @@
1980	2923	errout:
1981	2924	tcf_chain_put(chain);
1982	2925	errout_block:
1983		- tcf_block_release(q, block);
	2926	+ tcf_block_release(q, block, true);
1984	2927	if (err == -EAGAIN)
1985	2928	/* Replay the request. */
1986	2929	goto replay;
1987	2930	return err;
	2931	+
	2932	+errout_block_locked:
	2933	+ mutex_unlock(&block->lock);
	2934	+ goto errout_block;
1988	2935	}
1989	2936
1990	2937	/* called with RTNL */
..	..	@@ -1994,8 +2941,8 @@
1994	2941	struct nlattr *tca[TCA_MAX + 1];
1995	2942	struct Qdisc *q = NULL;
1996	2943	struct tcf_block *block;
1997		- struct tcf_chain *chain;
1998	2944	struct tcmsg *tcm = nlmsg_data(cb->nlh);
	2945	+ struct tcf_chain *chain;
1999	2946	long index_start;
2000	2947	long index;
2001	2948	u32 parent;
..	..	@@ -2004,13 +2951,13 @@
2004	2951	if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2005	2952	return skb->len;
2006	2953
2007		- err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy,
2008		- NULL);
	2954	+ err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
	2955	+ rtm_tca_policy, cb->extack);
2009	2956	if (err)
2010	2957	return err;
2011	2958
2012	2959	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2013		- block = tcf_block_lookup(net, tcm->tcm_block_index);
	2960	+ block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2014	2961	if (!block)
2015	2962	goto out;
2016	2963	/* If we work with block index, q is NULL and parent value
..	..	@@ -2031,7 +2978,7 @@
2031	2978
2032	2979	parent = tcm->tcm_parent;
2033	2980	if (!parent) {
2034		- q = dev->qdisc;
	2981	+ q = rtnl_dereference(dev->qdisc);
2035	2982	parent = q->handle;
2036	2983	} else {
2037	2984	q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
..	..	@@ -2058,6 +3005,7 @@
2058	3005	index_start = cb->args[0];
2059	3006	index = 0;
2060	3007
	3008	+ mutex_lock(&block->lock);
2061	3009	list_for_each_entry(chain, &block->chain_list, list) {
2062	3010	if ((tca[TCA_CHAIN] &&
2063	3011	nla_get_u32(tca[TCA_CHAIN]) != chain->index))
..	..	@@ -2068,7 +3016,8 @@
2068	3016	}
2069	3017	if (tcf_chain_held_by_acts_only(chain))
2070	3018	continue;
2071		- err = tc_chain_fill_node(chain, net, skb, block,
	3019	+ err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
	3020	+ chain->index, net, skb, block,
2072	3021	NETLINK_CB(cb->skb).portid,
2073	3022	cb->nlh->nlmsg_seq, NLM_F_MULTI,
2074	3023	RTM_NEWCHAIN);
..	..	@@ -2076,7 +3025,10 @@
2076	3025	break;
2077	3026	index++;
2078	3027	}
	3028	+ mutex_unlock(&block->lock);
2079	3029
	3030	+ if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
	3031	+ tcf_block_refcnt_put(block, true);
2080	3032	cb->args[0] = index;
2081	3033
2082	3034	out:
..	..	@@ -2100,35 +3052,43 @@
2100	3052
2101	3053	int tcf_exts_validate(struct net net, struct tcf_proto tp, struct nlattr **tb,
2102	3054	struct nlattr rate_tlv, struct tcf_exts exts, bool ovr,
2103		- struct netlink_ext_ack *extack)
	3055	+ bool rtnl_held, struct netlink_ext_ack *extack)
2104	3056	{
2105	3057	#ifdef CONFIG_NET_CLS_ACT
2106	3058	{
	3059	+ int init_res[TCA_ACT_MAX_PRIO] = {};
2107	3060	struct tc_action *act;
2108	3061	size_t attr_size = 0;
2109	3062
2110	3063	if (exts->police && tb[exts->police]) {
	3064	+ struct tc_action_ops *a_o;
	3065	+
	3066	+ a_o = tc_action_load_ops("police", tb[exts->police], rtnl_held, extack);
	3067	+ if (IS_ERR(a_o))
	3068	+ return PTR_ERR(a_o);
2111	3069	act = tcf_action_init_1(net, tp, tb[exts->police],
2112	3070	rate_tlv, "police", ovr,
2113		- TCA_ACT_BIND, true, extack);
	3071	+ TCA_ACT_BIND, a_o, init_res,
	3072	+ rtnl_held, extack);
	3073	+ module_put(a_o->owner);
2114	3074	if (IS_ERR(act))
2115	3075	return PTR_ERR(act);
2116	3076
2117	3077	act->type = exts->type = TCA_OLD_COMPAT;
2118	3078	exts->actions[0] = act;
2119	3079	exts->nr_actions = 1;
	3080	+ tcf_idr_insert_many(exts->actions);
2120	3081	} else if (exts->action && tb[exts->action]) {
2121	3082	int err;
2122	3083
2123	3084	err = tcf_action_init(net, tp, tb[exts->action],
2124	3085	rate_tlv, NULL, ovr, TCA_ACT_BIND,
2125		- exts->actions, &attr_size, true,
2126		- extack);
	3086	+ exts->actions, init_res,
	3087	+ &attr_size, rtnl_held, extack);
2127	3088	if (err < 0)
2128	3089	return err;
2129	3090	exts->nr_actions = err;
2130	3091	}
2131		- exts->net = net;
2132	3092	}
2133	3093	#else
2134	3094	if ((exts->action && tb[exts->action]) \|\|
..	..	@@ -2175,16 +3135,17 @@
2175	3135	* tc data even if iproute2 was newer - jhs
2176	3136	*/
2177	3137	if (exts->type != TCA_OLD_COMPAT) {
2178		- nest = nla_nest_start(skb, exts->action);
	3138	+ nest = nla_nest_start_noflag(skb, exts->action);
2179	3139	if (nest == NULL)
2180	3140	goto nla_put_failure;
2181	3141
2182		- if (tcf_action_dump(skb, exts->actions, 0, 0) < 0)
	3142	+ if (tcf_action_dump(skb, exts->actions, 0, 0, false)
	3143	+ < 0)
2183	3144	goto nla_put_failure;
2184	3145	nla_nest_end(skb, nest);
2185	3146	} else if (exts->police) {
2186	3147	struct tc_action *act = tcf_exts_first_act(exts);
2187		- nest = nla_nest_start(skb, exts->police);
	3148	+ nest = nla_nest_start_noflag(skb, exts->police);
2188	3149	if (nest == NULL \|\| !act)
2189	3150	goto nla_put_failure;
2190	3151	if (tcf_action_dump_old(skb, act, 0, 0) < 0)
..	..	@@ -2203,6 +3164,31 @@
2203	3164	}
2204	3165	EXPORT_SYMBOL(tcf_exts_dump);
2205	3166
	3167	+int tcf_exts_terse_dump(struct sk_buff skb, struct tcf_exts exts)
	3168	+{
	3169	+#ifdef CONFIG_NET_CLS_ACT
	3170	+ struct nlattr *nest;
	3171	+
	3172	+ if (!exts->action \|\| !tcf_exts_has_actions(exts))
	3173	+ return 0;
	3174	+
	3175	+ nest = nla_nest_start_noflag(skb, exts->action);
	3176	+ if (!nest)
	3177	+ goto nla_put_failure;
	3178	+
	3179	+ if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0)
	3180	+ goto nla_put_failure;
	3181	+ nla_nest_end(skb, nest);
	3182	+ return 0;
	3183	+
	3184	+nla_put_failure:
	3185	+ nla_nest_cancel(skb, nest);
	3186	+ return -1;
	3187	+#else
	3188	+ return 0;
	3189	+#endif
	3190	+}
	3191	+EXPORT_SYMBOL(tcf_exts_terse_dump);
2206	3192
2207	3193	int tcf_exts_dump_stats(struct sk_buff skb, struct tcf_exts exts)
2208	3194	{
..	..	@@ -2215,62 +3201,687 @@
2215	3201	}
2216	3202	EXPORT_SYMBOL(tcf_exts_dump_stats);
2217	3203
2218		-static int tc_exts_setup_cb_egdev_call(struct tcf_exts *exts,
2219		- enum tc_setup_type type,
2220		- void *type_data, bool err_stop)
	3204	+static void tcf_block_offload_inc(struct tcf_block block, u32 flags)
2221	3205	{
2222		- int ok_count = 0;
2223		-#ifdef CONFIG_NET_CLS_ACT
2224		- const struct tc_action *a;
2225		- struct net_device *dev;
2226		- int i, ret;
	3206	+ if (*flags & TCA_CLS_FLAGS_IN_HW)
	3207	+ return;
	3208	+ *flags \|= TCA_CLS_FLAGS_IN_HW;
	3209	+ atomic_inc(&block->offloadcnt);
	3210	+}
2227	3211
2228		- if (!tcf_exts_has_actions(exts))
2229		- return 0;
	3212	+static void tcf_block_offload_dec(struct tcf_block block, u32 flags)
	3213	+{
	3214	+ if (!(*flags & TCA_CLS_FLAGS_IN_HW))
	3215	+ return;
	3216	+ *flags &= ~TCA_CLS_FLAGS_IN_HW;
	3217	+ atomic_dec(&block->offloadcnt);
	3218	+}
2230	3219
2231		- for (i = 0; i < exts->nr_actions; i++) {
2232		- a = exts->actions[i];
2233		- if (!a->ops->get_dev)
2234		- continue;
2235		- dev = a->ops->get_dev(a);
2236		- if (!dev)
2237		- continue;
2238		- ret = tc_setup_cb_egdev_call(dev, type, type_data, err_stop);
2239		- a->ops->put_dev(dev);
2240		- if (ret < 0)
2241		- return ret;
2242		- ok_count += ret;
	3220	+static void tc_cls_offload_cnt_update(struct tcf_block *block,
	3221	+ struct tcf_proto tp, u32 cnt,
	3222	+ u32 *flags, u32 diff, bool add)
	3223	+{
	3224	+ lockdep_assert_held(&block->cb_lock);
	3225	+
	3226	+ spin_lock(&tp->lock);
	3227	+ if (add) {
	3228	+ if (!*cnt)
	3229	+ tcf_block_offload_inc(block, flags);
	3230	+ *cnt += diff;
	3231	+ } else {
	3232	+ *cnt -= diff;
	3233	+ if (!*cnt)
	3234	+ tcf_block_offload_dec(block, flags);
2243	3235	}
2244		-#endif
	3236	+ spin_unlock(&tp->lock);
	3237	+}
	3238	+
	3239	+static void
	3240	+tc_cls_offload_cnt_reset(struct tcf_block block, struct tcf_proto tp,
	3241	+ u32 cnt, u32 flags)
	3242	+{
	3243	+ lockdep_assert_held(&block->cb_lock);
	3244	+
	3245	+ spin_lock(&tp->lock);
	3246	+ tcf_block_offload_dec(block, flags);
	3247	+ *cnt = 0;
	3248	+ spin_unlock(&tp->lock);
	3249	+}
	3250	+
	3251	+static int
	3252	+__tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
	3253	+ void *type_data, bool err_stop)
	3254	+{
	3255	+ struct flow_block_cb *block_cb;
	3256	+ int ok_count = 0;
	3257	+ int err;
	3258	+
	3259	+ list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
	3260	+ err = block_cb->cb(type, type_data, block_cb->cb_priv);
	3261	+ if (err) {
	3262	+ if (err_stop)
	3263	+ return err;
	3264	+ } else {
	3265	+ ok_count++;
	3266	+ }
	3267	+ }
2245	3268	return ok_count;
2246	3269	}
2247	3270
2248		-int tc_setup_cb_call(struct tcf_block block, struct tcf_exts exts,
2249		- enum tc_setup_type type, void *type_data, bool err_stop)
	3271	+int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
	3272	+ void *type_data, bool err_stop, bool rtnl_held)
2250	3273	{
	3274	+ bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
2251	3275	int ok_count;
2252		- int ret;
2253	3276
2254		- ret = tcf_block_cb_call(block, type, type_data, err_stop);
2255		- if (ret < 0)
2256		- return ret;
2257		- ok_count = ret;
	3277	+retry:
	3278	+ if (take_rtnl)
	3279	+ rtnl_lock();
	3280	+ down_read(&block->cb_lock);
	3281	+ /* Need to obtain rtnl lock if block is bound to devs that require it.
	3282	+ * In block bind code cb_lock is obtained while holding rtnl, so we must
	3283	+ * obtain the locks in same order here.
	3284	+ */
	3285	+ if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
	3286	+ up_read(&block->cb_lock);
	3287	+ take_rtnl = true;
	3288	+ goto retry;
	3289	+ }
2258	3290
2259		- if (!exts \|\| ok_count)
2260		- return ok_count;
2261		- ret = tc_exts_setup_cb_egdev_call(exts, type, type_data, err_stop);
2262		- if (ret < 0)
2263		- return ret;
2264		- ok_count += ret;
	3291	+ ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
2265	3292
	3293	+ up_read(&block->cb_lock);
	3294	+ if (take_rtnl)
	3295	+ rtnl_unlock();
2266	3296	return ok_count;
2267	3297	}
2268	3298	EXPORT_SYMBOL(tc_setup_cb_call);
	3299	+
	3300	+/* Non-destructive filter add. If filter that wasn't already in hardware is
	3301	+ * successfully offloaded, increment block offloads counter. On failure,
	3302	+ * previously offloaded filter is considered to be intact and offloads counter
	3303	+ * is not decremented.
	3304	+ */
	3305	+
	3306	+int tc_setup_cb_add(struct tcf_block block, struct tcf_proto tp,
	3307	+ enum tc_setup_type type, void *type_data, bool err_stop,
	3308	+ u32 flags, unsigned int in_hw_count, bool rtnl_held)
	3309	+{
	3310	+ bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
	3311	+ int ok_count;
	3312	+
	3313	+retry:
	3314	+ if (take_rtnl)
	3315	+ rtnl_lock();
	3316	+ down_read(&block->cb_lock);
	3317	+ /* Need to obtain rtnl lock if block is bound to devs that require it.
	3318	+ * In block bind code cb_lock is obtained while holding rtnl, so we must
	3319	+ * obtain the locks in same order here.
	3320	+ */
	3321	+ if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
	3322	+ up_read(&block->cb_lock);
	3323	+ take_rtnl = true;
	3324	+ goto retry;
	3325	+ }
	3326	+
	3327	+ /* Make sure all netdevs sharing this block are offload-capable. */
	3328	+ if (block->nooffloaddevcnt && err_stop) {
	3329	+ ok_count = -EOPNOTSUPP;
	3330	+ goto err_unlock;
	3331	+ }
	3332	+
	3333	+ ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
	3334	+ if (ok_count < 0)
	3335	+ goto err_unlock;
	3336	+
	3337	+ if (tp->ops->hw_add)
	3338	+ tp->ops->hw_add(tp, type_data);
	3339	+ if (ok_count > 0)
	3340	+ tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
	3341	+ ok_count, true);
	3342	+err_unlock:
	3343	+ up_read(&block->cb_lock);
	3344	+ if (take_rtnl)
	3345	+ rtnl_unlock();
	3346	+ return ok_count < 0 ? ok_count : 0;
	3347	+}
	3348	+EXPORT_SYMBOL(tc_setup_cb_add);
	3349	+
	3350	+/* Destructive filter replace. If filter that wasn't already in hardware is
	3351	+ * successfully offloaded, increment block offload counter. On failure,
	3352	+ * previously offloaded filter is considered to be destroyed and offload counter
	3353	+ * is decremented.
	3354	+ */
	3355	+
	3356	+int tc_setup_cb_replace(struct tcf_block block, struct tcf_proto tp,
	3357	+ enum tc_setup_type type, void *type_data, bool err_stop,
	3358	+ u32 old_flags, unsigned int old_in_hw_count,
	3359	+ u32 new_flags, unsigned int new_in_hw_count,
	3360	+ bool rtnl_held)
	3361	+{
	3362	+ bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
	3363	+ int ok_count;
	3364	+
	3365	+retry:
	3366	+ if (take_rtnl)
	3367	+ rtnl_lock();
	3368	+ down_read(&block->cb_lock);
	3369	+ /* Need to obtain rtnl lock if block is bound to devs that require it.
	3370	+ * In block bind code cb_lock is obtained while holding rtnl, so we must
	3371	+ * obtain the locks in same order here.
	3372	+ */
	3373	+ if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
	3374	+ up_read(&block->cb_lock);
	3375	+ take_rtnl = true;
	3376	+ goto retry;
	3377	+ }
	3378	+
	3379	+ /* Make sure all netdevs sharing this block are offload-capable. */
	3380	+ if (block->nooffloaddevcnt && err_stop) {
	3381	+ ok_count = -EOPNOTSUPP;
	3382	+ goto err_unlock;
	3383	+ }
	3384	+
	3385	+ tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
	3386	+ if (tp->ops->hw_del)
	3387	+ tp->ops->hw_del(tp, type_data);
	3388	+
	3389	+ ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
	3390	+ if (ok_count < 0)
	3391	+ goto err_unlock;
	3392	+
	3393	+ if (tp->ops->hw_add)
	3394	+ tp->ops->hw_add(tp, type_data);
	3395	+ if (ok_count > 0)
	3396	+ tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
	3397	+ new_flags, ok_count, true);
	3398	+err_unlock:
	3399	+ up_read(&block->cb_lock);
	3400	+ if (take_rtnl)
	3401	+ rtnl_unlock();
	3402	+ return ok_count < 0 ? ok_count : 0;
	3403	+}
	3404	+EXPORT_SYMBOL(tc_setup_cb_replace);
	3405	+
	3406	+/* Destroy filter and decrement block offload counter, if filter was previously
	3407	+ * offloaded.
	3408	+ */
	3409	+
	3410	+int tc_setup_cb_destroy(struct tcf_block block, struct tcf_proto tp,
	3411	+ enum tc_setup_type type, void *type_data, bool err_stop,
	3412	+ u32 flags, unsigned int in_hw_count, bool rtnl_held)
	3413	+{
	3414	+ bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
	3415	+ int ok_count;
	3416	+
	3417	+retry:
	3418	+ if (take_rtnl)
	3419	+ rtnl_lock();
	3420	+ down_read(&block->cb_lock);
	3421	+ /* Need to obtain rtnl lock if block is bound to devs that require it.
	3422	+ * In block bind code cb_lock is obtained while holding rtnl, so we must
	3423	+ * obtain the locks in same order here.
	3424	+ */
	3425	+ if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
	3426	+ up_read(&block->cb_lock);
	3427	+ take_rtnl = true;
	3428	+ goto retry;
	3429	+ }
	3430	+
	3431	+ ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
	3432	+
	3433	+ tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
	3434	+ if (tp->ops->hw_del)
	3435	+ tp->ops->hw_del(tp, type_data);
	3436	+
	3437	+ up_read(&block->cb_lock);
	3438	+ if (take_rtnl)
	3439	+ rtnl_unlock();
	3440	+ return ok_count < 0 ? ok_count : 0;
	3441	+}
	3442	+EXPORT_SYMBOL(tc_setup_cb_destroy);
	3443	+
	3444	+int tc_setup_cb_reoffload(struct tcf_block block, struct tcf_proto tp,
	3445	+ bool add, flow_setup_cb_t *cb,
	3446	+ enum tc_setup_type type, void *type_data,
	3447	+ void cb_priv, u32 flags, unsigned int *in_hw_count)
	3448	+{
	3449	+ int err = cb(type, type_data, cb_priv);
	3450	+
	3451	+ if (err) {
	3452	+ if (add && tc_skip_sw(*flags))
	3453	+ return err;
	3454	+ } else {
	3455	+ tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,
	3456	+ add);
	3457	+ }
	3458	+
	3459	+ return 0;
	3460	+}
	3461	+EXPORT_SYMBOL(tc_setup_cb_reoffload);
	3462	+
	3463	+static int tcf_act_get_cookie(struct flow_action_entry *entry,
	3464	+ const struct tc_action *act)
	3465	+{
	3466	+ struct tc_cookie *cookie;
	3467	+ int err = 0;
	3468	+
	3469	+ rcu_read_lock();
	3470	+ cookie = rcu_dereference(act->act_cookie);
	3471	+ if (cookie) {
	3472	+ entry->cookie = flow_action_cookie_create(cookie->data,
	3473	+ cookie->len,
	3474	+ GFP_ATOMIC);
	3475	+ if (!entry->cookie)
	3476	+ err = -ENOMEM;
	3477	+ }
	3478	+ rcu_read_unlock();
	3479	+ return err;
	3480	+}
	3481	+
	3482	+static void tcf_act_put_cookie(struct flow_action_entry *entry)
	3483	+{
	3484	+ flow_action_cookie_destroy(entry->cookie);
	3485	+}
	3486	+
	3487	+void tc_cleanup_flow_action(struct flow_action *flow_action)
	3488	+{
	3489	+ struct flow_action_entry *entry;
	3490	+ int i;
	3491	+
	3492	+ flow_action_for_each(i, entry, flow_action) {
	3493	+ tcf_act_put_cookie(entry);
	3494	+ if (entry->destructor)
	3495	+ entry->destructor(entry->destructor_priv);
	3496	+ }
	3497	+}
	3498	+EXPORT_SYMBOL(tc_cleanup_flow_action);
	3499	+
	3500	+static void tcf_mirred_get_dev(struct flow_action_entry *entry,
	3501	+ const struct tc_action *act)
	3502	+{
	3503	+#ifdef CONFIG_NET_CLS_ACT
	3504	+ entry->dev = act->ops->get_dev(act, &entry->destructor);
	3505	+ if (!entry->dev)
	3506	+ return;
	3507	+ entry->destructor_priv = entry->dev;
	3508	+#endif
	3509	+}
	3510	+
	3511	+static void tcf_tunnel_encap_put_tunnel(void *priv)
	3512	+{
	3513	+ struct ip_tunnel_info *tunnel = priv;
	3514	+
	3515	+ kfree(tunnel);
	3516	+}
	3517	+
	3518	+static int tcf_tunnel_encap_get_tunnel(struct flow_action_entry *entry,
	3519	+ const struct tc_action *act)
	3520	+{
	3521	+ entry->tunnel = tcf_tunnel_info_copy(act);
	3522	+ if (!entry->tunnel)
	3523	+ return -ENOMEM;
	3524	+ entry->destructor = tcf_tunnel_encap_put_tunnel;
	3525	+ entry->destructor_priv = entry->tunnel;
	3526	+ return 0;
	3527	+}
	3528	+
	3529	+static void tcf_sample_get_group(struct flow_action_entry *entry,
	3530	+ const struct tc_action *act)
	3531	+{
	3532	+#ifdef CONFIG_NET_CLS_ACT
	3533	+ entry->sample.psample_group =
	3534	+ act->ops->get_psample_group(act, &entry->destructor);
	3535	+ entry->destructor_priv = entry->sample.psample_group;
	3536	+#endif
	3537	+}
	3538	+
	3539	+static void tcf_gate_entry_destructor(void *priv)
	3540	+{
	3541	+ struct action_gate_entry *oe = priv;
	3542	+
	3543	+ kfree(oe);
	3544	+}
	3545	+
	3546	+static int tcf_gate_get_entries(struct flow_action_entry *entry,
	3547	+ const struct tc_action *act)
	3548	+{
	3549	+ entry->gate.entries = tcf_gate_get_list(act);
	3550	+
	3551	+ if (!entry->gate.entries)
	3552	+ return -EINVAL;
	3553	+
	3554	+ entry->destructor = tcf_gate_entry_destructor;
	3555	+ entry->destructor_priv = entry->gate.entries;
	3556	+
	3557	+ return 0;
	3558	+}
	3559	+
	3560	+static enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats)
	3561	+{
	3562	+ if (WARN_ON_ONCE(hw_stats > TCA_ACT_HW_STATS_ANY))
	3563	+ return FLOW_ACTION_HW_STATS_DONT_CARE;
	3564	+ else if (!hw_stats)
	3565	+ return FLOW_ACTION_HW_STATS_DISABLED;
	3566	+
	3567	+ return hw_stats;
	3568	+}
	3569	+
	3570	+int tc_setup_flow_action(struct flow_action *flow_action,
	3571	+ const struct tcf_exts *exts)
	3572	+{
	3573	+ struct tc_action *act;
	3574	+ int i, j, k, err = 0;
	3575	+
	3576	+ BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY);
	3577	+ BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE);
	3578	+ BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED);
	3579	+
	3580	+ if (!exts)
	3581	+ return 0;
	3582	+
	3583	+ j = 0;
	3584	+ tcf_exts_for_each_action(i, act, exts) {
	3585	+ struct flow_action_entry *entry;
	3586	+
	3587	+ entry = &flow_action->entries[j];
	3588	+ spin_lock_bh(&act->tcfa_lock);
	3589	+ err = tcf_act_get_cookie(entry, act);
	3590	+ if (err)
	3591	+ goto err_out_locked;
	3592	+
	3593	+ entry->hw_stats = tc_act_hw_stats(act->hw_stats);
	3594	+
	3595	+ if (is_tcf_gact_ok(act)) {
	3596	+ entry->id = FLOW_ACTION_ACCEPT;
	3597	+ } else if (is_tcf_gact_shot(act)) {
	3598	+ entry->id = FLOW_ACTION_DROP;
	3599	+ } else if (is_tcf_gact_trap(act)) {
	3600	+ entry->id = FLOW_ACTION_TRAP;
	3601	+ } else if (is_tcf_gact_goto_chain(act)) {
	3602	+ entry->id = FLOW_ACTION_GOTO;
	3603	+ entry->chain_index = tcf_gact_goto_chain_index(act);
	3604	+ } else if (is_tcf_mirred_egress_redirect(act)) {
	3605	+ entry->id = FLOW_ACTION_REDIRECT;
	3606	+ tcf_mirred_get_dev(entry, act);
	3607	+ } else if (is_tcf_mirred_egress_mirror(act)) {
	3608	+ entry->id = FLOW_ACTION_MIRRED;
	3609	+ tcf_mirred_get_dev(entry, act);
	3610	+ } else if (is_tcf_mirred_ingress_redirect(act)) {
	3611	+ entry->id = FLOW_ACTION_REDIRECT_INGRESS;
	3612	+ tcf_mirred_get_dev(entry, act);
	3613	+ } else if (is_tcf_mirred_ingress_mirror(act)) {
	3614	+ entry->id = FLOW_ACTION_MIRRED_INGRESS;
	3615	+ tcf_mirred_get_dev(entry, act);
	3616	+ } else if (is_tcf_vlan(act)) {
	3617	+ switch (tcf_vlan_action(act)) {
	3618	+ case TCA_VLAN_ACT_PUSH:
	3619	+ entry->id = FLOW_ACTION_VLAN_PUSH;
	3620	+ entry->vlan.vid = tcf_vlan_push_vid(act);
	3621	+ entry->vlan.proto = tcf_vlan_push_proto(act);
	3622	+ entry->vlan.prio = tcf_vlan_push_prio(act);
	3623	+ break;
	3624	+ case TCA_VLAN_ACT_POP:
	3625	+ entry->id = FLOW_ACTION_VLAN_POP;
	3626	+ break;
	3627	+ case TCA_VLAN_ACT_MODIFY:
	3628	+ entry->id = FLOW_ACTION_VLAN_MANGLE;
	3629	+ entry->vlan.vid = tcf_vlan_push_vid(act);
	3630	+ entry->vlan.proto = tcf_vlan_push_proto(act);
	3631	+ entry->vlan.prio = tcf_vlan_push_prio(act);
	3632	+ break;
	3633	+ default:
	3634	+ err = -EOPNOTSUPP;
	3635	+ goto err_out_locked;
	3636	+ }
	3637	+ } else if (is_tcf_tunnel_set(act)) {
	3638	+ entry->id = FLOW_ACTION_TUNNEL_ENCAP;
	3639	+ err = tcf_tunnel_encap_get_tunnel(entry, act);
	3640	+ if (err)
	3641	+ goto err_out_locked;
	3642	+ } else if (is_tcf_tunnel_release(act)) {
	3643	+ entry->id = FLOW_ACTION_TUNNEL_DECAP;
	3644	+ } else if (is_tcf_pedit(act)) {
	3645	+ for (k = 0; k < tcf_pedit_nkeys(act); k++) {
	3646	+ switch (tcf_pedit_cmd(act, k)) {
	3647	+ case TCA_PEDIT_KEY_EX_CMD_SET:
	3648	+ entry->id = FLOW_ACTION_MANGLE;
	3649	+ break;
	3650	+ case TCA_PEDIT_KEY_EX_CMD_ADD:
	3651	+ entry->id = FLOW_ACTION_ADD;
	3652	+ break;
	3653	+ default:
	3654	+ err = -EOPNOTSUPP;
	3655	+ goto err_out_locked;
	3656	+ }
	3657	+ entry->mangle.htype = tcf_pedit_htype(act, k);
	3658	+ entry->mangle.mask = tcf_pedit_mask(act, k);
	3659	+ entry->mangle.val = tcf_pedit_val(act, k);
	3660	+ entry->mangle.offset = tcf_pedit_offset(act, k);
	3661	+ entry->hw_stats = tc_act_hw_stats(act->hw_stats);
	3662	+ entry = &flow_action->entries[++j];
	3663	+ }
	3664	+ } else if (is_tcf_csum(act)) {
	3665	+ entry->id = FLOW_ACTION_CSUM;
	3666	+ entry->csum_flags = tcf_csum_update_flags(act);
	3667	+ } else if (is_tcf_skbedit_mark(act)) {
	3668	+ entry->id = FLOW_ACTION_MARK;
	3669	+ entry->mark = tcf_skbedit_mark(act);
	3670	+ } else if (is_tcf_sample(act)) {
	3671	+ entry->id = FLOW_ACTION_SAMPLE;
	3672	+ entry->sample.trunc_size = tcf_sample_trunc_size(act);
	3673	+ entry->sample.truncate = tcf_sample_truncate(act);
	3674	+ entry->sample.rate = tcf_sample_rate(act);
	3675	+ tcf_sample_get_group(entry, act);
	3676	+ } else if (is_tcf_police(act)) {
	3677	+ entry->id = FLOW_ACTION_POLICE;
	3678	+ entry->police.burst = tcf_police_burst(act);
	3679	+ entry->police.rate_bytes_ps =
	3680	+ tcf_police_rate_bytes_ps(act);
	3681	+ entry->police.mtu = tcf_police_tcfp_mtu(act);
	3682	+ entry->police.index = act->tcfa_index;
	3683	+ } else if (is_tcf_ct(act)) {
	3684	+ entry->id = FLOW_ACTION_CT;
	3685	+ entry->ct.action = tcf_ct_action(act);
	3686	+ entry->ct.zone = tcf_ct_zone(act);
	3687	+ entry->ct.flow_table = tcf_ct_ft(act);
	3688	+ } else if (is_tcf_mpls(act)) {
	3689	+ switch (tcf_mpls_action(act)) {
	3690	+ case TCA_MPLS_ACT_PUSH:
	3691	+ entry->id = FLOW_ACTION_MPLS_PUSH;
	3692	+ entry->mpls_push.proto = tcf_mpls_proto(act);
	3693	+ entry->mpls_push.label = tcf_mpls_label(act);
	3694	+ entry->mpls_push.tc = tcf_mpls_tc(act);
	3695	+ entry->mpls_push.bos = tcf_mpls_bos(act);
	3696	+ entry->mpls_push.ttl = tcf_mpls_ttl(act);
	3697	+ break;
	3698	+ case TCA_MPLS_ACT_POP:
	3699	+ entry->id = FLOW_ACTION_MPLS_POP;
	3700	+ entry->mpls_pop.proto = tcf_mpls_proto(act);
	3701	+ break;
	3702	+ case TCA_MPLS_ACT_MODIFY:
	3703	+ entry->id = FLOW_ACTION_MPLS_MANGLE;
	3704	+ entry->mpls_mangle.label = tcf_mpls_label(act);
	3705	+ entry->mpls_mangle.tc = tcf_mpls_tc(act);
	3706	+ entry->mpls_mangle.bos = tcf_mpls_bos(act);
	3707	+ entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
	3708	+ break;
	3709	+ default:
	3710	+ err = -EOPNOTSUPP;
	3711	+ goto err_out_locked;
	3712	+ }
	3713	+ } else if (is_tcf_skbedit_ptype(act)) {
	3714	+ entry->id = FLOW_ACTION_PTYPE;
	3715	+ entry->ptype = tcf_skbedit_ptype(act);
	3716	+ } else if (is_tcf_skbedit_priority(act)) {
	3717	+ entry->id = FLOW_ACTION_PRIORITY;
	3718	+ entry->priority = tcf_skbedit_priority(act);
	3719	+ } else if (is_tcf_gate(act)) {
	3720	+ entry->id = FLOW_ACTION_GATE;
	3721	+ entry->gate.index = tcf_gate_index(act);
	3722	+ entry->gate.prio = tcf_gate_prio(act);
	3723	+ entry->gate.basetime = tcf_gate_basetime(act);
	3724	+ entry->gate.cycletime = tcf_gate_cycletime(act);
	3725	+ entry->gate.cycletimeext = tcf_gate_cycletimeext(act);
	3726	+ entry->gate.num_entries = tcf_gate_num_entries(act);
	3727	+ err = tcf_gate_get_entries(entry, act);
	3728	+ if (err)
	3729	+ goto err_out_locked;
	3730	+ } else {
	3731	+ err = -EOPNOTSUPP;
	3732	+ goto err_out_locked;
	3733	+ }
	3734	+ spin_unlock_bh(&act->tcfa_lock);
	3735	+
	3736	+ if (!is_tcf_pedit(act))
	3737	+ j++;
	3738	+ }
	3739	+
	3740	+err_out:
	3741	+ if (err)
	3742	+ tc_cleanup_flow_action(flow_action);
	3743	+
	3744	+ return err;
	3745	+err_out_locked:
	3746	+ spin_unlock_bh(&act->tcfa_lock);
	3747	+ goto err_out;
	3748	+}
	3749	+EXPORT_SYMBOL(tc_setup_flow_action);
	3750	+
	3751	+unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
	3752	+{
	3753	+ unsigned int num_acts = 0;
	3754	+ struct tc_action *act;
	3755	+ int i;
	3756	+
	3757	+ tcf_exts_for_each_action(i, act, exts) {
	3758	+ if (is_tcf_pedit(act))
	3759	+ num_acts += tcf_pedit_nkeys(act);
	3760	+ else
	3761	+ num_acts++;
	3762	+ }
	3763	+ return num_acts;
	3764	+}
	3765	+EXPORT_SYMBOL(tcf_exts_num_actions);
	3766	+
	3767	+#ifdef CONFIG_NET_CLS_ACT
	3768	+static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr,
	3769	+ u32 *p_block_index,
	3770	+ struct netlink_ext_ack *extack)
	3771	+{
	3772	+ *p_block_index = nla_get_u32(block_index_attr);
	3773	+ if (!*p_block_index) {
	3774	+ NL_SET_ERR_MSG(extack, "Block number may not be zero");
	3775	+ return -EINVAL;
	3776	+ }
	3777	+
	3778	+ return 0;
	3779	+}
	3780	+
	3781	+int tcf_qevent_init(struct tcf_qevent qe, struct Qdisc sch,
	3782	+ enum flow_block_binder_type binder_type,
	3783	+ struct nlattr *block_index_attr,
	3784	+ struct netlink_ext_ack *extack)
	3785	+{
	3786	+ u32 block_index;
	3787	+ int err;
	3788	+
	3789	+ if (!block_index_attr)
	3790	+ return 0;
	3791	+
	3792	+ err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
	3793	+ if (err)
	3794	+ return err;
	3795	+
	3796	+ if (!block_index)
	3797	+ return 0;
	3798	+
	3799	+ qe->info.binder_type = binder_type;
	3800	+ qe->info.chain_head_change = tcf_chain_head_change_dflt;
	3801	+ qe->info.chain_head_change_priv = &qe->filter_chain;
	3802	+ qe->info.block_index = block_index;
	3803	+
	3804	+ return tcf_block_get_ext(&qe->block, sch, &qe->info, extack);
	3805	+}
	3806	+EXPORT_SYMBOL(tcf_qevent_init);
	3807	+
	3808	+void tcf_qevent_destroy(struct tcf_qevent qe, struct Qdisc sch)
	3809	+{
	3810	+ if (qe->info.block_index)
	3811	+ tcf_block_put_ext(qe->block, sch, &qe->info);
	3812	+}
	3813	+EXPORT_SYMBOL(tcf_qevent_destroy);
	3814	+
	3815	+int tcf_qevent_validate_change(struct tcf_qevent qe, struct nlattr block_index_attr,
	3816	+ struct netlink_ext_ack *extack)
	3817	+{
	3818	+ u32 block_index;
	3819	+ int err;
	3820	+
	3821	+ if (!block_index_attr)
	3822	+ return 0;
	3823	+
	3824	+ err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
	3825	+ if (err)
	3826	+ return err;
	3827	+
	3828	+ /* Bounce newly-configured block or change in block. */
	3829	+ if (block_index != qe->info.block_index) {
	3830	+ NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
	3831	+ return -EINVAL;
	3832	+ }
	3833	+
	3834	+ return 0;
	3835	+}
	3836	+EXPORT_SYMBOL(tcf_qevent_validate_change);
	3837	+
	3838	+struct sk_buff tcf_qevent_handle(struct tcf_qevent qe, struct Qdisc sch, struct sk_buff skb,
	3839	+ struct sk_buff *to_free, int ret)
	3840	+{
	3841	+ struct tcf_result cl_res;
	3842	+ struct tcf_proto *fl;
	3843	+
	3844	+ if (!qe->info.block_index)
	3845	+ return skb;
	3846	+
	3847	+ fl = rcu_dereference_bh(qe->filter_chain);
	3848	+
	3849	+ switch (tcf_classify(skb, fl, &cl_res, false)) {
	3850	+ case TC_ACT_SHOT:
	3851	+ qdisc_qstats_drop(sch);
	3852	+ __qdisc_drop(skb, to_free);
	3853	+ *ret = __NET_XMIT_BYPASS;
	3854	+ return NULL;
	3855	+ case TC_ACT_STOLEN:
	3856	+ case TC_ACT_QUEUED:
	3857	+ case TC_ACT_TRAP:
	3858	+ __qdisc_drop(skb, to_free);
	3859	+ *ret = __NET_XMIT_STOLEN;
	3860	+ return NULL;
	3861	+ case TC_ACT_REDIRECT:
	3862	+ skb_do_redirect(skb);
	3863	+ *ret = __NET_XMIT_STOLEN;
	3864	+ return NULL;
	3865	+ }
	3866	+
	3867	+ return skb;
	3868	+}
	3869	+EXPORT_SYMBOL(tcf_qevent_handle);
	3870	+
	3871	+int tcf_qevent_dump(struct sk_buff skb, int attr_name, struct tcf_qevent qe)
	3872	+{
	3873	+ if (!qe->info.block_index)
	3874	+ return 0;
	3875	+ return nla_put_u32(skb, attr_name, qe->info.block_index);
	3876	+}
	3877	+EXPORT_SYMBOL(tcf_qevent_dump);
	3878	+#endif
2269	3879
2270	3880	static __net_init int tcf_net_init(struct net *net)
2271	3881	{
2272	3882	struct tcf_net *tn = net_generic(net, tcf_net_id);
2273	3883
	3884	+ spin_lock_init(&tn->idr_lock);
2274	3885	idr_init(&tn->idr);
2275	3886	return 0;
2276	3887	}
..	..	@@ -2301,10 +3912,12 @@
2301	3912	if (err)
2302	3913	goto err_register_pernet_subsys;
2303	3914
2304		- rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, 0);
2305		- rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, 0);
	3915	+ rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
	3916	+ RTNL_FLAG_DOIT_UNLOCKED);
	3917	+ rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
	3918	+ RTNL_FLAG_DOIT_UNLOCKED);
2306	3919	rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
2307		- tc_dump_tfilter, 0);
	3920	+ tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED);
2308	3921	rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
2309	3922	rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
2310	3923	rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,