forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/include/net/sch_generic.h
....@@ -10,19 +10,22 @@
1010 #include <linux/percpu.h>
1111 #include <linux/dynamic_queue_limits.h>
1212 #include <linux/list.h>
13
-#include <net/net_seq_lock.h>
1413 #include <linux/refcount.h>
1514 #include <linux/workqueue.h>
15
+#include <linux/mutex.h>
16
+#include <linux/rwsem.h>
17
+#include <linux/atomic.h>
18
+#include <linux/hashtable.h>
19
+#include <linux/android_kabi.h>
1620 #include <net/gen_stats.h>
1721 #include <net/rtnetlink.h>
22
+#include <net/flow_offload.h>
1823 
1924 struct Qdisc_ops;
2025 struct qdisc_walker;
2126 struct tcf_walker;
2227 struct module;
23
-
24
-typedef int tc_setup_cb_t(enum tc_setup_type type,
25
-			  void *type_data, void *cb_priv);
28
+struct bpf_flow_keys;
2629 
2730 struct qdisc_rate_table {
2831 	struct tc_ratespec rate;
....@@ -34,6 +37,7 @@
3437 enum qdisc_state_t {
3538 	__QDISC_STATE_SCHED,
3639 	__QDISC_STATE_DEACTIVATED,
40
+	__QDISC_STATE_MISSED,
3741 };
3842 
3943 struct qdisc_size_table {
....@@ -48,10 +52,7 @@
4852 struct qdisc_skb_head {
4953 	struct sk_buff	*head;
5054 	struct sk_buff	*tail;
51
-	union {
52
-		u32		qlen;
53
-		atomic_t	atomic_qlen;
54
-	};
55
+	__u32		qlen;
5556 	spinlock_t	lock;
5657 };
5758 
....@@ -92,7 +93,7 @@
9293 	struct net_rate_estimator __rcu *rate_est;
9394 	struct gnet_stats_basic_cpu __percpu *cpu_bstats;
9495 	struct gnet_stats_queue	__percpu *cpu_qstats;
95
-	int			padded;
96
+	int			pad;
9697 	refcount_t		refcnt;
9798 
9899 	/*
....@@ -101,7 +102,7 @@
101102 	struct sk_buff_head	gso_skb ____cacheline_aligned_in_smp;
102103 	struct qdisc_skb_head	q;
103104 	struct gnet_stats_basic_packed bstats;
104
-	net_seqlock_t		running;
105
+	seqcount_t		running;
105106 	struct gnet_stats_queue	qstats;
106107 	unsigned long		state;
107108 	struct Qdisc            *next_sched;
....@@ -109,9 +110,15 @@
109110 
110111 	spinlock_t		busylock ____cacheline_aligned_in_smp;
111112 	spinlock_t		seqlock;
112
-#ifndef __GENKSYMS__
113
+
114
+	/* for NOLOCK qdisc, true if there are no enqueued skbs */
115
+	bool			empty;
113116 	struct rcu_head		rcu;
114
-#endif
117
+
118
+	ANDROID_KABI_RESERVE(1);
119
+
120
+	/* private data */
121
+	long privdata[] ____cacheline_aligned;
115122 };
116123 
117124 static inline void qdisc_refcount_inc(struct Qdisc *qdisc)
....@@ -138,44 +145,73 @@
138145 {
139146 	if (qdisc->flags & TCQ_F_NOLOCK)
140147 		return spin_is_locked(&qdisc->seqlock);
141
-#ifdef CONFIG_PREEMPT_RT_BASE
142
-	return spin_is_locked(&qdisc->running.lock) ? true : false;
143
-#else
144148 	return (raw_read_seqcount(&qdisc->running) & 1) ? true : false;
145
-#endif
149
+}
150
+
151
+static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
152
+{
153
+	return q->flags & TCQ_F_CPUSTATS;
154
+}
155
+
156
+static inline bool qdisc_is_empty(const struct Qdisc *qdisc)
157
+{
158
+	if (qdisc_is_percpu_stats(qdisc))
159
+		return READ_ONCE(qdisc->empty);
160
+	return !READ_ONCE(qdisc->q.qlen);
146161 }
147162 
148163 static inline bool qdisc_run_begin(struct Qdisc *qdisc)
149164 {
150165 	if (qdisc->flags & TCQ_F_NOLOCK) {
166
+		if (spin_trylock(&qdisc->seqlock))
167
+			goto nolock_empty;
168
+
169
+		/* No need to insist if the MISSED flag was already set.
170
+		 * Note that test_and_set_bit() also gives us memory ordering
171
+		 * guarantees wrt potential earlier enqueue() and below
172
+		 * spin_trylock(), both of which are necessary to prevent races
173
+		 */
174
+		if (test_and_set_bit(__QDISC_STATE_MISSED, &qdisc->state))
175
+			return false;
176
+
177
+		/* Try to take the lock again to make sure that we will either
178
+		 * grab it or the CPU that still has it will see MISSED set
179
+		 * when testing it in qdisc_run_end()
180
+		 */
151181 		if (!spin_trylock(&qdisc->seqlock))
152182 			return false;
183
+
184
+nolock_empty:
185
+		WRITE_ONCE(qdisc->empty, false);
153186 	} else if (qdisc_is_running(qdisc)) {
154187 		return false;
155188 	}
156
-#ifdef CONFIG_PREEMPT_RT_BASE
157
-	if (try_write_seqlock(&qdisc->running))
158
-		return true;
159
-	return false;
160
-#else
161189 	/* Variant of write_seqcount_begin() telling lockdep a trylock
162190 	 * was attempted.
163191 	 */
164192 	raw_write_seqcount_begin(&qdisc->running);
165193 	seqcount_acquire(&qdisc->running.dep_map, 0, 1, _RET_IP_);
166194 	return true;
167
-#endif
168195 }
169196 
170197 static inline void qdisc_run_end(struct Qdisc *qdisc)
171198 {
172
-#ifdef CONFIG_PREEMPT_RT_BASE
173
-	write_sequnlock(&qdisc->running);
174
-#else
175199 	write_seqcount_end(&qdisc->running);
176
-#endif
177
-	if (qdisc->flags & TCQ_F_NOLOCK)
200
+	if (qdisc->flags & TCQ_F_NOLOCK) {
178201 		spin_unlock(&qdisc->seqlock);
202
+
203
+		/* spin_unlock() only has store-release semantic. The unlock
204
+		 * and test_bit() ordering is a store-load ordering, so a full
205
+		 * memory barrier is needed here.
206
+		 */
207
+		smp_mb();
208
+
209
+		if (unlikely(test_bit(__QDISC_STATE_MISSED,
210
+				      &qdisc->state))) {
211
+			clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
212
+			__netif_schedule(qdisc);
213
+		}
214
+	}
179215 }
180216 
181217 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)
....@@ -194,6 +230,7 @@
194230 }
195231 
196232 struct Qdisc_class_ops {
233
+	unsigned int		flags;
197234 	/* Child qdisc manipulation */
198235 	struct netdev_queue *	(*select_queue)(struct Qdisc *, struct tcmsg *);
199236 	int			(*graft)(struct Qdisc *, unsigned long cl,
....@@ -223,6 +260,15 @@
223260 					struct sk_buff *skb, struct tcmsg*);
224261 	int			(*dump_stats)(struct Qdisc *, unsigned long,
225262 					struct gnet_dump *);
263
+
264
+	ANDROID_KABI_RESERVE(1);
265
+};
266
+
267
+/* Qdisc_class_ops flag values */
268
+
269
+/* Implements API that doesn't require rtnl lock */
270
+enum qdisc_class_ops_flags {
271
+	QDISC_CLASS_OPS_DOIT_UNLOCKED = 1,
226272 };
227273 
228274 struct Qdisc_ops {
....@@ -259,6 +305,8 @@
259305 	u32			(*egress_block_get)(struct Qdisc *sch);
260306 
261307 	struct module		*owner;
308
+
309
+	ANDROID_KABI_RESERVE(1);
262310 };
263311 
264312 
....@@ -270,7 +318,7 @@
270318 		};
271319 		const struct tcf_proto *goto_tp;
272320 
273
-		/* used by the TC_ACT_REINSERT action */
321
+		/* used in the skb_tc_reinsert function */
274322 		struct {
275323 			bool		ingress;
276324 			struct gnet_stats_queue *qstats;
....@@ -288,22 +336,29 @@
288336 					    const struct tcf_proto *,
289337 					    struct tcf_result *);
290338 	int			(*init)(struct tcf_proto*);
291
-	void			(*destroy)(struct tcf_proto *tp,
339
+	void			(*destroy)(struct tcf_proto *tp, bool rtnl_held,
292340 					   struct netlink_ext_ack *extack);
293341 
294342 	void*			(*get)(struct tcf_proto*, u32 handle);
343
+	void			(*put)(struct tcf_proto *tp, void *f);
295344 	int			(*change)(struct net *net, struct sk_buff *,
296345 					struct tcf_proto*, unsigned long,
297346 					u32 handle, struct nlattr **,
298
-					void **, bool,
347
+					void **, bool, bool,
299348 					struct netlink_ext_ack *);
300349 	int			(*delete)(struct tcf_proto *tp, void *arg,
301
-					  bool *last,
350
+					  bool *last, bool rtnl_held,
302351 					  struct netlink_ext_ack *);
303
-	void			(*walk)(struct tcf_proto*, struct tcf_walker *arg);
352
+	bool			(*delete_empty)(struct tcf_proto *tp);
353
+	void			(*walk)(struct tcf_proto *tp,
354
+					struct tcf_walker *arg, bool rtnl_held);
304355 	int			(*reoffload)(struct tcf_proto *tp, bool add,
305
-					     tc_setup_cb_t *cb, void *cb_priv,
356
+					     flow_setup_cb_t *cb, void *cb_priv,
306357 					     struct netlink_ext_ack *extack);
358
+	void			(*hw_add)(struct tcf_proto *tp,
359
+					  void *type_data);
360
+	void			(*hw_del)(struct tcf_proto *tp,
361
+					  void *type_data);
307362 	void			(*bind_class)(void *, u32, unsigned long,
308363 					      void *, unsigned long);
309364 	void *			(*tmplt_create)(struct net *net,
....@@ -314,12 +369,26 @@
314369 
315370 	/* rtnetlink specific */
316371 	int			(*dump)(struct net*, struct tcf_proto*, void *,
317
-					struct sk_buff *skb, struct tcmsg*);
372
+					struct sk_buff *skb, struct tcmsg*,
373
+					bool);
374
+	int			(*terse_dump)(struct net *net,
375
+					      struct tcf_proto *tp, void *fh,
376
+					      struct sk_buff *skb,
377
+					      struct tcmsg *t, bool rtnl_held);
318378 	int			(*tmplt_dump)(struct sk_buff *skb,
319379 					      struct net *net,
320380 					      void *tmplt_priv);
321381 
322382 	struct module		*owner;
383
+	int			flags;
384
+};
385
+
386
+/* Classifiers setting TCF_PROTO_OPS_DOIT_UNLOCKED in tcf_proto_ops->flags
387
+ * are expected to implement tcf_proto_ops->delete_empty(), otherwise race
388
+ * conditions can occur when filters are inserted/deleted simultaneously.
389
+ */
390
+enum tcf_proto_ops_flags {
391
+	TCF_PROTO_OPS_DOIT_UNLOCKED = 1,
323392 };
324393 
325394 struct tcf_proto {
....@@ -338,20 +407,32 @@
338407 	void			*data;
339408 	const struct tcf_proto_ops	*ops;
340409 	struct tcf_chain	*chain;
410
+	/* Lock protects tcf_proto shared state and can be used by unlocked
411
+	 * classifiers to protect their private data.
412
+	 */
413
+	spinlock_t		lock;
414
+	bool			deleting;
415
+	refcount_t		refcnt;
341416 	struct rcu_head		rcu;
417
+	struct hlist_node	destroy_ht_node;
342418 };
343419 
344420 struct qdisc_skb_cb {
345
-	unsigned int		pkt_len;
346
-	u16			slave_dev_queue_mapping;
347
-	u16			tc_classid;
421
+	struct {
422
+		unsigned int		pkt_len;
423
+		u16			slave_dev_queue_mapping;
424
+		u16			tc_classid;
425
+	};
348426 #define QDISC_CB_PRIV_LEN 20
349427 	unsigned char		data[QDISC_CB_PRIV_LEN];
428
+	u16			mru;
350429 };
351430 
352431 typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
353432 
354433 struct tcf_chain {
434
+	/* Protects filter_chain. */
435
+	struct mutex filter_chain_lock;
355436 	struct tcf_proto __rcu *filter_chain;
356437 	struct list_head list;
357438 	struct tcf_block *block;
....@@ -359,64 +440,78 @@
359440 	unsigned int refcnt;
360441 	unsigned int action_refcnt;
361442 	bool explicitly_created;
443
+	bool flushing;
362444 	const struct tcf_proto_ops *tmplt_ops;
363445 	void *tmplt_priv;
446
+	struct rcu_head rcu;
364447 };
365448 
366449 struct tcf_block {
450
+	/* Lock protects tcf_block and lifetime-management data of chains
451
+	 * attached to the block (refcnt, action_refcnt, explicitly_created).
452
+	 */
453
+	struct mutex lock;
367454 	struct list_head chain_list;
368455 	u32 index; /* block index for shared blocks */
369
-	unsigned int refcnt;
456
+	u32 classid; /* which class this block belongs to */
457
+	refcount_t refcnt;
370458 	struct net *net;
371459 	struct Qdisc *q;
372
-	struct list_head cb_list;
460
+	struct rw_semaphore cb_lock; /* protects cb_list and offload counters */
461
+	struct flow_block flow_block;
373462 	struct list_head owner_list;
374463 	bool keep_dst;
375
-	unsigned int offloadcnt; /* Number of oddloaded filters */
464
+	atomic_t offloadcnt; /* Number of oddloaded filters */
376465 	unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */
466
+	unsigned int lockeddevcnt; /* Number of devs that require rtnl lock. */
377467 	struct {
378468 		struct tcf_chain *chain;
379469 		struct list_head filter_chain_list;
380470 	} chain0;
471
+	struct rcu_head rcu;
472
+	DECLARE_HASHTABLE(proto_destroy_ht, 7);
473
+	struct mutex proto_destroy_lock; /* Lock for proto_destroy hashtable. */
381474 };
382475 
383
-static inline void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
476
+#ifdef CONFIG_PROVE_LOCKING
477
+static inline bool lockdep_tcf_chain_is_locked(struct tcf_chain *chain)
384478 {
385
-	if (*flags & TCA_CLS_FLAGS_IN_HW)
386
-		return;
387
-	*flags |= TCA_CLS_FLAGS_IN_HW;
388
-	block->offloadcnt++;
479
+	return lockdep_is_held(&chain->filter_chain_lock);
389480 }
390481 
391
-static inline void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
482
+static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp)
392483 {
393
-	if (!(*flags & TCA_CLS_FLAGS_IN_HW))
394
-		return;
395
-	*flags &= ~TCA_CLS_FLAGS_IN_HW;
396
-	block->offloadcnt--;
484
+	return lockdep_is_held(&tp->lock);
485
+}
486
+#else
487
+static inline bool lockdep_tcf_chain_is_locked(struct tcf_block *chain)
488
+{
489
+	return true;
397490 }
398491 
399
-static inline void
400
-tc_cls_offload_cnt_update(struct tcf_block *block, unsigned int *cnt,
401
-			  u32 *flags, bool add)
492
+static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp)
402493 {
403
-	if (add) {
404
-		if (!*cnt)
405
-			tcf_block_offload_inc(block, flags);
406
-		(*cnt)++;
407
-	} else {
408
-		(*cnt)--;
409
-		if (!*cnt)
410
-			tcf_block_offload_dec(block, flags);
411
-	}
494
+	return true;
412495 }
496
+#endif /* #ifdef CONFIG_PROVE_LOCKING */
497
+
498
+#define tcf_chain_dereference(p, chain)					\
499
+	rcu_dereference_protected(p, lockdep_tcf_chain_is_locked(chain))
500
+
501
+#define tcf_proto_dereference(p, tp)					\
502
+	rcu_dereference_protected(p, lockdep_tcf_proto_is_locked(tp))
413503 
414504 static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz)
415505 {
416506 	struct qdisc_skb_cb *qcb;
417507 
418
-	BUILD_BUG_ON(sizeof(skb->cb) < offsetof(struct qdisc_skb_cb, data) + sz);
508
+	BUILD_BUG_ON(sizeof(skb->cb) < sizeof(*qcb));
419509 	BUILD_BUG_ON(sizeof(qcb->data) < sz);
510
+}
511
+
512
+static inline int qdisc_qlen_cpu(const struct Qdisc *q)
513
+{
514
+	return this_cpu_ptr(q->cpu_qstats)->qlen;
420515 }
421516 
422517 static inline int qdisc_qlen(const struct Qdisc *q)
....@@ -424,14 +519,17 @@
424519 	return q->q.qlen;
425520 }
426521 
427
-static inline u32 qdisc_qlen_sum(const struct Qdisc *q)
522
+static inline int qdisc_qlen_sum(const struct Qdisc *q)
428523 {
429
-	u32 qlen = q->qstats.qlen;
524
+	__u32 qlen = q->qstats.qlen;
525
+	int i;
430526 
431
-	if (q->flags & TCQ_F_NOLOCK)
432
-		qlen += atomic_read(&q->q.atomic_qlen);
433
-	else
527
+	if (qdisc_is_percpu_stats(q)) {
528
+		for_each_possible_cpu(i)
529
+			qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen;
530
+	} else {
434531 		qlen += q->q.qlen;
532
+	}
435533 
436534 	return qlen;
437535 }
....@@ -490,7 +588,7 @@
490588 	return qdisc_lock(root);
491589 }
492590 
493
-static inline net_seqlock_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
591
+static inline seqcount_t *qdisc_root_sleeping_running(const struct Qdisc *qdisc)
494592 {
495593 	struct Qdisc *root = qdisc_root_sleeping(qdisc);
496594 
....@@ -588,8 +686,31 @@
588686 void qdisc_reset(struct Qdisc *qdisc);
589687 void qdisc_put(struct Qdisc *qdisc);
590688 void qdisc_put_unlocked(struct Qdisc *qdisc);
591
-void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n,
592
-			       unsigned int len);
689
+void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len);
690
+#ifdef CONFIG_NET_SCHED
691
+int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
692
+			      void *type_data);
693
+void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
694
+				struct Qdisc *new, struct Qdisc *old,
695
+				enum tc_setup_type type, void *type_data,
696
+				struct netlink_ext_ack *extack);
697
+#else
698
+static inline int
699
+qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
700
+			  void *type_data)
701
+{
702
+	q->flags &= ~TCQ_F_OFFLOADED;
703
+	return 0;
704
+}
705
+
706
+static inline void
707
+qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
708
+			   struct Qdisc *new, struct Qdisc *old,
709
+			   enum tc_setup_type type, void *type_data,
710
+			   struct netlink_ext_ack *extack)
711
+{
712
+}
713
+#endif
593714 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
594715 			  const struct Qdisc_ops *ops,
595716 			  struct netlink_ext_ack *extack);
....@@ -600,22 +721,6 @@
600721 void __qdisc_calculate_pkt_len(struct sk_buff *skb,
601722 			       const struct qdisc_size_table *stab);
602723 int skb_do_redirect(struct sk_buff *);
603
-
604
-static inline void skb_reset_tc(struct sk_buff *skb)
605
-{
606
-#ifdef CONFIG_NET_CLS_ACT
607
-	skb->tc_redirected = 0;
608
-#endif
609
-}
610
-
611
-static inline bool skb_is_tc_redirected(const struct sk_buff *skb)
612
-{
613
-#ifdef CONFIG_NET_CLS_ACT
614
-	return skb->tc_redirected;
615
-#else
616
-	return false;
617
-#endif
618
-}
619724 
620725 static inline bool skb_at_tc_ingress(const struct sk_buff *skb)
621726 {
....@@ -652,11 +757,6 @@
652757 	}
653758 }
654759 
655
-static inline void qdisc_reset_all_tx(struct net_device *dev)
656
-{
657
-	qdisc_reset_all_tx_gt(dev, 0);
658
-}
659
-
660760 /* Are all TX queues of the device empty?  */
661761 static inline bool qdisc_all_tx_empty(const struct net_device *dev)
662762 {
....@@ -667,7 +767,7 @@
667767 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
668768 		const struct Qdisc *q = rcu_dereference(txq->qdisc);
669769 
670
-		if (q->q.qlen) {
770
+		if (!qdisc_is_empty(q)) {
671771 			rcu_read_unlock();
672772 			return false;
673773 		}
....@@ -737,11 +837,6 @@
737837 	return sch->enqueue(skb, sch, to_free);
738838 }
739839 
740
-static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
741
-{
742
-	return q->flags & TCQ_F_CPUSTATS;
743
-}
744
-
745840 static inline void _bstats_update(struct gnet_stats_basic_packed *bstats,
746841 				  __u64 bytes, __u32 packets)
747842 {
....@@ -809,14 +904,14 @@
809904 	this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
810905 }
811906 
812
-static inline void qdisc_qstats_atomic_qlen_inc(struct Qdisc *sch)
907
+static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch)
813908 {
814
-	atomic_inc(&sch->q.atomic_qlen);
909
+	this_cpu_inc(sch->cpu_qstats->qlen);
815910 }
816911 
817
-static inline void qdisc_qstats_atomic_qlen_dec(struct Qdisc *sch)
912
+static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch)
818913 {
819
-	atomic_dec(&sch->q.atomic_qlen);
914
+	this_cpu_dec(sch->cpu_qstats->qlen);
820915 }
821916 
822917 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch)
....@@ -854,6 +949,41 @@
854949 	sch->qstats.overlimits++;
855950 }
856951 
952
+static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch)
953
+{
954
+	__u32 qlen = qdisc_qlen_sum(sch);
955
+
956
+	return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen);
957
+}
958
+
959
+static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch,  __u32 *qlen,
960
+					     __u32 *backlog)
961
+{
962
+	struct gnet_stats_queue qstats = { 0 };
963
+	__u32 len = qdisc_qlen_sum(sch);
964
+
965
+	__gnet_stats_copy_queue(&qstats, sch->cpu_qstats, &sch->qstats, len);
966
+	*qlen = qstats.qlen;
967
+	*backlog = qstats.backlog;
968
+}
969
+
970
+static inline void qdisc_tree_flush_backlog(struct Qdisc *sch)
971
+{
972
+	__u32 qlen, backlog;
973
+
974
+	qdisc_qstats_qlen_backlog(sch, &qlen, &backlog);
975
+	qdisc_tree_reduce_backlog(sch, qlen, backlog);
976
+}
977
+
978
+static inline void qdisc_purge_queue(struct Qdisc *sch)
979
+{
980
+	__u32 qlen, backlog;
981
+
982
+	qdisc_qstats_qlen_backlog(sch, &qlen, &backlog);
983
+	qdisc_reset(sch);
984
+	qdisc_tree_reduce_backlog(sch, qlen, backlog);
985
+}
986
+
857987 static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh)
858988 {
859989 	qh->head = NULL;
....@@ -861,8 +991,8 @@
861991 	qh->qlen = 0;
862992 }
863993 
864
-static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch,
865
-				       struct qdisc_skb_head *qh)
994
+static inline void __qdisc_enqueue_tail(struct sk_buff *skb,
995
+					struct qdisc_skb_head *qh)
866996 {
867997 	struct sk_buff *last = qh->tail;
868998 
....@@ -875,14 +1005,24 @@
8751005 		qh->head = skb;
8761006 	}
8771007 	qh->qlen++;
878
-	qdisc_qstats_backlog_inc(sch, skb);
879
-
880
-	return NET_XMIT_SUCCESS;
8811008 }
8821009 
8831010 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch)
8841011 {
885
-	return __qdisc_enqueue_tail(skb, sch, &sch->q);
1012
+	__qdisc_enqueue_tail(skb, &sch->q);
1013
+	qdisc_qstats_backlog_inc(sch, skb);
1014
+	return NET_XMIT_SUCCESS;
1015
+}
1016
+
1017
+static inline void __qdisc_enqueue_head(struct sk_buff *skb,
1018
+					struct qdisc_skb_head *qh)
1019
+{
1020
+	skb->next = qh->head;
1021
+
1022
+	if (!qh->head)
1023
+		qh->tail = skb;
1024
+	qh->head = skb;
1025
+	qh->qlen++;
8861026 }
8871027 
8881028 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)
....@@ -948,12 +1088,6 @@
9481088 	return 0;
9491089 }
9501090 
951
-static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch,
952
-						 struct sk_buff **to_free)
953
-{
954
-	return __qdisc_queue_drop_head(sch, &sch->q, to_free);
955
-}
956
-
9571091 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)
9581092 {
9591093 	const struct qdisc_skb_head *qh = &sch->q;
....@@ -981,6 +1115,32 @@
9811115 	return skb;
9821116 }
9831117 
1118
+static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch,
1119
+						 struct sk_buff *skb)
1120
+{
1121
+	if (qdisc_is_percpu_stats(sch)) {
1122
+		qdisc_qstats_cpu_backlog_dec(sch, skb);
1123
+		qdisc_bstats_cpu_update(sch, skb);
1124
+		qdisc_qstats_cpu_qlen_dec(sch);
1125
+	} else {
1126
+		qdisc_qstats_backlog_dec(sch, skb);
1127
+		qdisc_bstats_update(sch, skb);
1128
+		sch->q.qlen--;
1129
+	}
1130
+}
1131
+
1132
+static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,
1133
+						 unsigned int pkt_len)
1134
+{
1135
+	if (qdisc_is_percpu_stats(sch)) {
1136
+		qdisc_qstats_cpu_qlen_inc(sch);
1137
+		this_cpu_add(sch->cpu_qstats->backlog, pkt_len);
1138
+	} else {
1139
+		sch->qstats.backlog += pkt_len;
1140
+		sch->q.qlen++;
1141
+	}
1142
+}
1143
+
9841144 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */
9851145 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)
9861146 {
....@@ -988,8 +1148,13 @@
9881148 
9891149 	if (skb) {
9901150 		skb = __skb_dequeue(&sch->gso_skb);
991
-		qdisc_qstats_backlog_dec(sch, skb);
992
-		sch->q.qlen--;
1151
+		if (qdisc_is_percpu_stats(sch)) {
1152
+			qdisc_qstats_cpu_backlog_dec(sch, skb);
1153
+			qdisc_qstats_cpu_qlen_dec(sch);
1154
+		} else {
1155
+			qdisc_qstats_backlog_dec(sch, skb);
1156
+			sch->q.qlen--;
1157
+		}
9931158 	} else {
9941159 		skb = sch->dequeue(sch);
9951160 	}
....@@ -1016,7 +1181,6 @@
10161181 static inline void qdisc_reset_queue(struct Qdisc *sch)
10171182 {
10181183 	__qdisc_reset_queue(&sch->q);
1019
-	sch->qstats.backlog = 0;
10201184 }
10211185 
10221186 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new,
....@@ -1027,13 +1191,8 @@
10271191 	sch_tree_lock(sch);
10281192 	old = *pold;
10291193 	*pold = new;
1030
-	if (old != NULL) {
1031
-		unsigned int qlen = old->q.qlen;
1032
-		unsigned int backlog = old->qstats.backlog;
1033
-
1034
-		qdisc_reset(old);
1035
-		qdisc_tree_reduce_backlog(old, qlen, backlog);
1036
-	}
1194
+	if (old != NULL)
1195
+		qdisc_purge_queue(old);
10371196 	sch_tree_unlock(sch);
10381197 
10391198 	return old;
....@@ -1134,6 +1293,7 @@
11341293  */
11351294 struct mini_Qdisc {
11361295 	struct tcf_proto *filter_list;
1296
+	struct tcf_block *block;
11371297 	struct gnet_stats_basic_cpu __percpu *cpu_bstats;
11381298 	struct gnet_stats_queue	__percpu *cpu_qstats;
11391299 	struct rcu_head rcu;
....@@ -1160,18 +1320,12 @@
11601320 			  struct tcf_proto *tp_head);
11611321 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
11621322 			  struct mini_Qdisc __rcu **p_miniq);
1323
+void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp,
1324
+				struct tcf_block *block);
11631325 
1164
-static inline void skb_tc_reinsert(struct sk_buff *skb, struct tcf_result *res)
1326
+static inline int skb_tc_reinsert(struct sk_buff *skb, struct tcf_result *res)
11651327 {
1166
-	struct gnet_stats_queue *stats = res->qstats;
1167
-	int ret;
1168
-
1169
-	if (res->ingress)
1170
-		ret = netif_receive_skb(skb);
1171
-	else
1172
-		ret = dev_queue_xmit(skb);
1173
-	if (ret && stats)
1174
-		qstats_overlimit_inc(res->qstats);
1328
+	return res->ingress ? netif_receive_skb(skb) : dev_queue_xmit(skb);
11751329 }
11761330 
11771331 #endif