forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/include/linux/rculist.h
....@@ -40,6 +40,42 @@
4040  */
4141 #define list_next_rcu(list)	(*((struct list_head __rcu **)(&(list)->next)))
4242 
43
+/**
44
+ * list_tail_rcu - returns the prev pointer of the head of the list
45
+ * @head: the head of the list
46
+ *
47
+ * Note: This should only be used with the list header, and even then
48
+ * only if list_del() and similar primitives are not also used on the
49
+ * list header.
50
+ */
51
+#define list_tail_rcu(head)	(*((struct list_head __rcu **)(&(head)->prev)))
52
+
53
+/*
54
+ * Check during list traversal that we are within an RCU reader
55
+ */
56
+
57
+#define check_arg_count_one(dummy)
58
+
59
+#ifdef CONFIG_PROVE_RCU_LIST
60
+#define __list_check_rcu(dummy, cond, extra...)				\
61
+	({								\
62
+	check_arg_count_one(extra);					\
63
+	RCU_LOCKDEP_WARN(!(cond) && !rcu_read_lock_any_held(),		\
64
+			 "RCU-list traversed in non-reader section!");	\
65
+	})
66
+
67
+#define __list_check_srcu(cond)					 \
68
+	({								 \
69
+	RCU_LOCKDEP_WARN(!(cond),					 \
70
+		"RCU-list traversed without holding the required lock!");\
71
+	})
72
+#else
73
+#define __list_check_rcu(dummy, cond, extra...)				\
74
+	({ check_arg_count_one(extra); })
75
+
76
+#define __list_check_srcu(cond) ({ })
77
+#endif
78
+
4379 /*
4480  * Insert a new entry between two known consecutive entries.
4581  *
....@@ -155,7 +191,7 @@
155191 {
156192 	if (!hlist_unhashed(n)) {
157193 		__hlist_del(n);
158
-		n->pprev = NULL;
194
+		WRITE_ONCE(n->pprev, NULL);
159195 	}
160196 }
161197 
....@@ -182,7 +218,7 @@
182218  * @list:	the RCU-protected list to splice
183219  * @prev:	points to the last element of the existing list
184220  * @next:	points to the first element of the existing list
185
- * @sync:	function to sync: synchronize_rcu(), synchronize_sched(), ...
221
+ * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
186222  *
187223  * The list pointed to by @prev and @next can be RCU-read traversed
188224  * concurrently with this function.
....@@ -220,6 +256,8 @@
220256 	 */
221257 
222258 	sync();
259
+	ASSERT_EXCLUSIVE_ACCESS(*first);
260
+	ASSERT_EXCLUSIVE_ACCESS(*last);
223261 
224262 	/*
225263 	 * Readers are finished with the source list, so perform splice.
....@@ -240,7 +278,7 @@
240278  *                        designed for stacks.
241279  * @list:	the RCU-protected list to splice
242280  * @head:	the place in the existing list to splice the first list into
243
- * @sync:	function to sync: synchronize_rcu(), synchronize_sched(), ...
281
+ * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
244282  */
245283 static inline void list_splice_init_rcu(struct list_head *list,
246284 					struct list_head *head,
....@@ -255,7 +293,7 @@
255293  *                             list, designed for queues.
256294  * @list:	the RCU-protected list to splice
257295  * @head:	the place in the existing list to splice the first list into
258
- * @sync:	function to sync: synchronize_rcu(), synchronize_sched(), ...
296
+ * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
259297  */
260298 static inline void list_splice_tail_init_rcu(struct list_head *list,
261299 					     struct list_head *head,
....@@ -343,14 +381,35 @@
343381  * @pos:	the type * to use as a loop cursor.
344382  * @head:	the head for your list.
345383  * @member:	the name of the list_head within the struct.
384
+ * @cond:	optional lockdep expression if called from non-RCU protection.
346385  *
347386  * This list-traversal primitive may safely run concurrently with
348387  * the _rcu list-mutation primitives such as list_add_rcu()
349388  * as long as the traversal is guarded by rcu_read_lock().
350389  */
351
-#define list_for_each_entry_rcu(pos, head, member) \
352
-	for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \
353
-		&pos->member != (head); \
390
+#define list_for_each_entry_rcu(pos, head, member, cond...)		\
391
+	for (__list_check_rcu(dummy, ## cond, 0),			\
392
+	     pos = list_entry_rcu((head)->next, typeof(*pos), member);	\
393
+		&pos->member != (head);					\
394
+		pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
395
+
396
+/**
397
+ * list_for_each_entry_srcu	-	iterate over rcu list of given type
398
+ * @pos:	the type * to use as a loop cursor.
399
+ * @head:	the head for your list.
400
+ * @member:	the name of the list_head within the struct.
401
+ * @cond:	lockdep expression for the lock required to traverse the list.
402
+ *
403
+ * This list-traversal primitive may safely run concurrently with
404
+ * the _rcu list-mutation primitives such as list_add_rcu()
405
+ * as long as the traversal is guarded by srcu_read_lock().
406
+ * The lockdep expression srcu_read_lock_held() can be passed as the
407
+ * cond argument from read side.
408
+ */
409
+#define list_for_each_entry_srcu(pos, head, member, cond)		\
410
+	for (__list_check_srcu(cond),					\
411
+	     pos = list_entry_rcu((head)->next, typeof(*pos), member);	\
412
+		&pos->member != (head);					\
354413 		pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
355414 
356415 /**
....@@ -359,13 +418,12 @@
359418  * @type:       the type of the struct this is embedded in.
360419  * @member:     the name of the list_head within the struct.
361420  *
362
- * This primitive may safely run concurrently with the _rcu list-mutation
363
- * primitives such as list_add_rcu(), but requires some implicit RCU
364
- * read-side guarding.  One example is running within a special
365
- * exception-time environment where preemption is disabled and where
366
- * lockdep cannot be invoked (in which case updaters must use RCU-sched,
367
- * as in synchronize_sched(), call_rcu_sched(), and friends).  Another
368
- * example is when items are added to the list, but never deleted.
421
+ * This primitive may safely run concurrently with the _rcu
422
+ * list-mutation primitives such as list_add_rcu(), but requires some
423
+ * implicit RCU read-side guarding.  One example is running within a special
424
+ * exception-time environment where preemption is disabled and where lockdep
425
+ * cannot be invoked.  Another example is when items are added to the list,
426
+ * but never deleted.
369427  */
370428 #define list_entry_lockless(ptr, type, member) \
371429 	container_of((typeof(ptr))READ_ONCE(ptr), type, member)
....@@ -376,13 +434,12 @@
376434  * @head:	the head for your list.
377435  * @member:	the name of the list_struct within the struct.
378436  *
379
- * This primitive may safely run concurrently with the _rcu list-mutation
380
- * primitives such as list_add_rcu(), but requires some implicit RCU
381
- * read-side guarding.  One example is running within a special
382
- * exception-time environment where preemption is disabled and where
383
- * lockdep cannot be invoked (in which case updaters must use RCU-sched,
384
- * as in synchronize_sched(), call_rcu_sched(), and friends).  Another
385
- * example is when items are added to the list, but never deleted.
437
+ * This primitive may safely run concurrently with the _rcu
438
+ * list-mutation primitives such as list_add_rcu(), but requires some
439
+ * implicit RCU read-side guarding.  One example is running within a special
440
+ * exception-time environment where preemption is disabled and where lockdep
441
+ * cannot be invoked.  Another example is when items are added to the list,
442
+ * but never deleted.
386443  */
387444 #define list_for_each_entry_lockless(pos, head, member) \
388445 	for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
....@@ -455,7 +512,7 @@
455512 static inline void hlist_del_rcu(struct hlist_node *n)
456513 {
457514 	__hlist_del(n);
458
-	n->pprev = LIST_POISON2;
515
+	WRITE_ONCE(n->pprev, LIST_POISON2);
459516 }
460517 
461518 /**
....@@ -471,11 +528,32 @@
471528 	struct hlist_node *next = old->next;
472529 
473530 	new->next = next;
474
-	new->pprev = old->pprev;
531
+	WRITE_ONCE(new->pprev, old->pprev);
475532 	rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
476533 	if (next)
477
-		new->next->pprev = &new->next;
478
-	old->pprev = LIST_POISON2;
534
+		WRITE_ONCE(new->next->pprev, &new->next);
535
+	WRITE_ONCE(old->pprev, LIST_POISON2);
536
+}
537
+
538
+/**
539
+ * hlists_swap_heads_rcu - swap the lists the hlist heads point to
540
+ * @left:  The hlist head on the left
541
+ * @right: The hlist head on the right
542
+ *
543
+ * The lists start out as [@left  ][node1 ... ] and
544
+ *                        [@right ][node2 ... ]
545
+ * The lists end up as    [@left  ][node2 ... ]
546
+ *                        [@right ][node1 ... ]
547
+ */
548
+static inline void hlists_swap_heads_rcu(struct hlist_head *left, struct hlist_head *right)
549
+{
550
+	struct hlist_node *node1 = left->first;
551
+	struct hlist_node *node2 = right->first;
552
+
553
+	rcu_assign_pointer(left->first, node2);
554
+	rcu_assign_pointer(right->first, node1);
555
+	WRITE_ONCE(node2->pprev, &left->first);
556
+	WRITE_ONCE(node1->pprev, &right->first);
479557 }
480558 
481559 /*
....@@ -510,10 +588,10 @@
510588 	struct hlist_node *first = h->first;
511589 
512590 	n->next = first;
513
-	n->pprev = &h->first;
591
+	WRITE_ONCE(n->pprev, &h->first);
514592 	rcu_assign_pointer(hlist_first_rcu(h), n);
515593 	if (first)
516
-		first->pprev = &n->next;
594
+		WRITE_ONCE(first->pprev, &n->next);
517595 }
518596 
519597 /**
....@@ -546,7 +624,7 @@
546624 
547625 	if (last) {
548626 		n->next = last->next;
549
-		n->pprev = &last->next;
627
+		WRITE_ONCE(n->pprev, &last->next);
550628 		rcu_assign_pointer(hlist_next_rcu(last), n);
551629 	} else {
552630 		hlist_add_head_rcu(n, h);
....@@ -574,10 +652,10 @@
574652 static inline void hlist_add_before_rcu(struct hlist_node *n,
575653 					struct hlist_node *next)
576654 {
577
-	n->pprev = next->pprev;
655
+	WRITE_ONCE(n->pprev, next->pprev);
578656 	n->next = next;
579657 	rcu_assign_pointer(hlist_pprev_rcu(n), n);
580
-	next->pprev = &n->next;
658
+	WRITE_ONCE(next->pprev, &n->next);
581659 }
582660 
583661 /**
....@@ -602,10 +680,10 @@
602680 					struct hlist_node *prev)
603681 {
604682 	n->next = prev->next;
605
-	n->pprev = &prev->next;
683
+	WRITE_ONCE(n->pprev, &prev->next);
606684 	rcu_assign_pointer(hlist_next_rcu(prev), n);
607685 	if (n->next)
608
-		n->next->pprev = &n->next;
686
+		WRITE_ONCE(n->next->pprev, &n->next);
609687 }
610688 
611689 #define __hlist_for_each_rcu(pos, head)				\
....@@ -618,13 +696,36 @@
618696  * @pos:	the type * to use as a loop cursor.
619697  * @head:	the head for your list.
620698  * @member:	the name of the hlist_node within the struct.
699
+ * @cond:	optional lockdep expression if called from non-RCU protection.
621700  *
622701  * This list-traversal primitive may safely run concurrently with
623702  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
624703  * as long as the traversal is guarded by rcu_read_lock().
625704  */
626
-#define hlist_for_each_entry_rcu(pos, head, member)			\
627
-	for (pos = hlist_entry_safe (rcu_dereference_raw(hlist_first_rcu(head)),\
705
+#define hlist_for_each_entry_rcu(pos, head, member, cond...)		\
706
+	for (__list_check_rcu(dummy, ## cond, 0),			\
707
+	     pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
708
+			typeof(*(pos)), member);			\
709
+		pos;							\
710
+		pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
711
+			&(pos)->member)), typeof(*(pos)), member))
712
+
713
+/**
714
+ * hlist_for_each_entry_srcu - iterate over rcu list of given type
715
+ * @pos:	the type * to use as a loop cursor.
716
+ * @head:	the head for your list.
717
+ * @member:	the name of the hlist_node within the struct.
718
+ * @cond:	lockdep expression for the lock required to traverse the list.
719
+ *
720
+ * This list-traversal primitive may safely run concurrently with
721
+ * the _rcu list-mutation primitives such as hlist_add_head_rcu()
722
+ * as long as the traversal is guarded by srcu_read_lock().
723
+ * The lockdep expression srcu_read_lock_held() can be passed as the
724
+ * cond argument from read side.
725
+ */
726
+#define hlist_for_each_entry_srcu(pos, head, member, cond)		\
727
+	for (__list_check_srcu(cond),					\
728
+	     pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
628729 			typeof(*(pos)), member);			\
629730 		pos;							\
630731 		pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
....@@ -644,10 +745,10 @@
644745  * not do any RCU debugging or tracing.
645746  */
646747 #define hlist_for_each_entry_rcu_notrace(pos, head, member)			\
647
-	for (pos = hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\
748
+	for (pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_first_rcu(head)),\
648749 			typeof(*(pos)), member);			\
649750 		pos;							\
650
-		pos = hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\
751
+		pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_next_rcu(\
651752 			&(pos)->member)), typeof(*(pos)), member))
652753 
653754 /**