forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 10ebd8556b7990499c896a550e3d416b444211e6 
 kernel/include/linux/rhashtable.h
....@@ -24,12 +24,27 @@
2424 #include <linux/list_nulls.h>
2525 #include <linux/workqueue.h>
2626 #include <linux/rculist.h>
27
+#include <linux/bit_spinlock.h>
2728 
2829 #include <linux/rhashtable-types.h>
2930 /*
31
+ * Objects in an rhashtable have an embedded struct rhash_head
32
+ * which is linked into as hash chain from the hash table - or one
33
+ * of two or more hash tables when the rhashtable is being resized.
3034  * The end of the chain is marked with a special nulls marks which has
31
- * the least significant bit set.
35
+ * the least significant bit set but otherwise stores the address of
36
+ * the hash bucket.  This allows us to be sure we've found the end
37
+ * of the right list.
38
+ * The value stored in the hash bucket has BIT(0) used as a lock bit.
39
+ * This bit must be atomically set before any changes are made to
40
+ * the chain.  To avoid dereferencing this pointer without clearing
41
+ * the bit first, we use an opaque 'struct rhash_lock_head *' for the
42
+ * pointer stored in the bucket.  This struct needs to be defined so
43
+ * that rcu_dereference() works on it, but it has no content so a
44
+ * cast is needed for it to be useful.  This ensures it isn't
45
+ * used by mistake with clearing the lock bit first.
3246  */
47
+struct rhash_lock_head {};
3348 
3449 /* Maximum chain length before rehash
3550  *
....@@ -52,8 +67,6 @@
5267  * @nest: Number of bits of first-level nested table.
5368  * @rehash: Current bucket being rehashed
5469  * @hash_rnd: Random seed to fold into hash
55
- * @locks_mask: Mask to apply before accessing locks[]
56
- * @locks: Array of spinlocks protecting individual buckets
5770  * @walkers: List of active walkers
5871  * @rcu: RCU structure for freeing the table
5972  * @future_tbl: Table under construction during rehashing
....@@ -63,20 +76,34 @@
6376 struct bucket_table {
6477 	unsigned int		size;
6578 	unsigned int		nest;
66
-	unsigned int		rehash;
6779 	u32			hash_rnd;
68
-	unsigned int		locks_mask;
69
-	spinlock_t		*locks;
7080 	struct list_head	walkers;
7181 	struct rcu_head		rcu;
7282 
7383 	struct bucket_table __rcu *future_tbl;
7484 
75
-	struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp;
85
+	struct lockdep_map	dep_map;
86
+
87
+	struct rhash_lock_head __rcu *buckets[] ____cacheline_aligned_in_smp;
7688 };
7789 
90
+/*
91
+ * NULLS_MARKER() expects a hash value with the low
92
+ * bits mostly likely to be significant, and it discards
93
+ * the msb.
94
+ * We give it an address, in which the bottom bit is
95
+ * always 0, and the msb might be significant.
96
+ * So we shift the address down one bit to align with
97
+ * expectations and avoid losing a significant bit.
98
+ *
99
+ * We never store the NULLS_MARKER in the hash table
100
+ * itself as we need the lsb for locking.
101
+ * Instead we store a NULL
102
+ */
103
+#define	RHT_NULLS_MARKER(ptr)	\
104
+	((void *)NULLS_MARKER(((unsigned long) (ptr)) >> 1))
78105 #define INIT_RHT_NULLS_HEAD(ptr)	\
79
-	((ptr) = (typeof(ptr)) NULLS_MARKER(0))
106
+	((ptr) = NULL)
80107 
81108 static inline bool rht_is_a_nulls(const struct rhash_head *ptr)
82109 {
....@@ -196,25 +223,6 @@
196223 	return atomic_read(&ht->nelems) >= ht->max_elems;
197224 }
198225 
199
-/* The bucket lock is selected based on the hash and protects mutations
200
- * on a group of hash buckets.
201
- *
202
- * A maximum of tbl->size/2 bucket locks is allocated. This ensures that
203
- * a single lock always covers both buckets which may both contains
204
- * entries which link to the same bucket of the old table during resizing.
205
- * This allows to simplify the locking as locking the bucket in both
206
- * tables during resize always guarantee protection.
207
- *
208
- * IMPORTANT: When holding the bucket lock of both the old and new table
209
- * during expansions and shrinking, the old bucket lock must always be
210
- * acquired first.
211
- */
212
-static inline spinlock_t *rht_bucket_lock(const struct bucket_table *tbl,
213
-					  unsigned int hash)
214
-{
215
-	return &tbl->locks[hash & tbl->locks_mask];
216
-}
217
-
218226 #ifdef CONFIG_PROVE_LOCKING
219227 int lockdep_rht_mutex_is_held(struct rhashtable *ht);
220228 int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash);
....@@ -253,11 +261,12 @@
253261 				 void *arg);
254262 void rhashtable_destroy(struct rhashtable *ht);
255263 
256
-struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
257
-					    unsigned int hash);
258
-struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
259
-						   struct bucket_table *tbl,
260
-						   unsigned int hash);
264
+struct rhash_lock_head __rcu **rht_bucket_nested(
265
+	const struct bucket_table *tbl, unsigned int hash);
266
+struct rhash_lock_head __rcu **__rht_bucket_nested(
267
+	const struct bucket_table *tbl, unsigned int hash);
268
+struct rhash_lock_head __rcu **rht_bucket_nested_insert(
269
+	struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash);
261270 
262271 #define rht_dereference(p, ht) \
263272 	rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
....@@ -274,37 +283,137 @@
274283 #define rht_entry(tpos, pos, member) \
275284 	({ tpos = container_of(pos, typeof(*tpos), member); 1; })
276285 
277
-static inline struct rhash_head __rcu *const *rht_bucket(
286
+static inline struct rhash_lock_head __rcu *const *rht_bucket(
278287 	const struct bucket_table *tbl, unsigned int hash)
279288 {
280289 	return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
281290 				     &tbl->buckets[hash];
282291 }
283292 
284
-static inline struct rhash_head __rcu **rht_bucket_var(
293
+static inline struct rhash_lock_head __rcu **rht_bucket_var(
285294 	struct bucket_table *tbl, unsigned int hash)
286295 {
287
-	return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
296
+	return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) :
288297 				     &tbl->buckets[hash];
289298 }
290299 
291
-static inline struct rhash_head __rcu **rht_bucket_insert(
300
+static inline struct rhash_lock_head __rcu **rht_bucket_insert(
292301 	struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
293302 {
294303 	return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :
295304 				     &tbl->buckets[hash];
296305 }
297306 
307
+/*
308
+ * We lock a bucket by setting BIT(0) in the pointer - this is always
309
+ * zero in real pointers.  The NULLS mark is never stored in the bucket,
310
+ * rather we store NULL if the bucket is empty.
311
+ * bit_spin_locks do not handle contention well, but the whole point
312
+ * of the hashtable design is to achieve minimum per-bucket contention.
313
+ * A nested hash table might not have a bucket pointer.  In that case
314
+ * we cannot get a lock.  For remove and replace the bucket cannot be
315
+ * interesting and doesn't need locking.
316
+ * For insert we allocate the bucket if this is the last bucket_table,
317
+ * and then take the lock.
318
+ * Sometimes we unlock a bucket by writing a new pointer there.  In that
319
+ * case we don't need to unlock, but we do need to reset state such as
320
+ * local_bh. For that we have rht_assign_unlock().  As rcu_assign_pointer()
321
+ * provides the same release semantics that bit_spin_unlock() provides,
322
+ * this is safe.
323
+ * When we write to a bucket without unlocking, we use rht_assign_locked().
324
+ */
325
+
326
+static inline void rht_lock(struct bucket_table *tbl,
327
+			    struct rhash_lock_head __rcu **bkt)
328
+{
329
+	local_bh_disable();
330
+	bit_spin_lock(0, (unsigned long *)bkt);
331
+	lock_map_acquire(&tbl->dep_map);
332
+}
333
+
334
+static inline void rht_lock_nested(struct bucket_table *tbl,
335
+				   struct rhash_lock_head __rcu **bucket,
336
+				   unsigned int subclass)
337
+{
338
+	local_bh_disable();
339
+	bit_spin_lock(0, (unsigned long *)bucket);
340
+	lock_acquire_exclusive(&tbl->dep_map, subclass, 0, NULL, _THIS_IP_);
341
+}
342
+
343
+static inline void rht_unlock(struct bucket_table *tbl,
344
+			      struct rhash_lock_head __rcu **bkt)
345
+{
346
+	lock_map_release(&tbl->dep_map);
347
+	bit_spin_unlock(0, (unsigned long *)bkt);
348
+	local_bh_enable();
349
+}
350
+
351
+static inline struct rhash_head *__rht_ptr(
352
+	struct rhash_lock_head *p, struct rhash_lock_head __rcu *const *bkt)
353
+{
354
+	return (struct rhash_head *)
355
+		((unsigned long)p & ~BIT(0) ?:
356
+		 (unsigned long)RHT_NULLS_MARKER(bkt));
357
+}
358
+
359
+/*
360
+ * Where 'bkt' is a bucket and might be locked:
361
+ *   rht_ptr_rcu() dereferences that pointer and clears the lock bit.
362
+ *   rht_ptr() dereferences in a context where the bucket is locked.
363
+ *   rht_ptr_exclusive() dereferences in a context where exclusive
364
+ *            access is guaranteed, such as when destroying the table.
365
+ */
366
+static inline struct rhash_head *rht_ptr_rcu(
367
+	struct rhash_lock_head __rcu *const *bkt)
368
+{
369
+	return __rht_ptr(rcu_dereference(*bkt), bkt);
370
+}
371
+
372
+static inline struct rhash_head *rht_ptr(
373
+	struct rhash_lock_head __rcu *const *bkt,
374
+	struct bucket_table *tbl,
375
+	unsigned int hash)
376
+{
377
+	return __rht_ptr(rht_dereference_bucket(*bkt, tbl, hash), bkt);
378
+}
379
+
380
+static inline struct rhash_head *rht_ptr_exclusive(
381
+	struct rhash_lock_head __rcu *const *bkt)
382
+{
383
+	return __rht_ptr(rcu_dereference_protected(*bkt, 1), bkt);
384
+}
385
+
386
+static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt,
387
+				     struct rhash_head *obj)
388
+{
389
+	if (rht_is_a_nulls(obj))
390
+		obj = NULL;
391
+	rcu_assign_pointer(*bkt, (void *)((unsigned long)obj | BIT(0)));
392
+}
393
+
394
+static inline void rht_assign_unlock(struct bucket_table *tbl,
395
+				     struct rhash_lock_head __rcu **bkt,
396
+				     struct rhash_head *obj)
397
+{
398
+	if (rht_is_a_nulls(obj))
399
+		obj = NULL;
400
+	lock_map_release(&tbl->dep_map);
401
+	rcu_assign_pointer(*bkt, (void *)obj);
402
+	preempt_enable();
403
+	__release(bitlock);
404
+	local_bh_enable();
405
+}
406
+
298407 /**
299
- * rht_for_each_continue - continue iterating over hash chain
408
+ * rht_for_each_from - iterate over hash chain from given head
300409  * @pos:	the &struct rhash_head to use as a loop cursor.
301
- * @head:	the previous &struct rhash_head to continue from
410
+ * @head:	the &struct rhash_head to start from
302411  * @tbl:	the &struct bucket_table
303412  * @hash:	the hash value / bucket index
304413  */
305
-#define rht_for_each_continue(pos, head, tbl, hash) \
306
-	for (pos = rht_dereference_bucket(head, tbl, hash); \
307
-	     !rht_is_a_nulls(pos); \
414
+#define rht_for_each_from(pos, head, tbl, hash) \
415
+	for (pos = head;			\
416
+	     !rht_is_a_nulls(pos);		\
308417 	     pos = rht_dereference_bucket((pos)->next, tbl, hash))
309418 
310419 /**
....@@ -314,19 +423,20 @@
314423  * @hash:	the hash value / bucket index
315424  */
316425 #define rht_for_each(pos, tbl, hash) \
317
-	rht_for_each_continue(pos, *rht_bucket(tbl, hash), tbl, hash)
426
+	rht_for_each_from(pos, rht_ptr(rht_bucket(tbl, hash), tbl, hash),  \
427
+			  tbl, hash)
318428 
319429 /**
320
- * rht_for_each_entry_continue - continue iterating over hash chain
430
+ * rht_for_each_entry_from - iterate over hash chain from given head
321431  * @tpos:	the type * to use as a loop cursor.
322432  * @pos:	the &struct rhash_head to use as a loop cursor.
323
- * @head:	the previous &struct rhash_head to continue from
433
+ * @head:	the &struct rhash_head to start from
324434  * @tbl:	the &struct bucket_table
325435  * @hash:	the hash value / bucket index
326436  * @member:	name of the &struct rhash_head within the hashable struct.
327437  */
328
-#define rht_for_each_entry_continue(tpos, pos, head, tbl, hash, member)	\
329
-	for (pos = rht_dereference_bucket(head, tbl, hash);		\
438
+#define rht_for_each_entry_from(tpos, pos, head, tbl, hash, member)	\
439
+	for (pos = head;						\
330440 	     (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member);	\
331441 	     pos = rht_dereference_bucket((pos)->next, tbl, hash))
332442 
....@@ -339,8 +449,9 @@
339449  * @member:	name of the &struct rhash_head within the hashable struct.
340450  */
341451 #define rht_for_each_entry(tpos, pos, tbl, hash, member)		\
342
-	rht_for_each_entry_continue(tpos, pos, *rht_bucket(tbl, hash),	\
343
-				    tbl, hash, member)
452
+	rht_for_each_entry_from(tpos, pos,				\
453
+				rht_ptr(rht_bucket(tbl, hash), tbl, hash), \
454
+				tbl, hash, member)
344455 
345456 /**
346457  * rht_for_each_entry_safe - safely iterate over hash chain of given type
....@@ -355,7 +466,7 @@
355466  * remove the loop cursor from the list.
356467  */
357468 #define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member)	      \
358
-	for (pos = rht_dereference_bucket(*rht_bucket(tbl, hash), tbl, hash), \
469
+	for (pos = rht_ptr(rht_bucket(tbl, hash), tbl, hash),		      \
359470 	     next = !rht_is_a_nulls(pos) ?				      \
360471 		       rht_dereference_bucket(pos->next, tbl, hash) : NULL;   \
361472 	     (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member);	      \
....@@ -364,9 +475,9 @@
364475 		       rht_dereference_bucket(pos->next, tbl, hash) : NULL)
365476 
366477 /**
367
- * rht_for_each_rcu_continue - continue iterating over rcu hash chain
478
+ * rht_for_each_rcu_from - iterate over rcu hash chain from given head
368479  * @pos:	the &struct rhash_head to use as a loop cursor.
369
- * @head:	the previous &struct rhash_head to continue from
480
+ * @head:	the &struct rhash_head to start from
370481  * @tbl:	the &struct bucket_table
371482  * @hash:	the hash value / bucket index
372483  *
....@@ -374,9 +485,9 @@
374485  * the _rcu mutation primitives such as rhashtable_insert() as long as the
375486  * traversal is guarded by rcu_read_lock().
376487  */
377
-#define rht_for_each_rcu_continue(pos, head, tbl, hash)			\
488
+#define rht_for_each_rcu_from(pos, head, tbl, hash)			\
378489 	for (({barrier(); }),						\
379
-	     pos = rht_dereference_bucket_rcu(head, tbl, hash);		\
490
+	     pos = head;						\
380491 	     !rht_is_a_nulls(pos);					\
381492 	     pos = rcu_dereference_raw(pos->next))
382493 
....@@ -390,14 +501,17 @@
390501  * the _rcu mutation primitives such as rhashtable_insert() as long as the
391502  * traversal is guarded by rcu_read_lock().
392503  */
393
-#define rht_for_each_rcu(pos, tbl, hash)				\
394
-	rht_for_each_rcu_continue(pos, *rht_bucket(tbl, hash), tbl, hash)
504
+#define rht_for_each_rcu(pos, tbl, hash)			\
505
+	for (({barrier(); }),					\
506
+	     pos = rht_ptr_rcu(rht_bucket(tbl, hash));		\
507
+	     !rht_is_a_nulls(pos);				\
508
+	     pos = rcu_dereference_raw(pos->next))
395509 
396510 /**
397
- * rht_for_each_entry_rcu_continue - continue iterating over rcu hash chain
511
+ * rht_for_each_entry_rcu_from - iterated over rcu hash chain from given head
398512  * @tpos:	the type * to use as a loop cursor.
399513  * @pos:	the &struct rhash_head to use as a loop cursor.
400
- * @head:	the previous &struct rhash_head to continue from
514
+ * @head:	the &struct rhash_head to start from
401515  * @tbl:	the &struct bucket_table
402516  * @hash:	the hash value / bucket index
403517  * @member:	name of the &struct rhash_head within the hashable struct.
....@@ -406,9 +520,9 @@
406520  * the _rcu mutation primitives such as rhashtable_insert() as long as the
407521  * traversal is guarded by rcu_read_lock().
408522  */
409
-#define rht_for_each_entry_rcu_continue(tpos, pos, head, tbl, hash, member) \
523
+#define rht_for_each_entry_rcu_from(tpos, pos, head, tbl, hash, member) \
410524 	for (({barrier(); }),						    \
411
-	     pos = rht_dereference_bucket_rcu(head, tbl, hash);		    \
525
+	     pos = head;						    \
412526 	     (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member);	    \
413527 	     pos = rht_dereference_bucket_rcu(pos->next, tbl, hash))
414528 
....@@ -425,8 +539,9 @@
425539  * traversal is guarded by rcu_read_lock().
426540  */
427541 #define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member)		   \
428
-	rht_for_each_entry_rcu_continue(tpos, pos, *rht_bucket(tbl, hash), \
429
-					tbl, hash, member)
542
+	rht_for_each_entry_rcu_from(tpos, pos,				   \
543
+				    rht_ptr_rcu(rht_bucket(tbl, hash)),	   \
544
+				    tbl, hash, member)
430545 
431546 /**
432547  * rhl_for_each_rcu - iterate over rcu hash table list
....@@ -471,6 +586,7 @@
471586 		.ht = ht,
472587 		.key = key,
473588 	};
589
+	struct rhash_lock_head __rcu *const *bkt;
474590 	struct bucket_table *tbl;
475591 	struct rhash_head *he;
476592 	unsigned int hash;
....@@ -478,13 +594,19 @@
478594 	tbl = rht_dereference_rcu(ht->tbl, ht);
479595 restart:
480596 	hash = rht_key_hashfn(ht, tbl, key, params);
481
-	rht_for_each_rcu(he, tbl, hash) {
482
-		if (params.obj_cmpfn ?
483
-		    params.obj_cmpfn(&arg, rht_obj(ht, he)) :
484
-		    rhashtable_compare(&arg, rht_obj(ht, he)))
485
-			continue;
486
-		return he;
487
-	}
597
+	bkt = rht_bucket(tbl, hash);
598
+	do {
599
+		rht_for_each_rcu_from(he, rht_ptr_rcu(bkt), tbl, hash) {
600
+			if (params.obj_cmpfn ?
601
+			    params.obj_cmpfn(&arg, rht_obj(ht, he)) :
602
+			    rhashtable_compare(&arg, rht_obj(ht, he)))
603
+				continue;
604
+			return he;
605
+		}
606
+		/* An object might have been moved to a different hash chain,
607
+		 * while we walk along it - better check and retry.
608
+		 */
609
+	} while (he != RHT_NULLS_MARKER(bkt));
488610 
489611 	/* Ensure we see any new tables. */
490612 	smp_rmb();
....@@ -580,10 +702,10 @@
580702 		.ht = ht,
581703 		.key = key,
582704 	};
705
+	struct rhash_lock_head __rcu **bkt;
583706 	struct rhash_head __rcu **pprev;
584707 	struct bucket_table *tbl;
585708 	struct rhash_head *head;
586
-	spinlock_t *lock;
587709 	unsigned int hash;
588710 	int elasticity;
589711 	void *data;
....@@ -592,23 +714,22 @@
592714 
593715 	tbl = rht_dereference_rcu(ht->tbl, ht);
594716 	hash = rht_head_hashfn(ht, tbl, obj, params);
595
-	lock = rht_bucket_lock(tbl, hash);
596
-	spin_lock_bh(lock);
717
+	elasticity = RHT_ELASTICITY;
718
+	bkt = rht_bucket_insert(ht, tbl, hash);
719
+	data = ERR_PTR(-ENOMEM);
720
+	if (!bkt)
721
+		goto out;
722
+	pprev = NULL;
723
+	rht_lock(tbl, bkt);
597724 
598725 	if (unlikely(rcu_access_pointer(tbl->future_tbl))) {
599726 slow_path:
600
-		spin_unlock_bh(lock);
727
+		rht_unlock(tbl, bkt);
601728 		rcu_read_unlock();
602729 		return rhashtable_insert_slow(ht, key, obj);
603730 	}
604731 
605
-	elasticity = RHT_ELASTICITY;
606
-	pprev = rht_bucket_insert(ht, tbl, hash);
607
-	data = ERR_PTR(-ENOMEM);
608
-	if (!pprev)
609
-		goto out;
610
-
611
-	rht_for_each_continue(head, *pprev, tbl, hash) {
732
+	rht_for_each_from(head, rht_ptr(bkt, tbl, hash), tbl, hash) {
612733 		struct rhlist_head *plist;
613734 		struct rhlist_head *list;
614735 
....@@ -624,7 +745,7 @@
624745 		data = rht_obj(ht, head);
625746 
626747 		if (!rhlist)
627
-			goto out;
748
+			goto out_unlock;
628749 
629750 
630751 		list = container_of(obj, struct rhlist_head, rhead);
....@@ -633,9 +754,13 @@
633754 		RCU_INIT_POINTER(list->next, plist);
634755 		head = rht_dereference_bucket(head->next, tbl, hash);
635756 		RCU_INIT_POINTER(list->rhead.next, head);
636
-		rcu_assign_pointer(*pprev, obj);
637
-
638
-		goto good;
757
+		if (pprev) {
758
+			rcu_assign_pointer(*pprev, obj);
759
+			rht_unlock(tbl, bkt);
760
+		} else
761
+			rht_assign_unlock(tbl, bkt, obj);
762
+		data = NULL;
763
+		goto out;
639764 	}
640765 
641766 	if (elasticity <= 0)
....@@ -643,12 +768,13 @@
643768 
644769 	data = ERR_PTR(-E2BIG);
645770 	if (unlikely(rht_grow_above_max(ht, tbl)))
646
-		goto out;
771
+		goto out_unlock;
647772 
648773 	if (unlikely(rht_grow_above_100(ht, tbl)))
649774 		goto slow_path;
650775 
651
-	head = rht_dereference_bucket(*pprev, tbl, hash);
776
+	/* Inserting at head of list makes unlocking free. */
777
+	head = rht_ptr(bkt, tbl, hash);
652778 
653779 	RCU_INIT_POINTER(obj->next, head);
654780 	if (rhlist) {
....@@ -658,20 +784,21 @@
658784 		RCU_INIT_POINTER(list->next, NULL);
659785 	}
660786 
661
-	rcu_assign_pointer(*pprev, obj);
662
-
663787 	atomic_inc(&ht->nelems);
788
+	rht_assign_unlock(tbl, bkt, obj);
789
+
664790 	if (rht_grow_above_75(ht, tbl))
665791 		schedule_work(&ht->run_work);
666792 
667
-good:
668793 	data = NULL;
669
-
670794 out:
671
-	spin_unlock_bh(lock);
672795 	rcu_read_unlock();
673796 
674797 	return data;
798
+
799
+out_unlock:
800
+	rht_unlock(tbl, bkt);
801
+	goto out;
675802 }
676803 
677804 /**
....@@ -680,9 +807,9 @@
680807  * @obj:	pointer to hash head inside object
681808  * @params:	hash table parameters
682809  *
683
- * Will take a per bucket spinlock to protect against mutual mutations
810
+ * Will take the per bucket bitlock to protect against mutual mutations
684811  * on the same bucket. Multiple insertions may occur in parallel unless
685
- * they map to the same bucket lock.
812
+ * they map to the same bucket.
686813  *
687814  * It is safe to call this function from atomic context.
688815  *
....@@ -709,9 +836,9 @@
709836  * @list:	pointer to hash list head inside object
710837  * @params:	hash table parameters
711838  *
712
- * Will take a per bucket spinlock to protect against mutual mutations
839
+ * Will take the per bucket bitlock to protect against mutual mutations
713840  * on the same bucket. Multiple insertions may occur in parallel unless
714
- * they map to the same bucket lock.
841
+ * they map to the same bucket.
715842  *
716843  * It is safe to call this function from atomic context.
717844  *
....@@ -732,9 +859,9 @@
732859  * @list:	pointer to hash list head inside object
733860  * @params:	hash table parameters
734861  *
735
- * Will take a per bucket spinlock to protect against mutual mutations
862
+ * Will take the per bucket bitlock to protect against mutual mutations
736863  * on the same bucket. Multiple insertions may occur in parallel unless
737
- * they map to the same bucket lock.
864
+ * they map to the same bucket.
738865  *
739866  * It is safe to call this function from atomic context.
740867  *
....@@ -757,12 +884,6 @@
757884  * @ht:		hash table
758885  * @obj:	pointer to hash head inside object
759886  * @params:	hash table parameters
760
- *
761
- * Locks down the bucket chain in both the old and new table if a resize
762
- * is in progress to ensure that writers can't remove from the old table
763
- * and can't insert to the new table during the atomic operation of search
764
- * and insertion. Searches for duplicates in both the old and new table if
765
- * a resize is in progress.
766887  *
767888  * This lookup function may only be used for fixed key hash table (key_len
768889  * parameter set). It will BUG() if used inappropriately.
....@@ -819,12 +940,6 @@
819940  * @obj:	pointer to hash head inside object
820941  * @params:	hash table parameters
821942  *
822
- * Locks down the bucket chain in both the old and new table if a resize
823
- * is in progress to ensure that writers can't remove from the old table
824
- * and can't insert to the new table during the atomic operation of search
825
- * and insertion. Searches for duplicates in both the old and new table if
826
- * a resize is in progress.
827
- *
828943  * Lookups may occur in parallel with hashtable mutations and resizing.
829944  *
830945  * Will trigger an automatic deferred table resizing if residency in the
....@@ -850,9 +965,9 @@
850965 /**
851966  * rhashtable_lookup_get_insert_key - lookup and insert object into hash table
852967  * @ht:		hash table
968
+ * @key:	key
853969  * @obj:	pointer to hash head inside object
854970  * @params:	hash table parameters
855
- * @data:	pointer to element data already in hashes
856971  *
857972  * Just like rhashtable_lookup_insert_key(), but this function returns the
858973  * object if it exists, NULL if it does not and the insertion was successful,
....@@ -873,19 +988,20 @@
873988 	struct rhash_head *obj, const struct rhashtable_params params,
874989 	bool rhlist)
875990 {
991
+	struct rhash_lock_head __rcu **bkt;
876992 	struct rhash_head __rcu **pprev;
877993 	struct rhash_head *he;
878
-	spinlock_t * lock;
879994 	unsigned int hash;
880995 	int err = -ENOENT;
881996 
882997 	hash = rht_head_hashfn(ht, tbl, obj, params);
883
-	lock = rht_bucket_lock(tbl, hash);
998
+	bkt = rht_bucket_var(tbl, hash);
999
+	if (!bkt)
1000
+		return -ENOENT;
1001
+	pprev = NULL;
1002
+	rht_lock(tbl, bkt);
8841003 
885
-	spin_lock_bh(lock);
886
-
887
-	pprev = rht_bucket_var(tbl, hash);
888
-	rht_for_each_continue(he, *pprev, tbl, hash) {
1004
+	rht_for_each_from(he, rht_ptr(bkt, tbl, hash), tbl, hash) {
8891005 		struct rhlist_head *list;
8901006 
8911007 		list = container_of(he, struct rhlist_head, rhead);
....@@ -925,12 +1041,17 @@
9251041 			}
9261042 		}
9271043 
928
-		rcu_assign_pointer(*pprev, obj);
929
-		break;
1044
+		if (pprev) {
1045
+			rcu_assign_pointer(*pprev, obj);
1046
+			rht_unlock(tbl, bkt);
1047
+		} else {
1048
+			rht_assign_unlock(tbl, bkt, obj);
1049
+		}
1050
+		goto unlocked;
9301051 	}
9311052 
932
-	spin_unlock_bh(lock);
933
-
1053
+	rht_unlock(tbl, bkt);
1054
+unlocked:
9341055 	if (err > 0) {
9351056 		atomic_dec(&ht->nelems);
9361057 		if (unlikely(ht->p.automatic_shrinking &&
....@@ -1019,9 +1140,9 @@
10191140 	struct rhash_head *obj_old, struct rhash_head *obj_new,
10201141 	const struct rhashtable_params params)
10211142 {
1143
+	struct rhash_lock_head __rcu **bkt;
10221144 	struct rhash_head __rcu **pprev;
10231145 	struct rhash_head *he;
1024
-	spinlock_t *lock;
10251146 	unsigned int hash;
10261147 	int err = -ENOENT;
10271148 
....@@ -1032,25 +1153,33 @@
10321153 	if (hash != rht_head_hashfn(ht, tbl, obj_new, params))
10331154 		return -EINVAL;
10341155 
1035
-	lock = rht_bucket_lock(tbl, hash);
1156
+	bkt = rht_bucket_var(tbl, hash);
1157
+	if (!bkt)
1158
+		return -ENOENT;
10361159 
1037
-	spin_lock_bh(lock);
1160
+	pprev = NULL;
1161
+	rht_lock(tbl, bkt);
10381162 
1039
-	pprev = rht_bucket_var(tbl, hash);
1040
-	rht_for_each_continue(he, *pprev, tbl, hash) {
1163
+	rht_for_each_from(he, rht_ptr(bkt, tbl, hash), tbl, hash) {
10411164 		if (he != obj_old) {
10421165 			pprev = &he->next;
10431166 			continue;
10441167 		}
10451168 
10461169 		rcu_assign_pointer(obj_new->next, obj_old->next);
1047
-		rcu_assign_pointer(*pprev, obj_new);
1170
+		if (pprev) {
1171
+			rcu_assign_pointer(*pprev, obj_new);
1172
+			rht_unlock(tbl, bkt);
1173
+		} else {
1174
+			rht_assign_unlock(tbl, bkt, obj_new);
1175
+		}
10481176 		err = 0;
1049
-		break;
1177
+		goto unlocked;
10501178 	}
10511179 
1052
-	spin_unlock_bh(lock);
1180
+	rht_unlock(tbl, bkt);
10531181 
1182
+unlocked:
10541183 	return err;
10551184 }
10561185 
....@@ -1093,14 +1222,6 @@
10931222 	rcu_read_unlock();
10941223 
10951224 	return err;
1096
-}
1097
-
1098
-/* Obsolete function, do not use in new code. */
1099
-static inline int rhashtable_walk_init(struct rhashtable *ht,
1100
-				       struct rhashtable_iter *iter, gfp_t gfp)
1101
-{
1102
-	rhashtable_walk_enter(ht, iter);
1103
-	return 0;
11041225 }
11051226 
11061227 /**