forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/drivers/infiniband/sw/rxe/rxe_pool.c
....@@ -1,34 +1,7 @@
1
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
12 /*
23  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
34  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4
- *
5
- * This software is available to you under a choice of one of two
6
- * licenses.  You may choose to be licensed under the terms of the GNU
7
- * General Public License (GPL) Version 2, available from the file
8
- * COPYING in the main directory of this source tree, or the
9
- * OpenIB.org BSD license below:
10
- *
11
- *	   Redistribution and use in source and binary forms, with or
12
- *	   without modification, are permitted provided that the following
13
- *	   conditions are met:
14
- *
15
- *		- Redistributions of source code must retain the above
16
- *		  copyright notice, this list of conditions and the following
17
- *		  disclaimer.
18
- *
19
- *		- Redistributions in binary form must reproduce the above
20
- *		  copyright notice, this list of conditions and the following
21
- *		  disclaimer in the documentation and/or other materials
22
- *		  provided with the distribution.
23
- *
24
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31
- * SOFTWARE.
325  */
336 
347 #include "rxe.h"
....@@ -42,20 +15,22 @@
4215 	[RXE_TYPE_UC] = {
4316 		.name		= "rxe-uc",
4417 		.size		= sizeof(struct rxe_ucontext),
18
+		.flags          = RXE_POOL_NO_ALLOC,
4519 	},
4620 	[RXE_TYPE_PD] = {
4721 		.name		= "rxe-pd",
4822 		.size		= sizeof(struct rxe_pd),
23
+		.flags		= RXE_POOL_NO_ALLOC,
4924 	},
5025 	[RXE_TYPE_AH] = {
5126 		.name		= "rxe-ah",
5227 		.size		= sizeof(struct rxe_ah),
53
-		.flags		= RXE_POOL_ATOMIC,
28
+		.flags		= RXE_POOL_ATOMIC | RXE_POOL_NO_ALLOC,
5429 	},
5530 	[RXE_TYPE_SRQ] = {
5631 		.name		= "rxe-srq",
5732 		.size		= sizeof(struct rxe_srq),
58
-		.flags		= RXE_POOL_INDEX,
33
+		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
5934 		.min_index	= RXE_MIN_SRQ_INDEX,
6035 		.max_index	= RXE_MAX_SRQ_INDEX,
6136 	},
....@@ -70,6 +45,7 @@
7045 	[RXE_TYPE_CQ] = {
7146 		.name		= "rxe-cq",
7247 		.size		= sizeof(struct rxe_cq),
48
+		.flags          = RXE_POOL_NO_ALLOC,
7349 		.cleanup	= rxe_cq_cleanup,
7450 	},
7551 	[RXE_TYPE_MR] = {
....@@ -105,57 +81,6 @@
10581 static inline const char *pool_name(struct rxe_pool *pool)
10682 {
10783 	return rxe_type_info[pool->type].name;
108
-}
109
-
110
-static inline struct kmem_cache *pool_cache(struct rxe_pool *pool)
111
-{
112
-	return rxe_type_info[pool->type].cache;
113
-}
114
-
115
-static void rxe_cache_clean(size_t cnt)
116
-{
117
-	int i;
118
-	struct rxe_type_info *type;
119
-
120
-	for (i = 0; i < cnt; i++) {
121
-		type = &rxe_type_info[i];
122
-		kmem_cache_destroy(type->cache);
123
-		type->cache = NULL;
124
-	}
125
-}
126
-
127
-int rxe_cache_init(void)
128
-{
129
-	int err;
130
-	int i;
131
-	size_t size;
132
-	struct rxe_type_info *type;
133
-
134
-	for (i = 0; i < RXE_NUM_TYPES; i++) {
135
-		type = &rxe_type_info[i];
136
-		size = ALIGN(type->size, RXE_POOL_ALIGN);
137
-		type->cache = kmem_cache_create(type->name, size,
138
-				RXE_POOL_ALIGN,
139
-				RXE_POOL_CACHE_FLAGS, NULL);
140
-		if (!type->cache) {
141
-			pr_err("Unable to init kmem cache for %s\n",
142
-			       type->name);
143
-			err = -ENOMEM;
144
-			goto err1;
145
-		}
146
-	}
147
-
148
-	return 0;
149
-
150
-err1:
151
-	rxe_cache_clean(i);
152
-
153
-	return err;
154
-}
155
-
156
-void rxe_cache_exit(void)
157
-{
158
-	rxe_cache_clean(RXE_NUM_TYPES);
15984 }
16085 
16186 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
....@@ -209,7 +134,7 @@
209134 
210135 	kref_init(&pool->ref_cnt);
211136 
212
-	spin_lock_init(&pool->pool_lock);
137
+	rwlock_init(&pool->pool_lock);
213138 
214139 	if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
215140 		err = rxe_pool_init_index(pool,
....@@ -224,7 +149,7 @@
224149 		pool->key_size = rxe_type_info[type].key_size;
225150 	}
226151 
227
-	pool->state = rxe_pool_valid;
152
+	pool->state = RXE_POOL_STATE_VALID;
228153 
229154 out:
230155 	return err;
....@@ -234,7 +159,7 @@
234159 {
235160 	struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
236161 
237
-	pool->state = rxe_pool_invalid;
162
+	pool->state = RXE_POOL_STATE_INVALID;
238163 	kfree(pool->table);
239164 }
240165 
....@@ -243,20 +168,18 @@
243168 	kref_put(&pool->ref_cnt, rxe_pool_release);
244169 }
245170 
246
-int rxe_pool_cleanup(struct rxe_pool *pool)
171
+void rxe_pool_cleanup(struct rxe_pool *pool)
247172 {
248173 	unsigned long flags;
249174 
250
-	spin_lock_irqsave(&pool->pool_lock, flags);
251
-	pool->state = rxe_pool_invalid;
175
+	write_lock_irqsave(&pool->pool_lock, flags);
176
+	pool->state = RXE_POOL_STATE_INVALID;
252177 	if (atomic_read(&pool->num_elem) > 0)
253178 		pr_warn("%s pool destroyed with unfree'd elem\n",
254179 			pool_name(pool));
255
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
180
+	write_unlock_irqrestore(&pool->pool_lock, flags);
256181 
257182 	rxe_pool_put(pool);
258
-
259
-	return 0;
260183 }
261184 
262185 static u32 alloc_index(struct rxe_pool *pool)
....@@ -338,10 +261,10 @@
338261 	struct rxe_pool *pool = elem->pool;
339262 	unsigned long flags;
340263 
341
-	spin_lock_irqsave(&pool->pool_lock, flags);
264
+	write_lock_irqsave(&pool->pool_lock, flags);
342265 	memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
343266 	insert_key(pool, elem);
344
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
267
+	write_unlock_irqrestore(&pool->pool_lock, flags);
345268 }
346269 
347270 void rxe_drop_key(void *arg)
....@@ -350,9 +273,9 @@
350273 	struct rxe_pool *pool = elem->pool;
351274 	unsigned long flags;
352275 
353
-	spin_lock_irqsave(&pool->pool_lock, flags);
276
+	write_lock_irqsave(&pool->pool_lock, flags);
354277 	rb_erase(&elem->node, &pool->tree);
355
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
278
+	write_unlock_irqrestore(&pool->pool_lock, flags);
356279 }
357280 
358281 void rxe_add_index(void *arg)
....@@ -361,10 +284,10 @@
361284 	struct rxe_pool *pool = elem->pool;
362285 	unsigned long flags;
363286 
364
-	spin_lock_irqsave(&pool->pool_lock, flags);
287
+	write_lock_irqsave(&pool->pool_lock, flags);
365288 	elem->index = alloc_index(pool);
366289 	insert_index(pool, elem);
367
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
290
+	write_unlock_irqrestore(&pool->pool_lock, flags);
368291 }
369292 
370293 void rxe_drop_index(void *arg)
....@@ -373,10 +296,10 @@
373296 	struct rxe_pool *pool = elem->pool;
374297 	unsigned long flags;
375298 
376
-	spin_lock_irqsave(&pool->pool_lock, flags);
299
+	write_lock_irqsave(&pool->pool_lock, flags);
377300 	clear_bit(elem->index - pool->min_index, pool->table);
378301 	rb_erase(&elem->node, &pool->tree);
379
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
302
+	write_unlock_irqrestore(&pool->pool_lock, flags);
380303 }
381304 
382305 void *rxe_alloc(struct rxe_pool *pool)
....@@ -386,35 +309,70 @@
386309 
387310 	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
388311 
389
-	spin_lock_irqsave(&pool->pool_lock, flags);
390
-	if (pool->state != rxe_pool_valid) {
391
-		spin_unlock_irqrestore(&pool->pool_lock, flags);
312
+	read_lock_irqsave(&pool->pool_lock, flags);
313
+	if (pool->state != RXE_POOL_STATE_VALID) {
314
+		read_unlock_irqrestore(&pool->pool_lock, flags);
392315 		return NULL;
393316 	}
394317 	kref_get(&pool->ref_cnt);
395
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
318
+	read_unlock_irqrestore(&pool->pool_lock, flags);
396319 
397
-	kref_get(&pool->rxe->ref_cnt);
398
-
399
-	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
320
+	if (!ib_device_try_get(&pool->rxe->ib_dev))
400321 		goto out_put_pool;
401322 
402
-	elem = kmem_cache_zalloc(pool_cache(pool),
323
+	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
324
+		goto out_cnt;
325
+
326
+	elem = kzalloc(rxe_type_info[pool->type].size,
403327 				 (pool->flags & RXE_POOL_ATOMIC) ?
404328 				 GFP_ATOMIC : GFP_KERNEL);
405329 	if (!elem)
406
-		goto out_put_pool;
330
+		goto out_cnt;
407331 
408332 	elem->pool = pool;
409333 	kref_init(&elem->ref_cnt);
410334 
411335 	return elem;
412336 
413
-out_put_pool:
337
+out_cnt:
414338 	atomic_dec(&pool->num_elem);
415
-	rxe_dev_put(pool->rxe);
339
+	ib_device_put(&pool->rxe->ib_dev);
340
+out_put_pool:
416341 	rxe_pool_put(pool);
417342 	return NULL;
343
+}
344
+
345
+int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
346
+{
347
+	unsigned long flags;
348
+
349
+	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
350
+
351
+	read_lock_irqsave(&pool->pool_lock, flags);
352
+	if (pool->state != RXE_POOL_STATE_VALID) {
353
+		read_unlock_irqrestore(&pool->pool_lock, flags);
354
+		return -EINVAL;
355
+	}
356
+	kref_get(&pool->ref_cnt);
357
+	read_unlock_irqrestore(&pool->pool_lock, flags);
358
+
359
+	if (!ib_device_try_get(&pool->rxe->ib_dev))
360
+		goto out_put_pool;
361
+
362
+	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
363
+		goto out_cnt;
364
+
365
+	elem->pool = pool;
366
+	kref_init(&elem->ref_cnt);
367
+
368
+	return 0;
369
+
370
+out_cnt:
371
+	atomic_dec(&pool->num_elem);
372
+	ib_device_put(&pool->rxe->ib_dev);
373
+out_put_pool:
374
+	rxe_pool_put(pool);
375
+	return -EINVAL;
418376 }
419377 
420378 void rxe_elem_release(struct kref *kref)
....@@ -426,9 +384,10 @@
426384 	if (pool->cleanup)
427385 		pool->cleanup(elem);
428386 
429
-	kmem_cache_free(pool_cache(pool), elem);
387
+	if (!(pool->flags & RXE_POOL_NO_ALLOC))
388
+		kfree(elem);
430389 	atomic_dec(&pool->num_elem);
431
-	rxe_dev_put(pool->rxe);
390
+	ib_device_put(&pool->rxe->ib_dev);
432391 	rxe_pool_put(pool);
433392 }
434393 
....@@ -438,9 +397,9 @@
438397 	struct rxe_pool_entry *elem = NULL;
439398 	unsigned long flags;
440399 
441
-	spin_lock_irqsave(&pool->pool_lock, flags);
400
+	read_lock_irqsave(&pool->pool_lock, flags);
442401 
443
-	if (pool->state != rxe_pool_valid)
402
+	if (pool->state != RXE_POOL_STATE_VALID)
444403 		goto out;
445404 
446405 	node = pool->tree.rb_node;
....@@ -452,15 +411,14 @@
452411 			node = node->rb_left;
453412 		else if (elem->index < index)
454413 			node = node->rb_right;
455
-		else
414
+		else {
415
+			kref_get(&elem->ref_cnt);
456416 			break;
417
+		}
457418 	}
458419 
459
-	if (node)
460
-		kref_get(&elem->ref_cnt);
461
-
462420 out:
463
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
421
+	read_unlock_irqrestore(&pool->pool_lock, flags);
464422 	return node ? elem : NULL;
465423 }
466424 
....@@ -471,9 +429,9 @@
471429 	int cmp;
472430 	unsigned long flags;
473431 
474
-	spin_lock_irqsave(&pool->pool_lock, flags);
432
+	read_lock_irqsave(&pool->pool_lock, flags);
475433 
476
-	if (pool->state != rxe_pool_valid)
434
+	if (pool->state != RXE_POOL_STATE_VALID)
477435 		goto out;
478436 
479437 	node = pool->tree.rb_node;
....@@ -496,6 +454,6 @@
496454 		kref_get(&elem->ref_cnt);
497455 
498456 out:
499
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
457
+	read_unlock_irqrestore(&pool->pool_lock, flags);
500458 	return node ? elem : NULL;
501459 }