change system file

hc

2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771

 kernel/drivers/infiniband/core/cq.c
..
..
@@ -1,19 +1,17 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  * Copyright (c) 2015 HGST, a Western Digital Company.
3
- *
4
- * This program is free software; you can redistribute it and/or modify it
5
- * under the terms and conditions of the GNU General Public License,
6
- * version 2, as published by the Free Software Foundation.
7
- *
8
- * This program is distributed in the hope it will be useful, but WITHOUT
9
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11
- * more details.
12
4
  */
13
5
 #include <linux/module.h>
14
6
 #include <linux/err.h>
15
7
 #include <linux/slab.h>
16
8
 #include <rdma/ib_verbs.h>
9
+
10
+#include "core_priv.h"
11
+
12
+#include <trace/events/rdma_core.h>
13
+/* Max size for shared CQ, may require tuning */
14
+#define IB_MAX_SHARED_CQ_SZ		4096U
17
15
 
18
16
 /* # of WCs to poll for with a single call to ib_poll_cq */
19
17
 #define IB_POLL_BATCH			16
..
..
@@ -26,18 +24,86 @@
26
24
 #define IB_POLL_FLAGS \
27
25
 	(IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
28
26
 
27
+static const struct dim_cq_moder
28
+rdma_dim_prof[RDMA_DIM_PARAMS_NUM_PROFILES] = {
29
+	{1,   0, 1,  0},
30
+	{1,   0, 4,  0},
31
+	{2,   0, 4,  0},
32
+	{2,   0, 8,  0},
33
+	{4,   0, 8,  0},
34
+	{16,  0, 8,  0},
35
+	{16,  0, 16, 0},
36
+	{32,  0, 16, 0},
37
+	{32,  0, 32, 0},
38
+};
39
+
40
+static void ib_cq_rdma_dim_work(struct work_struct *w)
41
+{
42
+	struct dim *dim = container_of(w, struct dim, work);
43
+	struct ib_cq *cq = dim->priv;
44
+
45
+	u16 usec = rdma_dim_prof[dim->profile_ix].usec;
46
+	u16 comps = rdma_dim_prof[dim->profile_ix].comps;
47
+
48
+	dim->state = DIM_START_MEASURE;
49
+
50
+	trace_cq_modify(cq, comps, usec);
51
+	cq->device->ops.modify_cq(cq, comps, usec);
52
+}
53
+
54
+static void rdma_dim_init(struct ib_cq *cq)
55
+{
56
+	struct dim *dim;
57
+
58
+	if (!cq->device->ops.modify_cq || !cq->device->use_cq_dim ||
59
+	    cq->poll_ctx == IB_POLL_DIRECT)
60
+		return;
61
+
62
+	dim = kzalloc(sizeof(struct dim), GFP_KERNEL);
63
+	if (!dim)
64
+		return;
65
+
66
+	dim->state = DIM_START_MEASURE;
67
+	dim->tune_state = DIM_GOING_RIGHT;
68
+	dim->profile_ix = RDMA_DIM_START_PROFILE;
69
+	dim->priv = cq;
70
+	cq->dim = dim;
71
+
72
+	INIT_WORK(&dim->work, ib_cq_rdma_dim_work);
73
+}
74
+
75
+static void rdma_dim_destroy(struct ib_cq *cq)
76
+{
77
+	if (!cq->dim)
78
+		return;
79
+
80
+	cancel_work_sync(&cq->dim->work);
81
+	kfree(cq->dim);
82
+}
83
+
84
+static int __poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
85
+{
86
+	int rc;
87
+
88
+	rc = ib_poll_cq(cq, num_entries, wc);
89
+	trace_cq_poll(cq, num_entries, rc);
90
+	return rc;
91
+}
92
+
29
93
 static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,
30
94
 			   int batch)
31
95
 {
32
96
 	int i, n, completed = 0;
97
+
98
+	trace_cq_process(cq);
33
99
 
34
100
 	/*
35
101
 	 * budget might be (-1) if the caller does not
36
102
 	 * want to bound this call, thus we need unsigned
37
103
 	 * minimum here.
38
104
 	 */
39
-	while ((n = ib_poll_cq(cq, min_t(u32, batch,
40
-					 budget - completed), wcs)) > 0) {
105
+	while ((n = __poll_cq(cq, min_t(u32, batch,
106
+					budget - completed), wcs)) > 0) {
41
107
 		for (i = 0; i < n; i++) {
42
108
 			struct ib_wc *wc = &wcs[i];
43
109
 
..
..
@@ -86,20 +152,27 @@
86
152
 static int ib_poll_handler(struct irq_poll *iop, int budget)
87
153
 {
88
154
 	struct ib_cq *cq = container_of(iop, struct ib_cq, iop);
155
+	struct dim *dim = cq->dim;
89
156
 	int completed;
90
157
 
91
158
 	completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);
92
159
 	if (completed < budget) {
93
160
 		irq_poll_complete(&cq->iop);
94
-		if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
161
+		if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) {
162
+			trace_cq_reschedule(cq);
95
163
 			irq_poll_sched(&cq->iop);
164
+		}
96
165
 	}
166
+
167
+	if (dim)
168
+		rdma_dim(dim, completed);
97
169
 
98
170
 	return completed;
99
171
 }
100
172
 
101
173
 static void ib_cq_completion_softirq(struct ib_cq *cq, void *private)
102
174
 {
175
+	trace_cq_schedule(cq);
103
176
 	irq_poll_sched(&cq->iop);
104
177
 }
105
178
 
..
..
@@ -113,15 +186,18 @@
113
186
 	if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
114
187
 	    ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
115
188
 		queue_work(cq->comp_wq, &cq->work);
189
+	else if (cq->dim)
190
+		rdma_dim(cq->dim, completed);
116
191
 }
117
192
 
118
193
 static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
119
194
 {
195
+	trace_cq_schedule(cq);
120
196
 	queue_work(cq->comp_wq, &cq->work);
121
197
 }
122
198
 
123
199
 /**
124
- * __ib_alloc_cq - allocate a completion queue
200
+ * __ib_alloc_cq        allocate a completion queue
125
201
  * @dev:		device to allocate the CQ for
126
202
  * @private:		driver private data, accessible from cq->cq_context
127
203
  * @nr_cqe:		number of CQEs to allocate
..
..
@@ -134,9 +210,9 @@
134
210
  * specified context. The ULP must use wr->wr_cqe instead of wr->wr_id
135
211
  * to use this CQ abstraction.
136
212
  */
137
-struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
138
-			    int nr_cqe, int comp_vector,
139
-			    enum ib_poll_context poll_ctx, const char *caller)
213
+struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
214
+			    int comp_vector, enum ib_poll_context poll_ctx,
215
+			    const char *caller)
140
216
 {
141
217
 	struct ib_cq_init_attr cq_attr = {
142
218
 		.cqe		= nr_cqe,
..
..
@@ -145,24 +221,28 @@
145
221
 	struct ib_cq *cq;
146
222
 	int ret = -ENOMEM;
147
223
 
148
-	cq = dev->create_cq(dev, &cq_attr, NULL, NULL);
149
-	if (IS_ERR(cq))
150
-		return cq;
224
+	cq = rdma_zalloc_drv_obj(dev, ib_cq);
225
+	if (!cq)
226
+		return ERR_PTR(ret);
151
227
 
152
228
 	cq->device = dev;
153
-	cq->uobject = NULL;
154
-	cq->event_handler = NULL;
155
229
 	cq->cq_context = private;
156
230
 	cq->poll_ctx = poll_ctx;
157
231
 	atomic_set(&cq->usecnt, 0);
232
+	cq->comp_vector = comp_vector;
158
233
 
159
234
 	cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL);
160
235
 	if (!cq->wc)
161
-		goto out_destroy_cq;
236
+		goto out_free_cq;
162
237
 
163
-	cq->res.type = RDMA_RESTRACK_CQ;
164
-	cq->res.kern_name = caller;
165
-	rdma_restrack_add(&cq->res);
238
+	rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
239
+	rdma_restrack_set_name(&cq->res, caller);
240
+
241
+	ret = dev->ops.create_cq(cq, &cq_attr, NULL);
242
+	if (ret)
243
+		goto out_free_wc;
244
+
245
+	rdma_dim_init(cq);
166
246
 
167
247
 	switch (cq->poll_ctx) {
168
248
 	case IB_POLL_DIRECT:
..
..
@@ -184,19 +264,53 @@
184
264
 		break;
185
265
 	default:
186
266
 		ret = -EINVAL;
187
-		goto out_free_wc;
267
+		goto out_destroy_cq;
188
268
 	}
189
269
 
270
+	rdma_restrack_add(&cq->res);
271
+	trace_cq_alloc(cq, nr_cqe, comp_vector, poll_ctx);
190
272
 	return cq;
191
273
 
192
-out_free_wc:
193
-	kfree(cq->wc);
194
-	rdma_restrack_del(&cq->res);
195
274
 out_destroy_cq:
196
-	cq->device->destroy_cq(cq);
275
+	rdma_dim_destroy(cq);
276
+	cq->device->ops.destroy_cq(cq, NULL);
277
+out_free_wc:
278
+	rdma_restrack_put(&cq->res);
279
+	kfree(cq->wc);
280
+out_free_cq:
281
+	kfree(cq);
282
+	trace_cq_alloc_error(nr_cqe, comp_vector, poll_ctx, ret);
197
283
 	return ERR_PTR(ret);
198
284
 }
199
285
 EXPORT_SYMBOL(__ib_alloc_cq);
286
+
287
+/**
288
+ * __ib_alloc_cq_any - allocate a completion queue
289
+ * @dev:		device to allocate the CQ for
290
+ * @private:		driver private data, accessible from cq->cq_context
291
+ * @nr_cqe:		number of CQEs to allocate
292
+ * @poll_ctx:		context to poll the CQ from
293
+ * @caller:		module owner name
294
+ *
295
+ * Attempt to spread ULP Completion Queues over each device's interrupt
296
+ * vectors. A simple best-effort mechanism is used.
297
+ */
298
+struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,
299
+				int nr_cqe, enum ib_poll_context poll_ctx,
300
+				const char *caller)
301
+{
302
+	static atomic_t counter;
303
+	int comp_vector = 0;
304
+
305
+	if (dev->num_comp_vectors > 1)
306
+		comp_vector =
307
+			atomic_inc_return(&counter) %
308
+			min_t(int, dev->num_comp_vectors, num_online_cpus());
309
+
310
+	return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,
311
+			     caller);
312
+}
313
+EXPORT_SYMBOL(__ib_alloc_cq_any);
200
314
 
201
315
 /**
202
316
  * ib_free_cq - free a completion queue
..
..
@@ -207,6 +321,8 @@
207
321
 	int ret;
208
322
 
209
323
 	if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
324
+		return;
325
+	if (WARN_ON_ONCE(cq->cqe_used))
210
326
 		return;
211
327
 
212
328
 	switch (cq->poll_ctx) {
..
..
@@ -223,9 +339,178 @@
223
339
 		WARN_ON_ONCE(1);
224
340
 	}
225
341
 
226
-	kfree(cq->wc);
342
+	rdma_dim_destroy(cq);
343
+	trace_cq_free(cq);
344
+	ret = cq->device->ops.destroy_cq(cq, NULL);
345
+	WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");
227
346
 	rdma_restrack_del(&cq->res);
228
-	ret = cq->device->destroy_cq(cq);
229
-	WARN_ON_ONCE(ret);
347
+	kfree(cq->wc);
348
+	kfree(cq);
230
349
 }
231
350
 EXPORT_SYMBOL(ib_free_cq);
351
+
352
+void ib_cq_pool_init(struct ib_device *dev)
353
+{
354
+	unsigned int i;
355
+
356
+	spin_lock_init(&dev->cq_pools_lock);
357
+	for (i = 0; i < ARRAY_SIZE(dev->cq_pools); i++)
358
+		INIT_LIST_HEAD(&dev->cq_pools[i]);
359
+}
360
+
361
+void ib_cq_pool_destroy(struct ib_device *dev)
362
+{
363
+	struct ib_cq *cq, *n;
364
+	unsigned int i;
365
+
366
+	for (i = 0; i < ARRAY_SIZE(dev->cq_pools); i++) {
367
+		list_for_each_entry_safe(cq, n, &dev->cq_pools[i],
368
+					 pool_entry) {
369
+			WARN_ON(cq->cqe_used);
370
+			cq->shared = false;
371
+			ib_free_cq(cq);
372
+		}
373
+	}
374
+}
375
+
376
+static int ib_alloc_cqs(struct ib_device *dev, unsigned int nr_cqes,
377
+			enum ib_poll_context poll_ctx)
378
+{
379
+	LIST_HEAD(tmp_list);
380
+	unsigned int nr_cqs, i;
381
+	struct ib_cq *cq, *n;
382
+	int ret;
383
+
384
+	if (poll_ctx > IB_POLL_LAST_POOL_TYPE) {
385
+		WARN_ON_ONCE(poll_ctx > IB_POLL_LAST_POOL_TYPE);
386
+		return -EINVAL;
387
+	}
388
+
389
+	/*
390
+	 * Allocate at least as many CQEs as requested, and otherwise
391
+	 * a reasonable batch size so that we can share CQs between
392
+	 * multiple users instead of allocating a larger number of CQs.
393
+	 */
394
+	nr_cqes = min_t(unsigned int, dev->attrs.max_cqe,
395
+			max(nr_cqes, IB_MAX_SHARED_CQ_SZ));
396
+	nr_cqs = min_t(unsigned int, dev->num_comp_vectors, num_online_cpus());
397
+	for (i = 0; i < nr_cqs; i++) {
398
+		cq = ib_alloc_cq(dev, NULL, nr_cqes, i, poll_ctx);
399
+		if (IS_ERR(cq)) {
400
+			ret = PTR_ERR(cq);
401
+			goto out_free_cqs;
402
+		}
403
+		cq->shared = true;
404
+		list_add_tail(&cq->pool_entry, &tmp_list);
405
+	}
406
+
407
+	spin_lock_irq(&dev->cq_pools_lock);
408
+	list_splice(&tmp_list, &dev->cq_pools[poll_ctx]);
409
+	spin_unlock_irq(&dev->cq_pools_lock);
410
+
411
+	return 0;
412
+
413
+out_free_cqs:
414
+	list_for_each_entry_safe(cq, n, &tmp_list, pool_entry) {
415
+		cq->shared = false;
416
+		ib_free_cq(cq);
417
+	}
418
+	return ret;
419
+}
420
+
421
+/**
422
+ * ib_cq_pool_get() - Find the least used completion queue that matches
423
+ *   a given cpu hint (or least used for wild card affinity) and fits
424
+ *   nr_cqe.
425
+ * @dev: rdma device
426
+ * @nr_cqe: number of needed cqe entries
427
+ * @comp_vector_hint: completion vector hint (-1) for the driver to assign
428
+ *   a comp vector based on internal counter
429
+ * @poll_ctx: cq polling context
430
+ *
431
+ * Finds a cq that satisfies @comp_vector_hint and @nr_cqe requirements and
432
+ * claim entries in it for us.  In case there is no available cq, allocate
433
+ * a new cq with the requirements and add it to the device pool.
434
+ * IB_POLL_DIRECT cannot be used for shared cqs so it is not a valid value
435
+ * for @poll_ctx.
436
+ */
437
+struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,
438
+			     int comp_vector_hint,
439
+			     enum ib_poll_context poll_ctx)
440
+{
441
+	static unsigned int default_comp_vector;
442
+	unsigned int vector, num_comp_vectors;
443
+	struct ib_cq *cq, *found = NULL;
444
+	int ret;
445
+
446
+	if (poll_ctx > IB_POLL_LAST_POOL_TYPE) {
447
+		WARN_ON_ONCE(poll_ctx > IB_POLL_LAST_POOL_TYPE);
448
+		return ERR_PTR(-EINVAL);
449
+	}
450
+
451
+	num_comp_vectors =
452
+		min_t(unsigned int, dev->num_comp_vectors, num_online_cpus());
453
+	/* Project the affinty to the device completion vector range */
454
+	if (comp_vector_hint < 0) {
455
+		comp_vector_hint =
456
+			(READ_ONCE(default_comp_vector) + 1) % num_comp_vectors;
457
+		WRITE_ONCE(default_comp_vector, comp_vector_hint);
458
+	}
459
+	vector = comp_vector_hint % num_comp_vectors;
460
+
461
+	/*
462
+	 * Find the least used CQ with correct affinity and
463
+	 * enough free CQ entries
464
+	 */
465
+	while (!found) {
466
+		spin_lock_irq(&dev->cq_pools_lock);
467
+		list_for_each_entry(cq, &dev->cq_pools[poll_ctx],
468
+				    pool_entry) {
469
+			/*
470
+			 * Check to see if we have found a CQ with the
471
+			 * correct completion vector
472
+			 */
473
+			if (vector != cq->comp_vector)
474
+				continue;
475
+			if (cq->cqe_used + nr_cqe > cq->cqe)
476
+				continue;
477
+			found = cq;
478
+			break;
479
+		}
480
+
481
+		if (found) {
482
+			found->cqe_used += nr_cqe;
483
+			spin_unlock_irq(&dev->cq_pools_lock);
484
+
485
+			return found;
486
+		}
487
+		spin_unlock_irq(&dev->cq_pools_lock);
488
+
489
+		/*
490
+		 * Didn't find a match or ran out of CQs in the device
491
+		 * pool, allocate a new array of CQs.
492
+		 */
493
+		ret = ib_alloc_cqs(dev, nr_cqe, poll_ctx);
494
+		if (ret)
495
+			return ERR_PTR(ret);
496
+	}
497
+
498
+	return found;
499
+}
500
+EXPORT_SYMBOL(ib_cq_pool_get);
501
+
502
+/**
503
+ * ib_cq_pool_put - Return a CQ taken from a shared pool.
504
+ * @cq: The CQ to return.
505
+ * @nr_cqe: The max number of cqes that the user had requested.
506
+ */
507
+void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe)
508
+{
509
+	if (WARN_ON_ONCE(nr_cqe > cq->cqe_used))
510
+		return;
511
+
512
+	spin_lock_irq(&cq->device->cq_pools_lock);
513
+	cq->cqe_used -= nr_cqe;
514
+	spin_unlock_irq(&cq->device->cq_pools_lock);
515
+}
516
+EXPORT_SYMBOL(ib_cq_pool_put);