disable FB

hc

2024-01-04 1543e317f1da31b75942316931e8f491a8920811

 kernel/drivers/infiniband/sw/rdmavt/srq.c
..
..
@@ -48,10 +48,11 @@
48
48
 #include <linux/err.h>
49
49
 #include <linux/slab.h>
50
50
 #include <linux/vmalloc.h>
51
+#include <rdma/uverbs_ioctl.h>
51
52
 
52
53
 #include "srq.h"
53
54
 #include "vt.h"
54
-
55
+#include "qp.h"
55
56
 /**
56
57
  * rvt_driver_srq_init - init srq resources on a per driver basis
57
58
  * @rdi: rvt dev structure
..
..
@@ -70,29 +71,24 @@
70
71
  * @srq_init_attr: the attributes of the SRQ
71
72
  * @udata: data from libibverbs when creating a user SRQ
72
73
  *
73
- * Return: Allocated srq object
74
+ * Return: 0 on success
74
75
  */
75
-struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
76
-			      struct ib_srq_init_attr *srq_init_attr,
77
-			      struct ib_udata *udata)
76
+int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr,
77
+		   struct ib_udata *udata)
78
78
 {
79
-	struct rvt_dev_info *dev = ib_to_rvt(ibpd->device);
80
-	struct rvt_srq *srq;
79
+	struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
80
+	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
81
81
 	u32 sz;
82
-	struct ib_srq *ret;
82
+	int ret;
83
83
 
84
84
 	if (srq_init_attr->srq_type != IB_SRQT_BASIC)
85
-		return ERR_PTR(-EOPNOTSUPP);
85
+		return -EOPNOTSUPP;
86
86
 
87
87
 	if (srq_init_attr->attr.max_sge == 0 ||
88
88
 	    srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge ||
89
89
 	    srq_init_attr->attr.max_wr == 0 ||
90
90
 	    srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr)
91
-		return ERR_PTR(-EINVAL);
92
-
93
-	srq = kzalloc_node(sizeof(*srq), GFP_KERNEL, dev->dparms.node);
94
-	if (!srq)
95
-		return ERR_PTR(-ENOMEM);
91
+		return -EINVAL;
96
92
 
97
93
 	/*
98
94
 	 * Need to use vmalloc() if we want to support large #s of entries.
..
..
@@ -101,12 +97,9 @@
101
97
 	srq->rq.max_sge = srq_init_attr->attr.max_sge;
102
98
 	sz = sizeof(struct ib_sge) * srq->rq.max_sge +
103
99
 		sizeof(struct rvt_rwqe);
104
-	srq->rq.wq = udata ?
105
-		vmalloc_user(sizeof(struct rvt_rwq) + srq->rq.size * sz) :
106
-		vzalloc_node(sizeof(struct rvt_rwq) + srq->rq.size * sz,
107
-			     dev->dparms.node);
108
-	if (!srq->rq.wq) {
109
-		ret = ERR_PTR(-ENOMEM);
100
+	if (rvt_alloc_rq(&srq->rq, srq->rq.size * sz,
101
+			 dev->dparms.node, udata)) {
102
+		ret = -ENOMEM;
110
103
 		goto bail_srq;
111
104
 	}
112
105
 
..
..
@@ -115,23 +108,18 @@
115
108
 	 * See rvt_mmap() for details.
116
109
 	 */
117
110
 	if (udata && udata->outlen >= sizeof(__u64)) {
118
-		int err;
119
111
 		u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz;
120
112
 
121
-		srq->ip =
122
-		    rvt_create_mmap_info(dev, s, ibpd->uobject->context,
123
-					 srq->rq.wq);
124
-		if (!srq->ip) {
125
-			ret = ERR_PTR(-ENOMEM);
113
+		srq->ip = rvt_create_mmap_info(dev, s, udata, srq->rq.wq);
114
+		if (IS_ERR(srq->ip)) {
115
+			ret = PTR_ERR(srq->ip);
126
116
 			goto bail_wq;
127
117
 		}
128
118
 
129
-		err = ib_copy_to_udata(udata, &srq->ip->offset,
119
+		ret = ib_copy_to_udata(udata, &srq->ip->offset,
130
120
 				       sizeof(srq->ip->offset));
131
-		if (err) {
132
-			ret = ERR_PTR(err);
121
+		if (ret)
133
122
 			goto bail_ip;
134
-		}
135
123
 	}
136
124
 
137
125
 	/*
..
..
@@ -143,7 +131,7 @@
143
131
 	spin_lock(&dev->n_srqs_lock);
144
132
 	if (dev->n_srqs_allocated == dev->dparms.props.max_srq) {
145
133
 		spin_unlock(&dev->n_srqs_lock);
146
-		ret = ERR_PTR(-ENOMEM);
134
+		ret = -ENOMEM;
147
135
 		goto bail_ip;
148
136
 	}
149
137
 
..
..
@@ -156,14 +144,13 @@
156
144
 		spin_unlock_irq(&dev->pending_lock);
157
145
 	}
158
146
 
159
-	return &srq->ibsrq;
147
+	return 0;
160
148
 
161
149
 bail_ip:
162
150
 	kfree(srq->ip);
163
151
 bail_wq:
164
-	vfree(srq->rq.wq);
152
+	rvt_free_rq(&srq->rq);
165
153
 bail_srq:
166
-	kfree(srq);
167
154
 	return ret;
168
155
 }
169
156
 
..
..
@@ -182,11 +169,12 @@
182
169
 {
183
170
 	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
184
171
 	struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
185
-	struct rvt_rwq *wq;
172
+	struct rvt_rq tmp_rq = {};
186
173
 	int ret = 0;
187
174
 
188
175
 	if (attr_mask & IB_SRQ_MAX_WR) {
189
-		struct rvt_rwq *owq;
176
+		struct rvt_krwq *okwq = NULL;
177
+		struct rvt_rwq *owq = NULL;
190
178
 		struct rvt_rwqe *p;
191
179
 		u32 sz, size, n, head, tail;
192
180
 
..
..
@@ -195,17 +183,12 @@
195
183
 		    ((attr_mask & IB_SRQ_LIMIT) ?
196
184
 		     attr->srq_limit : srq->limit) > attr->max_wr)
197
185
 			return -EINVAL;
198
-
199
186
 		sz = sizeof(struct rvt_rwqe) +
200
187
 			srq->rq.max_sge * sizeof(struct ib_sge);
201
188
 		size = attr->max_wr + 1;
202
-		wq = udata ?
203
-			vmalloc_user(sizeof(struct rvt_rwq) + size * sz) :
204
-			vzalloc_node(sizeof(struct rvt_rwq) + size * sz,
205
-				     dev->dparms.node);
206
-		if (!wq)
189
+		if (rvt_alloc_rq(&tmp_rq, size * sz, dev->dparms.node,
190
+				 udata))
207
191
 			return -ENOMEM;
208
-
209
192
 		/* Check that we can write the offset to mmap. */
210
193
 		if (udata && udata->inlen >= sizeof(__u64)) {
211
194
 			__u64 offset_addr;
..
..
@@ -223,14 +206,20 @@
223
206
 				goto bail_free;
224
207
 		}
225
208
 
226
-		spin_lock_irq(&srq->rq.lock);
209
+		spin_lock_irq(&srq->rq.kwq->c_lock);
227
210
 		/*
228
211
 		 * validate head and tail pointer values and compute
229
212
 		 * the number of remaining WQEs.
230
213
 		 */
231
-		owq = srq->rq.wq;
232
-		head = owq->head;
233
-		tail = owq->tail;
214
+		if (udata) {
215
+			owq = srq->rq.wq;
216
+			head = RDMA_READ_UAPI_ATOMIC(owq->head);
217
+			tail = RDMA_READ_UAPI_ATOMIC(owq->tail);
218
+		} else {
219
+			okwq = srq->rq.kwq;
220
+			head = okwq->head;
221
+			tail = okwq->tail;
222
+		}
234
223
 		if (head >= srq->rq.size || tail >= srq->rq.size) {
235
224
 			ret = -EINVAL;
236
225
 			goto bail_unlock;
..
..
@@ -245,7 +234,7 @@
245
234
 			goto bail_unlock;
246
235
 		}
247
236
 		n = 0;
248
-		p = wq->wq;
237
+		p = tmp_rq.kwq->curr_wq;
249
238
 		while (tail != head) {
250
239
 			struct rvt_rwqe *wqe;
251
240
 			int i;
..
..
@@ -260,22 +249,29 @@
260
249
 			if (++tail >= srq->rq.size)
261
250
 				tail = 0;
262
251
 		}
263
-		srq->rq.wq = wq;
252
+		srq->rq.kwq = tmp_rq.kwq;
253
+		if (udata) {
254
+			srq->rq.wq = tmp_rq.wq;
255
+			RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->head, n);
256
+			RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->tail, 0);
257
+		} else {
258
+			tmp_rq.kwq->head = n;
259
+			tmp_rq.kwq->tail = 0;
260
+		}
264
261
 		srq->rq.size = size;
265
-		wq->head = n;
266
-		wq->tail = 0;
267
262
 		if (attr_mask & IB_SRQ_LIMIT)
268
263
 			srq->limit = attr->srq_limit;
269
-		spin_unlock_irq(&srq->rq.lock);
264
+		spin_unlock_irq(&srq->rq.kwq->c_lock);
270
265
 
271
266
 		vfree(owq);
267
+		kvfree(okwq);
272
268
 
273
269
 		if (srq->ip) {
274
270
 			struct rvt_mmap_info *ip = srq->ip;
275
271
 			struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device);
276
272
 			u32 s = sizeof(struct rvt_rwq) + size * sz;
277
273
 
278
-			rvt_update_mmap_info(dev, ip, s, wq);
274
+			rvt_update_mmap_info(dev, ip, s, tmp_rq.wq);
279
275
 
280
276
 			/*
281
277
 			 * Return the offset to mmap.
..
..
@@ -299,19 +295,19 @@
299
295
 			spin_unlock_irq(&dev->pending_lock);
300
296
 		}
301
297
 	} else if (attr_mask & IB_SRQ_LIMIT) {
302
-		spin_lock_irq(&srq->rq.lock);
298
+		spin_lock_irq(&srq->rq.kwq->c_lock);
303
299
 		if (attr->srq_limit >= srq->rq.size)
304
300
 			ret = -EINVAL;
305
301
 		else
306
302
 			srq->limit = attr->srq_limit;
307
-		spin_unlock_irq(&srq->rq.lock);
303
+		spin_unlock_irq(&srq->rq.kwq->c_lock);
308
304
 	}
309
305
 	return ret;
310
306
 
311
307
 bail_unlock:
312
-	spin_unlock_irq(&srq->rq.lock);
308
+	spin_unlock_irq(&srq->rq.kwq->c_lock);
313
309
 bail_free:
314
-	vfree(wq);
310
+	rvt_free_rq(&tmp_rq);
315
311
 	return ret;
316
312
 }
317
313
 
..
..
@@ -335,9 +331,8 @@
335
331
  * rvt_destroy_srq - destory an srq
336
332
  * @ibsrq: srq object to destroy
337
333
  *
338
- * Return always 0
339
334
  */
340
-int rvt_destroy_srq(struct ib_srq *ibsrq)
335
+int rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
341
336
 {
342
337
 	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
343
338
 	struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
..
..
@@ -347,9 +342,6 @@
347
342
 	spin_unlock(&dev->n_srqs_lock);
348
343
 	if (srq->ip)
349
344
 		kref_put(&srq->ip->ref, rvt_release_mmap_info);
350
-	else
351
-		vfree(srq->rq.wq);
352
-	kfree(srq);
353
-
345
+	kvfree(srq->rq.kwq);
354
346
 	return 0;
355
347
 }