~hc/RK356X_SDK_RELEASE.git

..	..	@@ -47,11 +47,149 @@
47	47	#include "uverbs.h"
48	48	#include "core_priv.h"
49	49
	50	+/*
	51	+ * Copy a response to userspace. If the provided 'resp' is larger than the
	52	+ * user buffer it is silently truncated. If the user provided a larger buffer
	53	+ * then the trailing portion is zero filled.
	54	+ *
	55	+ * These semantics are intended to support future extension of the output
	56	+ * structures.
	57	+ */
	58	+static int uverbs_response(struct uverbs_attr_bundle attrs, const void resp,
	59	+ size_t resp_len)
	60	+{
	61	+ int ret;
	62	+
	63	+ if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
	64	+ return uverbs_copy_to_struct_or_zero(
	65	+ attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);
	66	+
	67	+ if (copy_to_user(attrs->ucore.outbuf, resp,
	68	+ min(attrs->ucore.outlen, resp_len)))
	69	+ return -EFAULT;
	70	+
	71	+ if (resp_len < attrs->ucore.outlen) {
	72	+ /*
	73	+ * Zero fill any extra memory that user
	74	+ * space might have provided.
	75	+ */
	76	+ ret = clear_user(attrs->ucore.outbuf + resp_len,
	77	+ attrs->ucore.outlen - resp_len);
	78	+ if (ret)
	79	+ return -EFAULT;
	80	+ }
	81	+
	82	+ return 0;
	83	+}
	84	+
	85	+/*
	86	+ * Copy a request from userspace. If the provided 'req' is larger than the
	87	+ * user buffer then the user buffer is zero extended into the 'req'. If 'req'
	88	+ * is smaller than the user buffer then the uncopied bytes in the user buffer
	89	+ * must be zero.
	90	+ */
	91	+static int uverbs_request(struct uverbs_attr_bundle attrs, void req,
	92	+ size_t req_len)
	93	+{
	94	+ if (copy_from_user(req, attrs->ucore.inbuf,
	95	+ min(attrs->ucore.inlen, req_len)))
	96	+ return -EFAULT;
	97	+
	98	+ if (attrs->ucore.inlen < req_len) {
	99	+ memset(req + attrs->ucore.inlen, 0,
	100	+ req_len - attrs->ucore.inlen);
	101	+ } else if (attrs->ucore.inlen > req_len) {
	102	+ if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,
	103	+ attrs->ucore.inlen - req_len))
	104	+ return -EOPNOTSUPP;
	105	+ }
	106	+ return 0;
	107	+}
	108	+
	109	+/*
	110	+ * Generate the value for the 'response_length' protocol used by write_ex.
	111	+ * This is the number of bytes the kernel actually wrote. Userspace can use
	112	+ * this to detect what structure members in the response the kernel
	113	+ * understood.
	114	+ */
	115	+static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,
	116	+ size_t resp_len)
	117	+{
	118	+ return min_t(size_t, attrs->ucore.outlen, resp_len);
	119	+}
	120	+
	121	+/*
	122	+ * The iterator version of the request interface is for handlers that need to
	123	+ * step over a flex array at the end of a command header.
	124	+ */
	125	+struct uverbs_req_iter {
	126	+ const void __user *cur;
	127	+ const void __user *end;
	128	+};
	129	+
	130	+static int uverbs_request_start(struct uverbs_attr_bundle *attrs,
	131	+ struct uverbs_req_iter *iter,
	132	+ void *req,
	133	+ size_t req_len)
	134	+{
	135	+ if (attrs->ucore.inlen < req_len)
	136	+ return -ENOSPC;
	137	+
	138	+ if (copy_from_user(req, attrs->ucore.inbuf, req_len))
	139	+ return -EFAULT;
	140	+
	141	+ iter->cur = attrs->ucore.inbuf + req_len;
	142	+ iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;
	143	+ return 0;
	144	+}
	145	+
	146	+static int uverbs_request_next(struct uverbs_req_iter iter, void val,
	147	+ size_t len)
	148	+{
	149	+ if (iter->cur + len > iter->end)
	150	+ return -ENOSPC;
	151	+
	152	+ if (copy_from_user(val, iter->cur, len))
	153	+ return -EFAULT;
	154	+
	155	+ iter->cur += len;
	156	+ return 0;
	157	+}
	158	+
	159	+static const void __user uverbs_request_next_ptr(struct uverbs_req_iter iter,
	160	+ size_t len)
	161	+{
	162	+ const void __user *res = iter->cur;
	163	+
	164	+ if (iter->cur + len > iter->end)
	165	+ return (void __force __user *)ERR_PTR(-ENOSPC);
	166	+ iter->cur += len;
	167	+ return res;
	168	+}
	169	+
	170	+static int uverbs_request_finish(struct uverbs_req_iter *iter)
	171	+{
	172	+ if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur))
	173	+ return -EOPNOTSUPP;
	174	+ return 0;
	175	+}
	176	+
	177	+/*
	178	+ * When calling a destroy function during an error unwind we need to pass in
	179	+ * the udata that is sanitized of all user arguments. Ie from the driver
	180	+ * perspective it looks like no udata was passed.
	181	+ */
	182	+struct ib_udata uverbs_get_cleared_udata(struct uverbs_attr_bundle attrs)
	183	+{
	184	+ attrs->driver_udata = (struct ib_udata){};
	185	+ return &attrs->driver_udata;
	186	+}
	187	+
50	188	static struct ib_uverbs_completion_event_file *
51		-_ib_uverbs_lookup_comp_file(s32 fd, struct ib_uverbs_file *ufile)
	189	+_ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)
52	190	{
53	191	struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
54		- fd, ufile);
	192	+ fd, attrs);
55	193
56	194	if (IS_ERR(uobj))
57	195	return (void *)uobj;
..	..	@@ -65,94 +203,56 @@
65	203	#define ib_uverbs_lookup_comp_file(_fd, _ufile) \
66	204	_ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
67	205
68		-ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
69		- const char __user *buf,
70		- int in_len, int out_len)
	206	+int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)
71	207	{
72		- struct ib_uverbs_get_context cmd;
73		- struct ib_uverbs_get_context_resp resp;
74		- struct ib_udata udata;
75		- struct ib_ucontext *ucontext;
76		- struct file *filp;
77		- struct ib_rdmacg_object cg_obj;
	208	+ struct ib_uverbs_file *ufile = attrs->ufile;
	209	+ struct ib_ucontext *ucontext;
78	210	struct ib_device *ib_dev;
	211	+
	212	+ ib_dev = srcu_dereference(ufile->device->ib_dev,
	213	+ &ufile->device->disassociate_srcu);
	214	+ if (!ib_dev)
	215	+ return -EIO;
	216	+
	217	+ ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);
	218	+ if (!ucontext)
	219	+ return -ENOMEM;
	220	+
	221	+ ucontext->device = ib_dev;
	222	+ ucontext->ufile = ufile;
	223	+ xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC);
	224	+
	225	+ rdma_restrack_new(&ucontext->res, RDMA_RESTRACK_CTX);
	226	+ rdma_restrack_set_name(&ucontext->res, NULL);
	227	+ attrs->context = ucontext;
	228	+ return 0;
	229	+}
	230	+
	231	+int ib_init_ucontext(struct uverbs_attr_bundle *attrs)
	232	+{
	233	+ struct ib_ucontext *ucontext = attrs->context;
	234	+ struct ib_uverbs_file *file = attrs->ufile;
79	235	int ret;
80	236
81		- if (out_len < sizeof resp)
82		- return -ENOSPC;
83		-
84		- if (copy_from_user(&cmd, buf, sizeof cmd))
85		- return -EFAULT;
86		-
	237	+ if (!down_read_trylock(&file->hw_destroy_rwsem))
	238	+ return -EIO;
87	239	mutex_lock(&file->ucontext_lock);
88		- ib_dev = srcu_dereference(file->device->ib_dev,
89		- &file->device->disassociate_srcu);
90		- if (!ib_dev) {
91		- ret = -EIO;
92		- goto err;
93		- }
94		-
95	240	if (file->ucontext) {
96	241	ret = -EINVAL;
97	242	goto err;
98	243	}
99	244
100		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
101		- u64_to_user_ptr(cmd.response) + sizeof(resp),
102		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
103		- out_len - sizeof(resp));
104		-
105		- ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
	245	+ ret = ib_rdmacg_try_charge(&ucontext->cg_obj, ucontext->device,
	246	+ RDMACG_RESOURCE_HCA_HANDLE);
106	247	if (ret)
107	248	goto err;
108	249
109		- ucontext = ib_dev->alloc_ucontext(ib_dev, &udata);
110		- if (IS_ERR(ucontext)) {
111		- ret = PTR_ERR(ucontext);
112		- goto err_alloc;
113		- }
	250	+ ret = ucontext->device->ops.alloc_ucontext(ucontext,
	251	+ &attrs->driver_udata);
	252	+ if (ret)
	253	+ goto err_uncharge;
114	254
115		- ucontext->device = ib_dev;
116		- ucontext->cg_obj = cg_obj;
117		- /* ufile is required when some objects are released */
118		- ucontext->ufile = file;
119		-
120		- rcu_read_lock();
121		- ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
122		- rcu_read_unlock();
123		- ucontext->closing = 0;
124		- ucontext->cleanup_retryable = false;
125		-
126		-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
127		- ucontext->umem_tree = RB_ROOT_CACHED;
128		- init_rwsem(&ucontext->umem_rwsem);
129		- ucontext->odp_mrs_count = 0;
130		- INIT_LIST_HEAD(&ucontext->no_private_counters);
131		-
132		- if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
133		- ucontext->invalidate_range = NULL;
134		-
135		-#endif
136		-
137		- resp.num_comp_vectors = file->device->num_comp_vectors;
138		-
139		- ret = get_unused_fd_flags(O_CLOEXEC);
140		- if (ret < 0)
141		- goto err_free;
142		- resp.async_fd = ret;
143		-
144		- filp = ib_uverbs_alloc_async_event_file(file, ib_dev);
145		- if (IS_ERR(filp)) {
146		- ret = PTR_ERR(filp);
147		- goto err_fd;
148		- }
149		-
150		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
151		- ret = -EFAULT;
152		- goto err_file;
153		- }
154		-
155		- fd_install(resp.async_fd, filp);
	255	+ rdma_restrack_add(&ucontext->res);
156	256
157	257	/*
158	258	* Make sure that ib_uverbs_get_ucontext() sees the pointer update
..	..	@@ -161,25 +261,63 @@
161	261	smp_store_release(&file->ucontext, ucontext);
162	262
163	263	mutex_unlock(&file->ucontext_lock);
	264	+ up_read(&file->hw_destroy_rwsem);
	265	+ return 0;
164	266
165		- return in_len;
166		-
167		-err_file:
168		- ib_uverbs_free_async_event_file(file);
169		- fput(filp);
170		-
171		-err_fd:
172		- put_unused_fd(resp.async_fd);
173		-
174		-err_free:
175		- put_pid(ucontext->tgid);
176		- ib_dev->dealloc_ucontext(ucontext);
177		-
178		-err_alloc:
179		- ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
180		-
	267	+err_uncharge:
	268	+ ib_rdmacg_uncharge(&ucontext->cg_obj, ucontext->device,
	269	+ RDMACG_RESOURCE_HCA_HANDLE);
181	270	err:
182	271	mutex_unlock(&file->ucontext_lock);
	272	+ up_read(&file->hw_destroy_rwsem);
	273	+ return ret;
	274	+}
	275	+
	276	+static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
	277	+{
	278	+ struct ib_uverbs_get_context_resp resp;
	279	+ struct ib_uverbs_get_context cmd;
	280	+ struct ib_device *ib_dev;
	281	+ struct ib_uobject *uobj;
	282	+ int ret;
	283	+
	284	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	285	+ if (ret)
	286	+ return ret;
	287	+
	288	+ ret = ib_alloc_ucontext(attrs);
	289	+ if (ret)
	290	+ return ret;
	291	+
	292	+ uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev);
	293	+ if (IS_ERR(uobj)) {
	294	+ ret = PTR_ERR(uobj);
	295	+ goto err_ucontext;
	296	+ }
	297	+
	298	+ resp = (struct ib_uverbs_get_context_resp){
	299	+ .num_comp_vectors = attrs->ufile->device->num_comp_vectors,
	300	+ .async_fd = uobj->id,
	301	+ };
	302	+ ret = uverbs_response(attrs, &resp, sizeof(resp));
	303	+ if (ret)
	304	+ goto err_uobj;
	305	+
	306	+ ret = ib_init_ucontext(attrs);
	307	+ if (ret)
	308	+ goto err_uobj;
	309	+
	310	+ ib_uverbs_init_async_event_file(
	311	+ container_of(uobj, struct ib_uverbs_async_event_file, uobj));
	312	+ rdma_alloc_commit_uobject(uobj, attrs);
	313	+ return 0;
	314	+
	315	+err_uobj:
	316	+ rdma_alloc_abort_uobject(uobj, attrs, false);
	317	+err_ucontext:
	318	+ rdma_restrack_put(&attrs->context->res);
	319	+ kfree(attrs->context);
	320	+ attrs->context = NULL;
183	321	return ret;
184	322	}
185	323
..	..	@@ -221,8 +359,6 @@
221	359	resp->max_mcast_qp_attach = attr->max_mcast_qp_attach;
222	360	resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
223	361	resp->max_ah = attr->max_ah;
224		- resp->max_fmr = attr->max_fmr;
225		- resp->max_map_per_fmr = attr->max_map_per_fmr;
226	362	resp->max_srq = attr->max_srq;
227	363	resp->max_srq_wr = attr->max_srq_wr;
228	364	resp->max_srq_sge = attr->max_srq_sge;
..	..	@@ -231,57 +367,28 @@
231	367	resp->phys_port_cnt = ib_dev->phys_port_cnt;
232	368	}
233	369
234		-ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
235		- const char __user *buf,
236		- int in_len, int out_len)
	370	+static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)
237	371	{
238	372	struct ib_uverbs_query_device cmd;
239	373	struct ib_uverbs_query_device_resp resp;
240	374	struct ib_ucontext *ucontext;
	375	+ int ret;
241	376
242		- ucontext = ib_uverbs_get_ucontext(file);
	377	+ ucontext = ib_uverbs_get_ucontext(attrs);
243	378	if (IS_ERR(ucontext))
244	379	return PTR_ERR(ucontext);
245	380
246		- if (out_len < sizeof resp)
247		- return -ENOSPC;
248		-
249		- if (copy_from_user(&cmd, buf, sizeof cmd))
250		- return -EFAULT;
	381	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	382	+ if (ret)
	383	+ return ret;
251	384
252	385	memset(&resp, 0, sizeof resp);
253	386	copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);
254	387
255		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
256		- return -EFAULT;
257		-
258		- return in_len;
	388	+ return uverbs_response(attrs, &resp, sizeof(resp));
259	389	}
260	390
261		-/*
262		- * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the
263		- * PortInfo CapabilityMask, but was extended with unique bits.
264		- */
265		-static u32 make_port_cap_flags(const struct ib_port_attr *attr)
266		-{
267		- u32 res;
268		-
269		- /* All IBA CapabilityMask bits are passed through here, except bit 26,
270		- * which is overridden with IP_BASED_GIDS. This is due to a historical
271		- * mistake in the implementation of IP_BASED_GIDS. Otherwise all other
272		- * bits match the IBA definition across all kernel versions.
273		- */
274		- res = attr->port_cap_flags & ~(u32)IB_UVERBS_PCF_IP_BASED_GIDS;
275		-
276		- if (attr->ip_gids)
277		- res \|= IB_UVERBS_PCF_IP_BASED_GIDS;
278		-
279		- return res;
280		-}
281		-
282		-ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
283		- const char __user *buf,
284		- int in_len, int out_len)
	391	+static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)
285	392	{
286	393	struct ib_uverbs_query_port cmd;
287	394	struct ib_uverbs_query_port_resp resp;
..	..	@@ -290,130 +397,84 @@
290	397	struct ib_ucontext *ucontext;
291	398	struct ib_device *ib_dev;
292	399
293		- ucontext = ib_uverbs_get_ucontext(file);
	400	+ ucontext = ib_uverbs_get_ucontext(attrs);
294	401	if (IS_ERR(ucontext))
295	402	return PTR_ERR(ucontext);
296	403	ib_dev = ucontext->device;
297	404
298		- if (out_len < sizeof resp)
299		- return -ENOSPC;
300		-
301		- if (copy_from_user(&cmd, buf, sizeof cmd))
302		- return -EFAULT;
	405	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	406	+ if (ret)
	407	+ return ret;
303	408
304	409	ret = ib_query_port(ib_dev, cmd.port_num, &attr);
305	410	if (ret)
306	411	return ret;
307	412
308	413	memset(&resp, 0, sizeof resp);
	414	+ copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num);
309	415
310		- resp.state = attr.state;
311		- resp.max_mtu = attr.max_mtu;
312		- resp.active_mtu = attr.active_mtu;
313		- resp.gid_tbl_len = attr.gid_tbl_len;
314		- resp.port_cap_flags = make_port_cap_flags(&attr);
315		- resp.max_msg_sz = attr.max_msg_sz;
316		- resp.bad_pkey_cntr = attr.bad_pkey_cntr;
317		- resp.qkey_viol_cntr = attr.qkey_viol_cntr;
318		- resp.pkey_tbl_len = attr.pkey_tbl_len;
319		-
320		- if (rdma_is_grh_required(ib_dev, cmd.port_num))
321		- resp.flags \|= IB_UVERBS_QPF_GRH_REQUIRED;
322		-
323		- if (rdma_cap_opa_ah(ib_dev, cmd.port_num)) {
324		- resp.lid = OPA_TO_IB_UCAST_LID(attr.lid);
325		- resp.sm_lid = OPA_TO_IB_UCAST_LID(attr.sm_lid);
326		- } else {
327		- resp.lid = ib_lid_cpu16(attr.lid);
328		- resp.sm_lid = ib_lid_cpu16(attr.sm_lid);
329		- }
330		- resp.lmc = attr.lmc;
331		- resp.max_vl_num = attr.max_vl_num;
332		- resp.sm_sl = attr.sm_sl;
333		- resp.subnet_timeout = attr.subnet_timeout;
334		- resp.init_type_reply = attr.init_type_reply;
335		- resp.active_width = attr.active_width;
336		- resp.active_speed = attr.active_speed;
337		- resp.phys_state = attr.phys_state;
338		- resp.link_layer = rdma_port_get_link_layer(ib_dev,
339		- cmd.port_num);
340		-
341		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
342		- return -EFAULT;
343		-
344		- return in_len;
	416	+ return uverbs_response(attrs, &resp, sizeof(resp));
345	417	}
346	418
347		-ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
348		- const char __user *buf,
349		- int in_len, int out_len)
	419	+static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
350	420	{
	421	+ struct ib_uverbs_alloc_pd_resp resp = {};
351	422	struct ib_uverbs_alloc_pd cmd;
352		- struct ib_uverbs_alloc_pd_resp resp;
353		- struct ib_udata udata;
354	423	struct ib_uobject *uobj;
355	424	struct ib_pd *pd;
356	425	int ret;
357	426	struct ib_device *ib_dev;
358	427
359		- if (out_len < sizeof resp)
360		- return -ENOSPC;
	428	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	429	+ if (ret)
	430	+ return ret;
361	431
362		- if (copy_from_user(&cmd, buf, sizeof cmd))
363		- return -EFAULT;
364		-
365		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
366		- u64_to_user_ptr(cmd.response) + sizeof(resp),
367		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
368		- out_len - sizeof(resp));
369		-
370		- uobj = uobj_alloc(UVERBS_OBJECT_PD, file, &ib_dev);
	432	+ uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);
371	433	if (IS_ERR(uobj))
372	434	return PTR_ERR(uobj);
373	435
374		- pd = ib_dev->alloc_pd(ib_dev, uobj->context, &udata);
375		- if (IS_ERR(pd)) {
376		- ret = PTR_ERR(pd);
	436	+ pd = rdma_zalloc_drv_obj(ib_dev, ib_pd);
	437	+ if (!pd) {
	438	+ ret = -ENOMEM;
377	439	goto err;
378	440	}
379	441
380	442	pd->device = ib_dev;
381	443	pd->uobject = uobj;
382		- pd->__internal_mr = NULL;
383	444	atomic_set(&pd->usecnt, 0);
384	445
385		- uobj->object = pd;
386		- memset(&resp, 0, sizeof resp);
387		- resp.pd_handle = uobj->id;
388		- pd->res.type = RDMA_RESTRACK_PD;
	446	+ rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
	447	+ rdma_restrack_set_name(&pd->res, NULL);
	448	+
	449	+ ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
	450	+ if (ret)
	451	+ goto err_alloc;
389	452	rdma_restrack_add(&pd->res);
390	453
391		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
392		- ret = -EFAULT;
393		- goto err_copy;
394		- }
	454	+ uobj->object = pd;
	455	+ uobj_finalize_uobj_create(uobj, attrs);
395	456
396		- return uobj_alloc_commit(uobj, in_len);
	457	+ resp.pd_handle = uobj->id;
	458	+ return uverbs_response(attrs, &resp, sizeof(resp));
397	459
398		-err_copy:
399		- ib_dealloc_pd(pd);
400		-
	460	+err_alloc:
	461	+ rdma_restrack_put(&pd->res);
	462	+ kfree(pd);
401	463	err:
402		- uobj_alloc_abort(uobj);
	464	+ uobj_alloc_abort(uobj, attrs);
403	465	return ret;
404	466	}
405	467
406		-ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file,
407		- const char __user *buf,
408		- int in_len, int out_len)
	468	+static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)
409	469	{
410	470	struct ib_uverbs_dealloc_pd cmd;
	471	+ int ret;
411	472
412		- if (copy_from_user(&cmd, buf, sizeof cmd))
413		- return -EFAULT;
	473	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	474	+ if (ret)
	475	+ return ret;
414	476
415		- return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, file,
416		- in_len);
	477	+ return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
417	478	}
418	479
419	480	struct xrcd_table_entry {
..	..	@@ -501,33 +562,24 @@
501	562	}
502	563	}
503	564
504		-ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
505		- const char __user *buf, int in_len,
506		- int out_len)
	565	+static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
507	566	{
	567	+ struct ib_uverbs_device *ibudev = attrs->ufile->device;
	568	+ struct ib_uverbs_open_xrcd_resp resp = {};
508	569	struct ib_uverbs_open_xrcd cmd;
509		- struct ib_uverbs_open_xrcd_resp resp;
510		- struct ib_udata udata;
511	570	struct ib_uxrcd_object *obj;
512	571	struct ib_xrcd *xrcd = NULL;
513		- struct fd f = {NULL, 0};
514	572	struct inode *inode = NULL;
515		- int ret = 0;
516	573	int new_xrcd = 0;
517	574	struct ib_device *ib_dev;
	575	+ struct fd f = {};
	576	+ int ret;
518	577
519		- if (out_len < sizeof resp)
520		- return -ENOSPC;
	578	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	579	+ if (ret)
	580	+ return ret;
521	581
522		- if (copy_from_user(&cmd, buf, sizeof cmd))
523		- return -EFAULT;
524		-
525		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
526		- u64_to_user_ptr(cmd.response) + sizeof(resp),
527		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
528		- out_len - sizeof(resp));
529		-
530		- mutex_lock(&file->device->xrcd_tree_mutex);
	582	+ mutex_lock(&ibudev->xrcd_tree_mutex);
531	583
532	584	if (cmd.fd != -1) {
533	585	/* search for file descriptor */
..	..	@@ -538,7 +590,7 @@
538	590	}
539	591
540	592	inode = file_inode(f.file);
541		- xrcd = find_xrcd(file->device, inode);
	593	+ xrcd = find_xrcd(ibudev, inode);
542	594	if (!xrcd && !(cmd.oflags & O_CREAT)) {
543	595	/* no file descriptor. Need CREATE flag */
544	596	ret = -EAGAIN;
..	..	@@ -551,7 +603,7 @@
551	603	}
552	604	}
553	605
554		- obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, file,
	606	+ obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,
555	607	&ib_dev);
556	608	if (IS_ERR(obj)) {
557	609	ret = PTR_ERR(obj);
..	..	@@ -559,95 +611,76 @@
559	611	}
560	612
561	613	if (!xrcd) {
562		- xrcd = ib_dev->alloc_xrcd(ib_dev, obj->uobject.context, &udata);
	614	+ xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata);
563	615	if (IS_ERR(xrcd)) {
564	616	ret = PTR_ERR(xrcd);
565	617	goto err;
566	618	}
567		-
568		- xrcd->inode = inode;
569		- xrcd->device = ib_dev;
570		- atomic_set(&xrcd->usecnt, 0);
571		- mutex_init(&xrcd->tgt_qp_mutex);
572		- INIT_LIST_HEAD(&xrcd->tgt_qp_list);
573	619	new_xrcd = 1;
574	620	}
575	621
576	622	atomic_set(&obj->refcnt, 0);
577	623	obj->uobject.object = xrcd;
578		- memset(&resp, 0, sizeof resp);
579		- resp.xrcd_handle = obj->uobject.id;
580	624
581	625	if (inode) {
582	626	if (new_xrcd) {
583	627	/* create new inode/xrcd table entry */
584		- ret = xrcd_table_insert(file->device, inode, xrcd);
	628	+ ret = xrcd_table_insert(ibudev, inode, xrcd);
585	629	if (ret)
586	630	goto err_dealloc_xrcd;
587	631	}
588	632	atomic_inc(&xrcd->usecnt);
589	633	}
590	634
591		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
592		- ret = -EFAULT;
593		- goto err_copy;
594		- }
595		-
596	635	if (f.file)
597	636	fdput(f);
598	637
599		- mutex_unlock(&file->device->xrcd_tree_mutex);
	638	+ mutex_unlock(&ibudev->xrcd_tree_mutex);
	639	+ uobj_finalize_uobj_create(&obj->uobject, attrs);
600	640
601		- return uobj_alloc_commit(&obj->uobject, in_len);
602		-
603		-err_copy:
604		- if (inode) {
605		- if (new_xrcd)
606		- xrcd_table_delete(file->device, inode);
607		- atomic_dec(&xrcd->usecnt);
608		- }
	641	+ resp.xrcd_handle = obj->uobject.id;
	642	+ return uverbs_response(attrs, &resp, sizeof(resp));
609	643
610	644	err_dealloc_xrcd:
611		- ib_dealloc_xrcd(xrcd);
	645	+ ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));
612	646
613	647	err:
614		- uobj_alloc_abort(&obj->uobject);
	648	+ uobj_alloc_abort(&obj->uobject, attrs);
615	649
616	650	err_tree_mutex_unlock:
617	651	if (f.file)
618	652	fdput(f);
619	653
620		- mutex_unlock(&file->device->xrcd_tree_mutex);
	654	+ mutex_unlock(&ibudev->xrcd_tree_mutex);
621	655
622	656	return ret;
623	657	}
624	658
625		-ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file,
626		- const char __user *buf, int in_len,
627		- int out_len)
	659	+static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
628	660	{
629	661	struct ib_uverbs_close_xrcd cmd;
	662	+ int ret;
630	663
631		- if (copy_from_user(&cmd, buf, sizeof cmd))
632		- return -EFAULT;
	664	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	665	+ if (ret)
	666	+ return ret;
633	667
634		- return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, file,
635		- in_len);
	668	+ return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
636	669	}
637	670
638		-int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject,
639		- struct ib_xrcd *xrcd,
640		- enum rdma_remove_reason why)
	671	+int ib_uverbs_dealloc_xrcd(struct ib_uobject uobject, struct ib_xrcd xrcd,
	672	+ enum rdma_remove_reason why,
	673	+ struct uverbs_attr_bundle *attrs)
641	674	{
642	675	struct inode *inode;
643	676	int ret;
644		- struct ib_uverbs_device *dev = uobject->context->ufile->device;
	677	+ struct ib_uverbs_device *dev = attrs->ufile->device;
645	678
646	679	inode = xrcd->inode;
647	680	if (inode && !atomic_dec_and_test(&xrcd->usecnt))
648	681	return 0;
649	682
650		- ret = ib_dealloc_xrcd(xrcd);
	683	+ ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);
651	684
652	685	if (ib_is_destroy_retryable(ret, why, uobject)) {
653	686	atomic_inc(&xrcd->usecnt);
..	..	@@ -660,29 +693,19 @@
660	693	return ret;
661	694	}
662	695
663		-ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
664		- const char __user *buf, int in_len,
665		- int out_len)
	696	+static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
666	697	{
	698	+ struct ib_uverbs_reg_mr_resp resp = {};
667	699	struct ib_uverbs_reg_mr cmd;
668		- struct ib_uverbs_reg_mr_resp resp;
669		- struct ib_udata udata;
670	700	struct ib_uobject *uobj;
671	701	struct ib_pd *pd;
672	702	struct ib_mr *mr;
673	703	int ret;
674	704	struct ib_device *ib_dev;
675	705
676		- if (out_len < sizeof resp)
677		- return -ENOSPC;
678		-
679		- if (copy_from_user(&cmd, buf, sizeof cmd))
680		- return -EFAULT;
681		-
682		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
683		- u64_to_user_ptr(cmd.response) + sizeof(resp),
684		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
685		- out_len - sizeof(resp));
	706	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	707	+ if (ret)
	708	+ return ret;
686	709
687	710	if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
688	711	return -EINVAL;
..	..	@@ -691,11 +714,11 @@
691	714	if (ret)
692	715	return ret;
693	716
694		- uobj = uobj_alloc(UVERBS_OBJECT_MR, file, &ib_dev);
	717	+ uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
695	718	if (IS_ERR(uobj))
696	719	return PTR_ERR(uobj);
697	720
698		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file);
	721	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
699	722	if (!pd) {
700	723	ret = -EINVAL;
701	724	goto err_free;
..	..	@@ -710,8 +733,9 @@
710	733	}
711	734	}
712	735
713		- mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
714		- cmd.access_flags, &udata);
	736	+ mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
	737	+ cmd.access_flags,
	738	+ &attrs->driver_udata);
715	739	if (IS_ERR(mr)) {
716	740	ret = PTR_ERR(mr);
717	741	goto err_put;
..	..	@@ -719,62 +743,47 @@
719	743
720	744	mr->device = pd->device;
721	745	mr->pd = pd;
	746	+ mr->type = IB_MR_TYPE_USER;
722	747	mr->dm = NULL;
	748	+ mr->sig_attrs = NULL;
723	749	mr->uobject = uobj;
724	750	atomic_inc(&pd->usecnt);
725		- mr->res.type = RDMA_RESTRACK_MR;
	751	+ mr->iova = cmd.hca_va;
	752	+ mr->length = cmd.length;
	753	+
	754	+ rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
	755	+ rdma_restrack_set_name(&mr->res, NULL);
726	756	rdma_restrack_add(&mr->res);
727	757
728	758	uobj->object = mr;
729		-
730		- memset(&resp, 0, sizeof resp);
731		- resp.lkey = mr->lkey;
732		- resp.rkey = mr->rkey;
733		- resp.mr_handle = uobj->id;
734		-
735		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
736		- ret = -EFAULT;
737		- goto err_copy;
738		- }
739		-
740	759	uobj_put_obj_read(pd);
	760	+ uobj_finalize_uobj_create(uobj, attrs);
741	761
742		- return uobj_alloc_commit(uobj, in_len);
743		-
744		-err_copy:
745		- ib_dereg_mr(mr);
	762	+ resp.lkey = mr->lkey;
	763	+ resp.rkey = mr->rkey;
	764	+ resp.mr_handle = uobj->id;
	765	+ return uverbs_response(attrs, &resp, sizeof(resp));
746	766
747	767	err_put:
748	768	uobj_put_obj_read(pd);
749		-
750	769	err_free:
751		- uobj_alloc_abort(uobj);
	770	+ uobj_alloc_abort(uobj, attrs);
752	771	return ret;
753	772	}
754	773
755		-ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file,
756		- const char __user *buf, int in_len,
757		- int out_len)
	774	+static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
758	775	{
759	776	struct ib_uverbs_rereg_mr cmd;
760	777	struct ib_uverbs_rereg_mr_resp resp;
761		- struct ib_udata udata;
762	778	struct ib_pd *pd = NULL;
763	779	struct ib_mr *mr;
764	780	struct ib_pd *old_pd;
765	781	int ret;
766	782	struct ib_uobject *uobj;
767	783
768		- if (out_len < sizeof(resp))
769		- return -ENOSPC;
770		-
771		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
772		- return -EFAULT;
773		-
774		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
775		- u64_to_user_ptr(cmd.response) + sizeof(resp),
776		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
777		- out_len - sizeof(resp));
	784	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	785	+ if (ret)
	786	+ return ret;
778	787
779	788	if (cmd.flags & ~IB_MR_REREG_SUPPORTED \|\| !cmd.flags)
780	789	return -EINVAL;
..	..	@@ -784,7 +793,7 @@
784	793	(cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
785	794	return -EINVAL;
786	795
787		- uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, file);
	796	+ uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
788	797	if (IS_ERR(uobj))
789	798	return PTR_ERR(uobj);
790	799
..	..	@@ -803,7 +812,7 @@
803	812
804	813	if (cmd.flags & IB_MR_REREG_PD) {
805	814	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
806		- file);
	815	+ attrs);
807	816	if (!pd) {
808	817	ret = -EINVAL;
809	818	goto put_uobjs;
..	..	@@ -811,27 +820,29 @@
811	820	}
812	821
813	822	old_pd = mr->pd;
814		- ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start,
815		- cmd.length, cmd.hca_va,
816		- cmd.access_flags, pd, &udata);
817		- if (!ret) {
818		- if (cmd.flags & IB_MR_REREG_PD) {
819		- atomic_inc(&pd->usecnt);
820		- mr->pd = pd;
821		- atomic_dec(&old_pd->usecnt);
822		- }
823		- } else {
	823	+ ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start,
	824	+ cmd.length, cmd.hca_va,
	825	+ cmd.access_flags, pd,
	826	+ &attrs->driver_udata);
	827	+ if (ret)
824	828	goto put_uobj_pd;
	829	+
	830	+ if (cmd.flags & IB_MR_REREG_PD) {
	831	+ atomic_inc(&pd->usecnt);
	832	+ mr->pd = pd;
	833	+ atomic_dec(&old_pd->usecnt);
	834	+ }
	835	+
	836	+ if (cmd.flags & IB_MR_REREG_TRANS) {
	837	+ mr->iova = cmd.hca_va;
	838	+ mr->length = cmd.length;
825	839	}
826	840
827	841	memset(&resp, 0, sizeof(resp));
828	842	resp.lkey = mr->lkey;
829	843	resp.rkey = mr->rkey;
830	844
831		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp)))
832		- ret = -EFAULT;
833		- else
834		- ret = in_len;
	845	+ ret = uverbs_response(attrs, &resp, sizeof(resp));
835	846
836	847	put_uobj_pd:
837	848	if (cmd.flags & IB_MR_REREG_PD)
..	..	@@ -843,268 +854,205 @@
843	854	return ret;
844	855	}
845	856
846		-ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file,
847		- const char __user *buf, int in_len,
848		- int out_len)
	857	+static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)
849	858	{
850	859	struct ib_uverbs_dereg_mr cmd;
	860	+ int ret;
851	861
852		- if (copy_from_user(&cmd, buf, sizeof cmd))
853		- return -EFAULT;
	862	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	863	+ if (ret)
	864	+ return ret;
854	865
855		- return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, file,
856		- in_len);
	866	+ return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
857	867	}
858	868
859		-ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file,
860		- const char __user *buf, int in_len,
861		- int out_len)
	869	+static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
862	870	{
863	871	struct ib_uverbs_alloc_mw cmd;
864		- struct ib_uverbs_alloc_mw_resp resp;
	872	+ struct ib_uverbs_alloc_mw_resp resp = {};
865	873	struct ib_uobject *uobj;
866	874	struct ib_pd *pd;
867	875	struct ib_mw *mw;
868		- struct ib_udata udata;
869	876	int ret;
870	877	struct ib_device *ib_dev;
871	878
872		- if (out_len < sizeof(resp))
873		- return -ENOSPC;
	879	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	880	+ if (ret)
	881	+ return ret;
874	882
875		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
876		- return -EFAULT;
877		-
878		- uobj = uobj_alloc(UVERBS_OBJECT_MW, file, &ib_dev);
	883	+ uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);
879	884	if (IS_ERR(uobj))
880	885	return PTR_ERR(uobj);
881	886
882		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file);
	887	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
883	888	if (!pd) {
884	889	ret = -EINVAL;
885	890	goto err_free;
886	891	}
887	892
888		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
889		- u64_to_user_ptr(cmd.response) + sizeof(resp),
890		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
891		- out_len - sizeof(resp));
892		-
893		- mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata);
894		- if (IS_ERR(mw)) {
895		- ret = PTR_ERR(mw);
	893	+ if (cmd.mw_type != IB_MW_TYPE_1 && cmd.mw_type != IB_MW_TYPE_2) {
	894	+ ret = -EINVAL;
896	895	goto err_put;
897	896	}
898	897
899		- mw->device = pd->device;
900		- mw->pd = pd;
	898	+ mw = rdma_zalloc_drv_obj(ib_dev, ib_mw);
	899	+ if (!mw) {
	900	+ ret = -ENOMEM;
	901	+ goto err_put;
	902	+ }
	903	+
	904	+ mw->device = ib_dev;
	905	+ mw->pd = pd;
901	906	mw->uobject = uobj;
	907	+ mw->type = cmd.mw_type;
	908	+
	909	+ ret = pd->device->ops.alloc_mw(mw, &attrs->driver_udata);
	910	+ if (ret)
	911	+ goto err_alloc;
	912	+
902	913	atomic_inc(&pd->usecnt);
903	914
904	915	uobj->object = mw;
905		-
906		- memset(&resp, 0, sizeof(resp));
907		- resp.rkey = mw->rkey;
908		- resp.mw_handle = uobj->id;
909		-
910		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) {
911		- ret = -EFAULT;
912		- goto err_copy;
913		- }
914		-
915	916	uobj_put_obj_read(pd);
916		- return uobj_alloc_commit(uobj, in_len);
	917	+ uobj_finalize_uobj_create(uobj, attrs);
917	918
918		-err_copy:
919		- uverbs_dealloc_mw(mw);
	919	+ resp.rkey = mw->rkey;
	920	+ resp.mw_handle = uobj->id;
	921	+ return uverbs_response(attrs, &resp, sizeof(resp));
	922	+
	923	+err_alloc:
	924	+ kfree(mw);
920	925	err_put:
921	926	uobj_put_obj_read(pd);
922	927	err_free:
923		- uobj_alloc_abort(uobj);
	928	+ uobj_alloc_abort(uobj, attrs);
924	929	return ret;
925	930	}
926	931
927		-ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file,
928		- const char __user *buf, int in_len,
929		- int out_len)
	932	+static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)
930	933	{
931	934	struct ib_uverbs_dealloc_mw cmd;
	935	+ int ret;
932	936
933		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
934		- return -EFAULT;
	937	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	938	+ if (ret)
	939	+ return ret;
935	940
936		- return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, file,
937		- in_len);
	941	+ return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);
938	942	}
939	943
940		-ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file,
941		- const char __user *buf, int in_len,
942		- int out_len)
	944	+static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
943	945	{
944	946	struct ib_uverbs_create_comp_channel cmd;
945	947	struct ib_uverbs_create_comp_channel_resp resp;
946	948	struct ib_uobject *uobj;
947	949	struct ib_uverbs_completion_event_file *ev_file;
948	950	struct ib_device *ib_dev;
	951	+ int ret;
949	952
950		- if (out_len < sizeof resp)
951		- return -ENOSPC;
	953	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	954	+ if (ret)
	955	+ return ret;
952	956
953		- if (copy_from_user(&cmd, buf, sizeof cmd))
954		- return -EFAULT;
955		-
956		- uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, file, &ib_dev);
	957	+ uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);
957	958	if (IS_ERR(uobj))
958	959	return PTR_ERR(uobj);
959		-
960		- resp.fd = uobj->id;
961	960
962	961	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
963	962	uobj);
964	963	ib_uverbs_init_event_queue(&ev_file->ev_queue);
	964	+ uobj_finalize_uobj_create(uobj, attrs);
965	965
966		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
967		- uobj_alloc_abort(uobj);
968		- return -EFAULT;
969		- }
970		-
971		- return uobj_alloc_commit(uobj, in_len);
	966	+ resp.fd = uobj->id;
	967	+ return uverbs_response(attrs, &resp, sizeof(resp));
972	968	}
973	969
974		-static struct ib_ucq_object create_cq(struct ib_uverbs_file file,
975		- struct ib_udata *ucore,
976		- struct ib_udata *uhw,
977		- struct ib_uverbs_ex_create_cq *cmd,
978		- size_t cmd_sz,
979		- int (cb)(struct ib_uverbs_file file,
980		- struct ib_ucq_object *obj,
981		- struct ib_uverbs_ex_create_cq_resp *resp,
982		- struct ib_udata *udata,
983		- void *context),
984		- void *context)
	970	+static int create_cq(struct uverbs_attr_bundle *attrs,
	971	+ struct ib_uverbs_ex_create_cq *cmd)
985	972	{
986	973	struct ib_ucq_object *obj;
987	974	struct ib_uverbs_completion_event_file *ev_file = NULL;
988	975	struct ib_cq *cq;
989	976	int ret;
990		- struct ib_uverbs_ex_create_cq_resp resp;
	977	+ struct ib_uverbs_ex_create_cq_resp resp = {};
991	978	struct ib_cq_init_attr attr = {};
992	979	struct ib_device *ib_dev;
993	980
994		- if (cmd->comp_vector >= file->device->num_comp_vectors)
995		- return ERR_PTR(-EINVAL);
	981	+ if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
	982	+ return -EINVAL;
996	983
997		- obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, file,
	984	+ obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
998	985	&ib_dev);
999	986	if (IS_ERR(obj))
1000		- return obj;
1001		-
1002		- if (!ib_dev->create_cq) {
1003		- ret = -EOPNOTSUPP;
1004		- goto err;
1005		- }
	987	+ return PTR_ERR(obj);
1006	988
1007	989	if (cmd->comp_channel >= 0) {
1008		- ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, file);
	990	+ ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
1009	991	if (IS_ERR(ev_file)) {
1010	992	ret = PTR_ERR(ev_file);
1011	993	goto err;
1012	994	}
1013	995	}
1014	996
1015		- obj->uobject.user_handle = cmd->user_handle;
1016		- obj->comp_events_reported = 0;
1017		- obj->async_events_reported = 0;
	997	+ obj->uevent.uobject.user_handle = cmd->user_handle;
1018	998	INIT_LIST_HEAD(&obj->comp_list);
1019		- INIT_LIST_HEAD(&obj->async_list);
	999	+ INIT_LIST_HEAD(&obj->uevent.event_list);
1020	1000
1021	1001	attr.cqe = cmd->cqe;
1022	1002	attr.comp_vector = cmd->comp_vector;
	1003	+ attr.flags = cmd->flags;
1023	1004
1024		- if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags))
1025		- attr.flags = cmd->flags;
1026		-
1027		- cq = ib_dev->create_cq(ib_dev, &attr, obj->uobject.context, uhw);
1028		- if (IS_ERR(cq)) {
1029		- ret = PTR_ERR(cq);
	1005	+ cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
	1006	+ if (!cq) {
	1007	+ ret = -ENOMEM;
1030	1008	goto err_file;
1031	1009	}
1032		-
1033	1010	cq->device = ib_dev;
1034		- cq->uobject = &obj->uobject;
	1011	+ cq->uobject = obj;
1035	1012	cq->comp_handler = ib_uverbs_comp_handler;
1036	1013	cq->event_handler = ib_uverbs_cq_event_handler;
1037	1014	cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
1038	1015	atomic_set(&cq->usecnt, 0);
1039	1016
1040		- obj->uobject.object = cq;
1041		- memset(&resp, 0, sizeof resp);
1042		- resp.base.cq_handle = obj->uobject.id;
1043		- resp.base.cqe = cq->cqe;
	1017	+ rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
	1018	+ rdma_restrack_set_name(&cq->res, NULL);
1044	1019
1045		- resp.response_length = offsetof(typeof(resp), response_length) +
1046		- sizeof(resp.response_length);
1047		-
1048		- cq->res.type = RDMA_RESTRACK_CQ;
	1020	+ ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
	1021	+ if (ret)
	1022	+ goto err_free;
1049	1023	rdma_restrack_add(&cq->res);
1050	1024
1051		- ret = cb(file, obj, &resp, ucore, context);
1052		- if (ret)
1053		- goto err_cb;
	1025	+ obj->uevent.uobject.object = cq;
	1026	+ obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
	1027	+ if (obj->uevent.event_file)
	1028	+ uverbs_uobject_get(&obj->uevent.event_file->uobj);
	1029	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1054	1030
1055		- ret = uobj_alloc_commit(&obj->uobject, 0);
1056		- if (ret)
1057		- return ERR_PTR(ret);
1058		- return obj;
	1031	+ resp.base.cq_handle = obj->uevent.uobject.id;
	1032	+ resp.base.cqe = cq->cqe;
	1033	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	1034	+ return uverbs_response(attrs, &resp, sizeof(resp));
1059	1035
1060		-err_cb:
1061		- ib_destroy_cq(cq);
1062		-
	1036	+err_free:
	1037	+ rdma_restrack_put(&cq->res);
	1038	+ kfree(cq);
1063	1039	err_file:
1064	1040	if (ev_file)
1065		- ib_uverbs_release_ucq(file, ev_file, obj);
1066		-
	1041	+ ib_uverbs_release_ucq(ev_file, obj);
1067	1042	err:
1068		- uobj_alloc_abort(&obj->uobject);
1069		-
1070		- return ERR_PTR(ret);
	1043	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
	1044	+ return ret;
1071	1045	}
1072	1046
1073		-static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file,
1074		- struct ib_ucq_object *obj,
1075		- struct ib_uverbs_ex_create_cq_resp *resp,
1076		- struct ib_udata ucore, void context)
1077		-{
1078		- if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1079		- return -EFAULT;
1080		-
1081		- return 0;
1082		-}
1083		-
1084		-ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
1085		- const char __user *buf, int in_len,
1086		- int out_len)
	1047	+static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
1087	1048	{
1088	1049	struct ib_uverbs_create_cq cmd;
1089	1050	struct ib_uverbs_ex_create_cq cmd_ex;
1090		- struct ib_uverbs_create_cq_resp resp;
1091		- struct ib_udata ucore;
1092		- struct ib_udata uhw;
1093		- struct ib_ucq_object *obj;
	1051	+ int ret;
1094	1052
1095		- if (out_len < sizeof(resp))
1096		- return -ENOSPC;
1097		-
1098		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
1099		- return -EFAULT;
1100		-
1101		- ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response),
1102		- sizeof(cmd), sizeof(resp));
1103		-
1104		- ib_uverbs_init_udata(&uhw, buf + sizeof(cmd),
1105		- u64_to_user_ptr(cmd.response) + sizeof(resp),
1106		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1107		- out_len - sizeof(resp));
	1053	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1054	+ if (ret)
	1055	+ return ret;
1108	1056
1109	1057	memset(&cmd_ex, 0, sizeof(cmd_ex));
1110	1058	cmd_ex.user_handle = cmd.user_handle;
..	..	@@ -1112,43 +1060,17 @@
1112	1060	cmd_ex.comp_vector = cmd.comp_vector;
1113	1061	cmd_ex.comp_channel = cmd.comp_channel;
1114	1062
1115		- obj = create_cq(file, &ucore, &uhw, &cmd_ex,
1116		- offsetof(typeof(cmd_ex), comp_channel) +
1117		- sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb,
1118		- NULL);
1119		-
1120		- if (IS_ERR(obj))
1121		- return PTR_ERR(obj);
1122		-
1123		- return in_len;
	1063	+ return create_cq(attrs, &cmd_ex);
1124	1064	}
1125	1065
1126		-static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file,
1127		- struct ib_ucq_object *obj,
1128		- struct ib_uverbs_ex_create_cq_resp *resp,
1129		- struct ib_udata ucore, void context)
	1066	+static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
1130	1067	{
1131		- if (ib_copy_to_udata(ucore, resp, resp->response_length))
1132		- return -EFAULT;
1133		-
1134		- return 0;
1135		-}
1136		-
1137		-int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file,
1138		- struct ib_udata *ucore,
1139		- struct ib_udata *uhw)
1140		-{
1141		- struct ib_uverbs_ex_create_cq_resp resp;
1142	1068	struct ib_uverbs_ex_create_cq cmd;
1143		- struct ib_ucq_object *obj;
1144		- int err;
	1069	+ int ret;
1145	1070
1146		- if (ucore->inlen < sizeof(cmd))
1147		- return -EINVAL;
1148		-
1149		- err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
1150		- if (err)
1151		- return err;
	1071	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1072	+ if (ret)
	1073	+ return ret;
1152	1074
1153	1075	if (cmd.comp_mask)
1154	1076	return -EINVAL;
..	..	@@ -1156,52 +1078,36 @@
1156	1078	if (cmd.reserved)
1157	1079	return -EINVAL;
1158	1080
1159		- if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1160		- sizeof(resp.response_length)))
1161		- return -ENOSPC;
1162		-
1163		- obj = create_cq(file, ucore, uhw, &cmd,
1164		- min(ucore->inlen, sizeof(cmd)),
1165		- ib_uverbs_ex_create_cq_cb, NULL);
1166		-
1167		- return PTR_ERR_OR_ZERO(obj);
	1081	+ return create_cq(attrs, &cmd);
1168	1082	}
1169	1083
1170		-ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
1171		- const char __user *buf, int in_len,
1172		- int out_len)
	1084	+static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
1173	1085	{
1174	1086	struct ib_uverbs_resize_cq cmd;
1175	1087	struct ib_uverbs_resize_cq_resp resp = {};
1176		- struct ib_udata udata;
1177	1088	struct ib_cq *cq;
1178		- int ret = -EINVAL;
	1089	+ int ret;
1179	1090
1180		- if (copy_from_user(&cmd, buf, sizeof cmd))
1181		- return -EFAULT;
	1091	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1092	+ if (ret)
	1093	+ return ret;
1182	1094
1183		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
1184		- u64_to_user_ptr(cmd.response) + sizeof(resp),
1185		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1186		- out_len - sizeof(resp));
1187		-
1188		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	1095	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1189	1096	if (!cq)
1190	1097	return -EINVAL;
1191	1098
1192		- ret = cq->device->resize_cq(cq, cmd.cqe, &udata);
	1099	+ ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);
1193	1100	if (ret)
1194	1101	goto out;
1195	1102
1196	1103	resp.cqe = cq->cqe;
1197	1104
1198		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp.cqe))
1199		- ret = -EFAULT;
1200		-
	1105	+ ret = uverbs_response(attrs, &resp, sizeof(resp));
1201	1106	out:
1202		- uobj_put_obj_read(cq);
	1107	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	1108	+ UVERBS_LOOKUP_READ);
1203	1109
1204		- return ret ? ret : in_len;
	1110	+ return ret;
1205	1111	}
1206	1112
1207	1113	static int copy_wc_to_user(struct ib_device ib_dev, void __user dest,
..	..	@@ -1234,9 +1140,7 @@
1234	1140	return 0;
1235	1141	}
1236	1142
1237		-ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
1238		- const char __user *buf, int in_len,
1239		- int out_len)
	1143	+static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
1240	1144	{
1241	1145	struct ib_uverbs_poll_cq cmd;
1242	1146	struct ib_uverbs_poll_cq_resp resp;
..	..	@@ -1246,15 +1150,16 @@
1246	1150	struct ib_wc wc;
1247	1151	int ret;
1248	1152
1249		- if (copy_from_user(&cmd, buf, sizeof cmd))
1250		- return -EFAULT;
	1153	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1154	+ if (ret)
	1155	+ return ret;
1251	1156
1252		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	1157	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1253	1158	if (!cq)
1254	1159	return -EINVAL;
1255	1160
1256	1161	/* we copy a struct ib_uverbs_poll_cq_resp to user space */
1257		- header_ptr = u64_to_user_ptr(cmd.response);
	1162	+ header_ptr = attrs->ucore.outbuf;
1258	1163	data_ptr = header_ptr + sizeof resp;
1259	1164
1260	1165	memset(&resp, 0, sizeof resp);
..	..	@@ -1277,74 +1182,67 @@
1277	1182	ret = -EFAULT;
1278	1183	goto out_put;
1279	1184	}
	1185	+ ret = 0;
1280	1186
1281		- ret = in_len;
	1187	+ if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
	1188	+ ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);
1282	1189
1283	1190	out_put:
1284		- uobj_put_obj_read(cq);
	1191	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	1192	+ UVERBS_LOOKUP_READ);
1285	1193	return ret;
1286	1194	}
1287	1195
1288		-ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file,
1289		- const char __user *buf, int in_len,
1290		- int out_len)
	1196	+static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)
1291	1197	{
1292	1198	struct ib_uverbs_req_notify_cq cmd;
1293	1199	struct ib_cq *cq;
	1200	+ int ret;
1294	1201
1295		- if (copy_from_user(&cmd, buf, sizeof cmd))
1296		- return -EFAULT;
	1202	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1203	+ if (ret)
	1204	+ return ret;
1297	1205
1298		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	1206	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1299	1207	if (!cq)
1300	1208	return -EINVAL;
1301	1209
1302	1210	ib_req_notify_cq(cq, cmd.solicited_only ?
1303	1211	IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1304	1212
1305		- uobj_put_obj_read(cq);
1306		-
1307		- return in_len;
	1213	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	1214	+ UVERBS_LOOKUP_READ);
	1215	+ return 0;
1308	1216	}
1309	1217
1310		-ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
1311		- const char __user *buf, int in_len,
1312		- int out_len)
	1218	+static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)
1313	1219	{
1314	1220	struct ib_uverbs_destroy_cq cmd;
1315	1221	struct ib_uverbs_destroy_cq_resp resp;
1316	1222	struct ib_uobject *uobj;
1317	1223	struct ib_ucq_object *obj;
	1224	+ int ret;
1318	1225
1319		- if (copy_from_user(&cmd, buf, sizeof cmd))
1320		- return -EFAULT;
	1226	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1227	+ if (ret)
	1228	+ return ret;
1321	1229
1322		- uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	1230	+ uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1323	1231	if (IS_ERR(uobj))
1324	1232	return PTR_ERR(uobj);
1325	1233
1326		- obj = container_of(uobj, struct ib_ucq_object, uobject);
	1234	+ obj = container_of(uobj, struct ib_ucq_object, uevent.uobject);
1327	1235	memset(&resp, 0, sizeof(resp));
1328	1236	resp.comp_events_reported = obj->comp_events_reported;
1329		- resp.async_events_reported = obj->async_events_reported;
	1237	+ resp.async_events_reported = obj->uevent.events_reported;
1330	1238
1331	1239	uobj_put_destroy(uobj);
1332	1240
1333		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
1334		- return -EFAULT;
1335		-
1336		- return in_len;
	1241	+ return uverbs_response(attrs, &resp, sizeof(resp));
1337	1242	}
1338	1243
1339		-static int create_qp(struct ib_uverbs_file *file,
1340		- struct ib_udata *ucore,
1341		- struct ib_udata *uhw,
1342		- struct ib_uverbs_ex_create_qp *cmd,
1343		- size_t cmd_sz,
1344		- int (cb)(struct ib_uverbs_file file,
1345		- struct ib_uverbs_ex_create_qp_resp *resp,
1346		- struct ib_udata *udata),
1347		- void *context)
	1244	+static int create_qp(struct uverbs_attr_bundle *attrs,
	1245	+ struct ib_uverbs_ex_create_qp *cmd)
1348	1246	{
1349	1247	struct ib_uqp_object *obj;
1350	1248	struct ib_device *device;
..	..	@@ -1354,18 +1252,30 @@
1354	1252	struct ib_cq scq = NULL, rcq = NULL;
1355	1253	struct ib_srq *srq = NULL;
1356	1254	struct ib_qp *qp;
1357		- char *buf;
1358	1255	struct ib_qp_init_attr attr = {};
1359		- struct ib_uverbs_ex_create_qp_resp resp;
	1256	+ struct ib_uverbs_ex_create_qp_resp resp = {};
1360	1257	int ret;
1361	1258	struct ib_rwq_ind_table *ind_tbl = NULL;
1362	1259	bool has_sq = true;
1363	1260	struct ib_device *ib_dev;
1364	1261
1365		- if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
1366		- return -EPERM;
	1262	+ switch (cmd->qp_type) {
	1263	+ case IB_QPT_RAW_PACKET:
	1264	+ if (!capable(CAP_NET_RAW))
	1265	+ return -EPERM;
	1266	+ break;
	1267	+ case IB_QPT_RC:
	1268	+ case IB_QPT_UC:
	1269	+ case IB_QPT_UD:
	1270	+ case IB_QPT_XRC_INI:
	1271	+ case IB_QPT_XRC_TGT:
	1272	+ case IB_QPT_DRIVER:
	1273	+ break;
	1274	+ default:
	1275	+ return -EINVAL;
	1276	+ }
1367	1277
1368		- obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, file,
	1278	+ obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1369	1279	&ib_dev);
1370	1280	if (IS_ERR(obj))
1371	1281	return PTR_ERR(obj);
..	..	@@ -1373,25 +1283,16 @@
1373	1283	obj->uevent.uobject.user_handle = cmd->user_handle;
1374	1284	mutex_init(&obj->mcast_lock);
1375	1285
1376		- if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) +
1377		- sizeof(cmd->rwq_ind_tbl_handle) &&
1378		- (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) {
	1286	+ if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {
1379	1287	ind_tbl = uobj_get_obj_read(rwq_ind_table,
1380	1288	UVERBS_OBJECT_RWQ_IND_TBL,
1381		- cmd->rwq_ind_tbl_handle, file);
	1289	+ cmd->rwq_ind_tbl_handle, attrs);
1382	1290	if (!ind_tbl) {
1383	1291	ret = -EINVAL;
1384	1292	goto err_put;
1385	1293	}
1386	1294
1387	1295	attr.rwq_ind_tbl = ind_tbl;
1388		- }
1389		-
1390		- if (cmd_sz > sizeof(*cmd) &&
1391		- !ib_is_udata_cleared(ucore, sizeof(*cmd),
1392		- cmd_sz - sizeof(*cmd))) {
1393		- ret = -EOPNOTSUPP;
1394		- goto err_put;
1395	1296	}
1396	1297
1397	1298	if (ind_tbl && (cmd->max_recv_wr \|\| cmd->max_recv_sge \|\| cmd->is_srq)) {
..	..	@@ -1404,7 +1305,7 @@
1404	1305
1405	1306	if (cmd->qp_type == IB_QPT_XRC_TGT) {
1406	1307	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,
1407		- file);
	1308	+ attrs);
1408	1309
1409	1310	if (IS_ERR(xrcd_uobj)) {
1410	1311	ret = -EINVAL;
..	..	@@ -1424,7 +1325,7 @@
1424	1325	} else {
1425	1326	if (cmd->is_srq) {
1426	1327	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,
1427		- cmd->srq_handle, file);
	1328	+ cmd->srq_handle, attrs);
1428	1329	if (!srq \|\| srq->srq_type == IB_SRQT_XRC) {
1429	1330	ret = -EINVAL;
1430	1331	goto err_put;
..	..	@@ -1435,7 +1336,7 @@
1435	1336	if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1436	1337	rcq = uobj_get_obj_read(
1437	1338	cq, UVERBS_OBJECT_CQ,
1438		- cmd->recv_cq_handle, file);
	1339	+ cmd->recv_cq_handle, attrs);
1439	1340	if (!rcq) {
1440	1341	ret = -EINVAL;
1441	1342	goto err_put;
..	..	@@ -1446,11 +1347,11 @@
1446	1347
1447	1348	if (has_sq)
1448	1349	scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
1449		- cmd->send_cq_handle, file);
	1350	+ cmd->send_cq_handle, attrs);
1450	1351	if (!ind_tbl)
1451	1352	rcq = rcq ?: scq;
1452	1353	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,
1453		- file);
	1354	+ attrs);
1454	1355	if (!pd \|\| (!scq && has_sq)) {
1455	1356	ret = -EINVAL;
1456	1357	goto err_put;
..	..	@@ -1460,7 +1361,6 @@
1460	1361	}
1461	1362
1462	1363	attr.event_handler = ib_uverbs_qp_event_handler;
1463		- attr.qp_context = file;
1464	1364	attr.send_cq = scq;
1465	1365	attr.recv_cq = rcq;
1466	1366	attr.srq = srq;
..	..	@@ -1476,14 +1376,10 @@
1476	1376	attr.cap.max_recv_sge = cmd->max_recv_sge;
1477	1377	attr.cap.max_inline_data = cmd->max_inline_data;
1478	1378
1479		- obj->uevent.events_reported = 0;
1480	1379	INIT_LIST_HEAD(&obj->uevent.event_list);
1481	1380	INIT_LIST_HEAD(&obj->mcast_list);
1482	1381
1483		- if (cmd_sz >= offsetof(typeof(*cmd), create_flags) +
1484		- sizeof(cmd->create_flags))
1485		- attr.create_flags = cmd->create_flags;
1486		-
	1382	+ attr.create_flags = cmd->create_flags;
1487	1383	if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK \|
1488	1384	IB_QP_CREATE_CROSS_CHANNEL \|
1489	1385	IB_QP_CREATE_MANAGED_SEND \|
..	..	@@ -1505,19 +1401,11 @@
1505	1401	attr.source_qpn = cmd->source_qpn;
1506	1402	}
1507	1403
1508		- buf = (void )cmd + sizeof(cmd);
1509		- if (cmd_sz > sizeof(*cmd))
1510		- if (!(buf[0] == 0 && !memcmp(buf, buf + 1,
1511		- cmd_sz - sizeof(*cmd) - 1))) {
1512		- ret = -EINVAL;
1513		- goto err_put;
1514		- }
1515		-
1516	1404	if (cmd->qp_type == IB_QPT_XRC_TGT)
1517	1405	qp = ib_create_qp(pd, &attr);
1518	1406	else
1519		- qp = _ib_create_qp(device, pd, &attr, uhw,
1520		- &obj->uevent.uobject);
	1407	+ qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
	1408	+ obj);
1521	1409
1522	1410	if (IS_ERR(qp)) {
1523	1411	ret = PTR_ERR(qp);
..	..	@@ -1529,18 +1417,7 @@
1529	1417	if (ret)
1530	1418	goto err_cb;
1531	1419
1532		- qp->real_qp = qp;
1533		- qp->pd = pd;
1534		- qp->send_cq = attr.send_cq;
1535		- qp->recv_cq = attr.recv_cq;
1536		- qp->srq = attr.srq;
1537		- qp->rwq_ind_tbl = ind_tbl;
1538		- qp->event_handler = attr.event_handler;
1539		- qp->qp_context = attr.qp_context;
1540		- qp->qp_type = attr.qp_type;
1541		- atomic_set(&qp->usecnt, 0);
1542	1420	atomic_inc(&pd->usecnt);
1543		- qp->port = 0;
1544	1421	if (attr.send_cq)
1545	1422	atomic_inc(&attr.send_cq->usecnt);
1546	1423	if (attr.recv_cq)
..	..	@@ -1551,26 +1428,13 @@
1551	1428	atomic_inc(&ind_tbl->usecnt);
1552	1429	} else {
1553	1430	/* It is done in _ib_create_qp for other QP types */
1554		- qp->uobject = &obj->uevent.uobject;
	1431	+ qp->uobject = obj;
1555	1432	}
1556	1433
1557	1434	obj->uevent.uobject.object = qp;
1558		-
1559		- memset(&resp, 0, sizeof resp);
1560		- resp.base.qpn = qp->qp_num;
1561		- resp.base.qp_handle = obj->uevent.uobject.id;
1562		- resp.base.max_recv_sge = attr.cap.max_recv_sge;
1563		- resp.base.max_send_sge = attr.cap.max_send_sge;
1564		- resp.base.max_recv_wr = attr.cap.max_recv_wr;
1565		- resp.base.max_send_wr = attr.cap.max_send_wr;
1566		- resp.base.max_inline_data = attr.cap.max_inline_data;
1567		-
1568		- resp.response_length = offsetof(typeof(resp), response_length) +
1569		- sizeof(resp.response_length);
1570		-
1571		- ret = cb(file, &resp, ucore);
1572		- if (ret)
1573		- goto err_cb;
	1435	+ obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
	1436	+ if (obj->uevent.event_file)
	1437	+ uverbs_uobject_get(&obj->uevent.event_file->uobj);
1574	1438
1575	1439	if (xrcd) {
1576	1440	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
..	..	@@ -1582,17 +1446,30 @@
1582	1446	if (pd)
1583	1447	uobj_put_obj_read(pd);
1584	1448	if (scq)
1585		- uobj_put_obj_read(scq);
	1449	+ rdma_lookup_put_uobject(&scq->uobject->uevent.uobject,
	1450	+ UVERBS_LOOKUP_READ);
1586	1451	if (rcq && rcq != scq)
1587		- uobj_put_obj_read(rcq);
	1452	+ rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject,
	1453	+ UVERBS_LOOKUP_READ);
1588	1454	if (srq)
1589		- uobj_put_obj_read(srq);
	1455	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	1456	+ UVERBS_LOOKUP_READ);
1590	1457	if (ind_tbl)
1591	1458	uobj_put_obj_read(ind_tbl);
	1459	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1592	1460
1593		- return uobj_alloc_commit(&obj->uevent.uobject, 0);
	1461	+ resp.base.qpn = qp->qp_num;
	1462	+ resp.base.qp_handle = obj->uevent.uobject.id;
	1463	+ resp.base.max_recv_sge = attr.cap.max_recv_sge;
	1464	+ resp.base.max_send_sge = attr.cap.max_send_sge;
	1465	+ resp.base.max_recv_wr = attr.cap.max_recv_wr;
	1466	+ resp.base.max_send_wr = attr.cap.max_send_wr;
	1467	+ resp.base.max_inline_data = attr.cap.max_inline_data;
	1468	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	1469	+ return uverbs_response(attrs, &resp, sizeof(resp));
	1470	+
1594	1471	err_cb:
1595		- ib_destroy_qp(qp);
	1472	+ ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
1596	1473
1597	1474	err_put:
1598	1475	if (!IS_ERR(xrcd_uobj))
..	..	@@ -1600,51 +1477,30 @@
1600	1477	if (pd)
1601	1478	uobj_put_obj_read(pd);
1602	1479	if (scq)
1603		- uobj_put_obj_read(scq);
	1480	+ rdma_lookup_put_uobject(&scq->uobject->uevent.uobject,
	1481	+ UVERBS_LOOKUP_READ);
1604	1482	if (rcq && rcq != scq)
1605		- uobj_put_obj_read(rcq);
	1483	+ rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject,
	1484	+ UVERBS_LOOKUP_READ);
1606	1485	if (srq)
1607		- uobj_put_obj_read(srq);
	1486	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	1487	+ UVERBS_LOOKUP_READ);
1608	1488	if (ind_tbl)
1609	1489	uobj_put_obj_read(ind_tbl);
1610	1490
1611		- uobj_alloc_abort(&obj->uevent.uobject);
	1491	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
1612	1492	return ret;
1613	1493	}
1614	1494
1615		-static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file,
1616		- struct ib_uverbs_ex_create_qp_resp *resp,
1617		- struct ib_udata *ucore)
1618		-{
1619		- if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1620		- return -EFAULT;
1621		-
1622		- return 0;
1623		-}
1624		-
1625		-ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
1626		- const char __user *buf, int in_len,
1627		- int out_len)
	1495	+static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)
1628	1496	{
1629	1497	struct ib_uverbs_create_qp cmd;
1630	1498	struct ib_uverbs_ex_create_qp cmd_ex;
1631		- struct ib_udata ucore;
1632		- struct ib_udata uhw;
1633		- ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp);
1634		- int err;
	1499	+ int ret;
1635	1500
1636		- if (out_len < resp_size)
1637		- return -ENOSPC;
1638		-
1639		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
1640		- return -EFAULT;
1641		-
1642		- ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response),
1643		- sizeof(cmd), resp_size);
1644		- ib_uverbs_init_udata(&uhw, buf + sizeof(cmd),
1645		- u64_to_user_ptr(cmd.response) + resp_size,
1646		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1647		- out_len - resp_size);
	1501	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1502	+ if (ret)
	1503	+ return ret;
1648	1504
1649	1505	memset(&cmd_ex, 0, sizeof(cmd_ex));
1650	1506	cmd_ex.user_handle = cmd.user_handle;
..	..	@@ -1661,42 +1517,17 @@
1661	1517	cmd_ex.qp_type = cmd.qp_type;
1662	1518	cmd_ex.is_srq = cmd.is_srq;
1663	1519
1664		- err = create_qp(file, &ucore, &uhw, &cmd_ex,
1665		- offsetof(typeof(cmd_ex), is_srq) +
1666		- sizeof(cmd.is_srq), ib_uverbs_create_qp_cb,
1667		- NULL);
1668		-
1669		- if (err)
1670		- return err;
1671		-
1672		- return in_len;
	1520	+ return create_qp(attrs, &cmd_ex);
1673	1521	}
1674	1522
1675		-static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file,
1676		- struct ib_uverbs_ex_create_qp_resp *resp,
1677		- struct ib_udata *ucore)
	1523	+static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)
1678	1524	{
1679		- if (ib_copy_to_udata(ucore, resp, resp->response_length))
1680		- return -EFAULT;
	1525	+ struct ib_uverbs_ex_create_qp cmd;
	1526	+ int ret;
1681	1527
1682		- return 0;
1683		-}
1684		-
1685		-int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file,
1686		- struct ib_udata *ucore,
1687		- struct ib_udata *uhw)
1688		-{
1689		- struct ib_uverbs_ex_create_qp_resp resp;
1690		- struct ib_uverbs_ex_create_qp cmd = {0};
1691		- int err;
1692		-
1693		- if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) +
1694		- sizeof(cmd.comp_mask)))
1695		- return -EINVAL;
1696		-
1697		- err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
1698		- if (err)
1699		- return err;
	1528	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1529	+ if (ret)
	1530	+ return ret;
1700	1531
1701	1532	if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1702	1533	return -EINVAL;
..	..	@@ -1704,51 +1535,31 @@
1704	1535	if (cmd.reserved)
1705	1536	return -EINVAL;
1706	1537
1707		- if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1708		- sizeof(resp.response_length)))
1709		- return -ENOSPC;
1710		-
1711		- err = create_qp(file, ucore, uhw, &cmd,
1712		- min(ucore->inlen, sizeof(cmd)),
1713		- ib_uverbs_ex_create_qp_cb, NULL);
1714		-
1715		- if (err)
1716		- return err;
1717		-
1718		- return 0;
	1538	+ return create_qp(attrs, &cmd);
1719	1539	}
1720	1540
1721		-ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
1722		- const char __user *buf, int in_len, int out_len)
	1541	+static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
1723	1542	{
	1543	+ struct ib_uverbs_create_qp_resp resp = {};
1724	1544	struct ib_uverbs_open_qp cmd;
1725		- struct ib_uverbs_create_qp_resp resp;
1726		- struct ib_udata udata;
1727	1545	struct ib_uqp_object *obj;
1728	1546	struct ib_xrcd *xrcd;
1729		- struct ib_uobject *uninitialized_var(xrcd_uobj);
1730	1547	struct ib_qp *qp;
1731		- struct ib_qp_open_attr attr;
	1548	+ struct ib_qp_open_attr attr = {};
1732	1549	int ret;
	1550	+ struct ib_uobject *xrcd_uobj;
1733	1551	struct ib_device *ib_dev;
1734	1552
1735		- if (out_len < sizeof resp)
1736		- return -ENOSPC;
	1553	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1554	+ if (ret)
	1555	+ return ret;
1737	1556
1738		- if (copy_from_user(&cmd, buf, sizeof cmd))
1739		- return -EFAULT;
1740		-
1741		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
1742		- u64_to_user_ptr(cmd.response) + sizeof(resp),
1743		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1744		- out_len - sizeof(resp));
1745		-
1746		- obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, file,
	1557	+ obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1747	1558	&ib_dev);
1748	1559	if (IS_ERR(obj))
1749	1560	return PTR_ERR(obj);
1750	1561
1751		- xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, file);
	1562	+ xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);
1752	1563	if (IS_ERR(xrcd_uobj)) {
1753	1564	ret = -EINVAL;
1754	1565	goto err_put;
..	..	@@ -1761,11 +1572,9 @@
1761	1572	}
1762	1573
1763	1574	attr.event_handler = ib_uverbs_qp_event_handler;
1764		- attr.qp_context = file;
1765	1575	attr.qp_num = cmd.qpn;
1766	1576	attr.qp_type = cmd.qp_type;
1767	1577
1768		- obj->uevent.events_reported = 0;
1769	1578	INIT_LIST_HEAD(&obj->uevent.event_list);
1770	1579	INIT_LIST_HEAD(&obj->mcast_list);
1771	1580
..	..	@@ -1778,28 +1587,20 @@
1778	1587	obj->uevent.uobject.object = qp;
1779	1588	obj->uevent.uobject.user_handle = cmd.user_handle;
1780	1589
1781		- memset(&resp, 0, sizeof resp);
1782		- resp.qpn = qp->qp_num;
1783		- resp.qp_handle = obj->uevent.uobject.id;
1784		-
1785		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
1786		- ret = -EFAULT;
1787		- goto err_destroy;
1788		- }
1789		-
1790	1590	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1791	1591	atomic_inc(&obj->uxrcd->refcnt);
1792		- qp->uobject = &obj->uevent.uobject;
	1592	+ qp->uobject = obj;
1793	1593	uobj_put_read(xrcd_uobj);
	1594	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1794	1595
1795		- return uobj_alloc_commit(&obj->uevent.uobject, in_len);
	1596	+ resp.qpn = qp->qp_num;
	1597	+ resp.qp_handle = obj->uevent.uobject.id;
	1598	+ return uverbs_response(attrs, &resp, sizeof(resp));
1796	1599
1797		-err_destroy:
1798		- ib_destroy_qp(qp);
1799	1600	err_xrcd:
1800	1601	uobj_put_read(xrcd_uobj);
1801	1602	err_put:
1802		- uobj_alloc_abort(&obj->uevent.uobject);
	1603	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
1803	1604	return ret;
1804	1605	}
1805	1606
..	..	@@ -1825,9 +1626,7 @@
1825	1626	uverb_attr->port_num = rdma_ah_get_port_num(rdma_attr);
1826	1627	}
1827	1628
1828		-ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
1829		- const char __user *buf, int in_len,
1830		- int out_len)
	1629	+static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)
1831	1630	{
1832	1631	struct ib_uverbs_query_qp cmd;
1833	1632	struct ib_uverbs_query_qp_resp resp;
..	..	@@ -1836,8 +1635,9 @@
1836	1635	struct ib_qp_init_attr *init_attr;
1837	1636	int ret;
1838	1637
1839		- if (copy_from_user(&cmd, buf, sizeof cmd))
1840		- return -EFAULT;
	1638	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1639	+ if (ret)
	1640	+ return ret;
1841	1641
1842	1642	attr = kmalloc(sizeof *attr, GFP_KERNEL);
1843	1643	init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
..	..	@@ -1846,7 +1646,7 @@
1846	1646	goto out;
1847	1647	}
1848	1648
1849		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
	1649	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1850	1650	if (!qp) {
1851	1651	ret = -EINVAL;
1852	1652	goto out;
..	..	@@ -1854,7 +1654,8 @@
1854	1654
1855	1655	ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1856	1656
1857		- uobj_put_obj_read(qp);
	1657	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	1658	+ UVERBS_LOOKUP_READ);
1858	1659
1859	1660	if (ret)
1860	1661	goto out;
..	..	@@ -1893,14 +1694,13 @@
1893	1694	resp.max_inline_data = init_attr->cap.max_inline_data;
1894	1695	resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1895	1696
1896		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
1897		- ret = -EFAULT;
	1697	+ ret = uverbs_response(attrs, &resp, sizeof(resp));
1898	1698
1899	1699	out:
1900	1700	kfree(attr);
1901	1701	kfree(init_attr);
1902	1702
1903		- return ret ? ret : in_len;
	1703	+ return ret;
1904	1704	}
1905	1705
1906	1706	/* Remove ignored fields set in the attribute mask */
..	..	@@ -1940,8 +1740,8 @@
1940	1740	rdma_ah_set_make_grd(rdma_attr, false);
1941	1741	}
1942	1742
1943		-static int modify_qp(struct ib_uverbs_file *file,
1944		- struct ib_uverbs_ex_modify_qp cmd, struct ib_udata udata)
	1743	+static int modify_qp(struct uverbs_attr_bundle *attrs,
	1744	+ struct ib_uverbs_ex_modify_qp *cmd)
1945	1745	{
1946	1746	struct ib_qp_attr *attr;
1947	1747	struct ib_qp *qp;
..	..	@@ -1951,7 +1751,8 @@
1951	1751	if (!attr)
1952	1752	return -ENOMEM;
1953	1753
1954		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle, file);
	1754	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,
	1755	+ attrs);
1955	1756	if (!qp) {
1956	1757	ret = -EINVAL;
1957	1758	goto out;
..	..	@@ -2088,90 +1889,75 @@
2088	1889	ret = ib_modify_qp_with_udata(qp, attr,
2089	1890	modify_qp_mask(qp->qp_type,
2090	1891	cmd->base.attr_mask),
2091		- udata);
	1892	+ &attrs->driver_udata);
2092	1893
2093	1894	release_qp:
2094		- uobj_put_obj_read(qp);
	1895	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	1896	+ UVERBS_LOOKUP_READ);
2095	1897	out:
2096	1898	kfree(attr);
2097	1899
2098	1900	return ret;
2099	1901	}
2100	1902
2101		-ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
2102		- const char __user *buf, int in_len,
2103		- int out_len)
	1903	+static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)
2104	1904	{
2105		- struct ib_uverbs_ex_modify_qp cmd = {};
2106		- struct ib_udata udata;
	1905	+ struct ib_uverbs_ex_modify_qp cmd;
2107	1906	int ret;
2108	1907
2109		- if (copy_from_user(&cmd.base, buf, sizeof(cmd.base)))
2110		- return -EFAULT;
	1908	+ ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));
	1909	+ if (ret)
	1910	+ return ret;
2111	1911
2112	1912	if (cmd.base.attr_mask &
2113	1913	~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
2114	1914	return -EOPNOTSUPP;
2115	1915
2116		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd.base), NULL,
2117		- in_len - sizeof(cmd.base) - sizeof(struct ib_uverbs_cmd_hdr),
2118		- out_len);
2119		-
2120		- ret = modify_qp(file, &cmd, &udata);
2121		- if (ret)
2122		- return ret;
2123		-
2124		- return in_len;
	1916	+ return modify_qp(attrs, &cmd);
2125	1917	}
2126	1918
2127		-int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file,
2128		- struct ib_udata *ucore,
2129		- struct ib_udata *uhw)
	1919	+static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)
2130	1920	{
2131		- struct ib_uverbs_ex_modify_qp cmd = {};
	1921	+ struct ib_uverbs_ex_modify_qp cmd;
	1922	+ struct ib_uverbs_ex_modify_qp_resp resp = {
	1923	+ .response_length = uverbs_response_length(attrs, sizeof(resp))
	1924	+ };
2132	1925	int ret;
	1926	+
	1927	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1928	+ if (ret)
	1929	+ return ret;
2133	1930
2134	1931	/*
2135	1932	* Last bit is reserved for extending the attr_mask by
2136	1933	* using another field.
2137	1934	*/
2138		- BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
2139		-
2140		- if (ucore->inlen < sizeof(cmd.base))
2141		- return -EINVAL;
2142		-
2143		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
2144		- if (ret)
2145		- return ret;
	1935	+ BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1ULL << 31));
2146	1936
2147	1937	if (cmd.base.attr_mask &
2148	1938	~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
2149	1939	return -EOPNOTSUPP;
2150	1940
2151		- if (ucore->inlen > sizeof(cmd)) {
2152		- if (!ib_is_udata_cleared(ucore, sizeof(cmd),
2153		- ucore->inlen - sizeof(cmd)))
2154		- return -EOPNOTSUPP;
2155		- }
	1941	+ ret = modify_qp(attrs, &cmd);
	1942	+ if (ret)
	1943	+ return ret;
2156	1944
2157		- ret = modify_qp(file, &cmd, uhw);
2158		-
2159		- return ret;
	1945	+ return uverbs_response(attrs, &resp, sizeof(resp));
2160	1946	}
2161	1947
2162		-ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
2163		- const char __user *buf, int in_len,
2164		- int out_len)
	1948	+static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)
2165	1949	{
2166	1950	struct ib_uverbs_destroy_qp cmd;
2167	1951	struct ib_uverbs_destroy_qp_resp resp;
2168	1952	struct ib_uobject *uobj;
2169	1953	struct ib_uqp_object *obj;
	1954	+ int ret;
2170	1955
2171		- if (copy_from_user(&cmd, buf, sizeof cmd))
2172		- return -EFAULT;
	1956	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1957	+ if (ret)
	1958	+ return ret;
2173	1959
2174		- uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, file);
	1960	+ uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2175	1961	if (IS_ERR(uobj))
2176	1962	return PTR_ERR(uobj);
2177	1963
..	..	@@ -2181,10 +1967,7 @@
2181	1967
2182	1968	uobj_put_destroy(uobj);
2183	1969
2184		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2185		- return -EFAULT;
2186		-
2187		- return in_len;
	1970	+ return uverbs_response(attrs, &resp, sizeof(resp));
2188	1971	}
2189	1972
2190	1973	static void *alloc_wr(size_t wr_size, __u32 num_sge)
..	..	@@ -2197,9 +1980,7 @@
2197	1980	num_sge * sizeof (struct ib_sge), GFP_KERNEL);
2198	1981	}
2199	1982
2200		-ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
2201		- const char __user *buf, int in_len,
2202		- int out_len)
	1983	+static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)
2203	1984	{
2204	1985	struct ib_uverbs_post_send cmd;
2205	1986	struct ib_uverbs_post_send_resp resp;
..	..	@@ -2209,33 +1990,41 @@
2209	1990	struct ib_qp *qp;
2210	1991	int i, sg_ind;
2211	1992	int is_ud;
2212		- ssize_t ret = -EINVAL;
	1993	+ int ret, ret2;
2213	1994	size_t next_size;
	1995	+ const struct ib_sge __user *sgls;
	1996	+ const void __user *wqes;
	1997	+ struct uverbs_req_iter iter;
2214	1998
2215		- if (copy_from_user(&cmd, buf, sizeof cmd))
2216		- return -EFAULT;
2217		-
2218		- if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count +
2219		- cmd.sge_count * sizeof (struct ib_uverbs_sge))
2220		- return -EINVAL;
2221		-
2222		- if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr))
2223		- return -EINVAL;
	1999	+ ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
	2000	+ if (ret)
	2001	+ return ret;
	2002	+ wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count);
	2003	+ if (IS_ERR(wqes))
	2004	+ return PTR_ERR(wqes);
	2005	+ sgls = uverbs_request_next_ptr(
	2006	+ &iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));
	2007	+ if (IS_ERR(sgls))
	2008	+ return PTR_ERR(sgls);
	2009	+ ret = uverbs_request_finish(&iter);
	2010	+ if (ret)
	2011	+ return ret;
2224	2012
2225	2013	user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2226	2014	if (!user_wr)
2227	2015	return -ENOMEM;
2228	2016
2229		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
2230		- if (!qp)
	2017	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
	2018	+ if (!qp) {
	2019	+ ret = -EINVAL;
2231	2020	goto out;
	2021	+ }
2232	2022
2233	2023	is_ud = qp->qp_type == IB_QPT_UD;
2234	2024	sg_ind = 0;
2235	2025	last = NULL;
2236	2026	for (i = 0; i < cmd.wr_count; ++i) {
2237		- if (copy_from_user(user_wr,
2238		- buf + sizeof cmd + i * cmd.wqe_size,
	2027	+ if (copy_from_user(user_wr, wqes + i * cmd.wqe_size,
2239	2028	cmd.wqe_size)) {
2240	2029	ret = -EFAULT;
2241	2030	goto out_put;
..	..	@@ -2263,7 +2052,7 @@
2263	2052	}
2264	2053
2265	2054	ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,
2266		- user_wr->wr.ud.ah, file);
	2055	+ user_wr->wr.ud.ah, attrs);
2267	2056	if (!ud->ah) {
2268	2057	kfree(ud);
2269	2058	ret = -EINVAL;
..	..	@@ -2343,11 +2132,9 @@
2343	2132	if (next->num_sge) {
2344	2133	next->sg_list = (void *) next +
2345	2134	ALIGN(next_size, sizeof(struct ib_sge));
2346		- if (copy_from_user(next->sg_list,
2347		- buf + sizeof cmd +
2348		- cmd.wr_count * cmd.wqe_size +
2349		- sg_ind * sizeof (struct ib_sge),
2350		- next->num_sge * sizeof (struct ib_sge))) {
	2135	+ if (copy_from_user(next->sg_list, sgls + sg_ind,
	2136	+ next->num_sge *
	2137	+ sizeof(struct ib_sge))) {
2351	2138	ret = -EFAULT;
2352	2139	goto out_put;
2353	2140	}
..	..	@@ -2357,7 +2144,7 @@
2357	2144	}
2358	2145
2359	2146	resp.bad_wr = 0;
2360		- ret = qp->device->post_send(qp->real_qp, wr, &bad_wr);
	2147	+ ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);
2361	2148	if (ret)
2362	2149	for (next = wr; next; next = next->next) {
2363	2150	++resp.bad_wr;
..	..	@@ -2365,11 +2152,13 @@
2365	2152	break;
2366	2153	}
2367	2154
2368		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2369		- ret = -EFAULT;
	2155	+ ret2 = uverbs_response(attrs, &resp, sizeof(resp));
	2156	+ if (ret2)
	2157	+ ret = ret2;
2370	2158
2371	2159	out_put:
2372		- uobj_put_obj_read(qp);
	2160	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	2161	+ UVERBS_LOOKUP_READ);
2373	2162
2374	2163	while (wr) {
2375	2164	if (is_ud && ud_wr(wr)->ah)
..	..	@@ -2382,27 +2171,34 @@
2382	2171	out:
2383	2172	kfree(user_wr);
2384	2173
2385		- return ret ? ret : in_len;
	2174	+ return ret;
2386	2175	}
2387	2176
2388		-static struct ib_recv_wr ib_uverbs_unmarshall_recv(const char __user buf,
2389		- int in_len,
2390		- u32 wr_count,
2391		- u32 sge_count,
2392		- u32 wqe_size)
	2177	+static struct ib_recv_wr *
	2178	+ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,
	2179	+ u32 wqe_size, u32 sge_count)
2393	2180	{
2394	2181	struct ib_uverbs_recv_wr *user_wr;
2395	2182	struct ib_recv_wr wr = NULL, last, *next;
2396	2183	int sg_ind;
2397	2184	int i;
2398	2185	int ret;
2399		-
2400		- if (in_len < wqe_size * wr_count +
2401		- sge_count * sizeof (struct ib_uverbs_sge))
2402		- return ERR_PTR(-EINVAL);
	2186	+ const struct ib_sge __user *sgls;
	2187	+ const void __user *wqes;
2403	2188
2404	2189	if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2405	2190	return ERR_PTR(-EINVAL);
	2191	+
	2192	+ wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);
	2193	+ if (IS_ERR(wqes))
	2194	+ return ERR_CAST(wqes);
	2195	+ sgls = uverbs_request_next_ptr(
	2196	+ iter, sge_count * sizeof(struct ib_uverbs_sge));
	2197	+ if (IS_ERR(sgls))
	2198	+ return ERR_CAST(sgls);
	2199	+ ret = uverbs_request_finish(iter);
	2200	+ if (ret)
	2201	+ return ERR_PTR(ret);
2406	2202
2407	2203	user_wr = kmalloc(wqe_size, GFP_KERNEL);
2408	2204	if (!user_wr)
..	..	@@ -2411,7 +2207,7 @@
2411	2207	sg_ind = 0;
2412	2208	last = NULL;
2413	2209	for (i = 0; i < wr_count; ++i) {
2414		- if (copy_from_user(user_wr, buf + i * wqe_size,
	2210	+ if (copy_from_user(user_wr, wqes + i * wqe_size,
2415	2211	wqe_size)) {
2416	2212	ret = -EFAULT;
2417	2213	goto err;
..	..	@@ -2450,10 +2246,9 @@
2450	2246	if (next->num_sge) {
2451	2247	next->sg_list = (void *) next +
2452	2248	ALIGN(sizeof *next, sizeof (struct ib_sge));
2453		- if (copy_from_user(next->sg_list,
2454		- buf + wr_count * wqe_size +
2455		- sg_ind * sizeof (struct ib_sge),
2456		- next->num_sge * sizeof (struct ib_sge))) {
	2249	+ if (copy_from_user(next->sg_list, sgls + sg_ind,
	2250	+ next->num_sge *
	2251	+ sizeof(struct ib_sge))) {
2457	2252	ret = -EFAULT;
2458	2253	goto err;
2459	2254	}
..	..	@@ -2477,34 +2272,36 @@
2477	2272	return ERR_PTR(ret);
2478	2273	}
2479	2274
2480		-ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file,
2481		- const char __user *buf, int in_len,
2482		- int out_len)
	2275	+static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)
2483	2276	{
2484	2277	struct ib_uverbs_post_recv cmd;
2485	2278	struct ib_uverbs_post_recv_resp resp;
2486	2279	struct ib_recv_wr wr, next;
2487	2280	const struct ib_recv_wr *bad_wr;
2488	2281	struct ib_qp *qp;
2489		- ssize_t ret = -EINVAL;
	2282	+ int ret, ret2;
	2283	+ struct uverbs_req_iter iter;
2490	2284
2491		- if (copy_from_user(&cmd, buf, sizeof cmd))
2492		- return -EFAULT;
	2285	+ ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
	2286	+ if (ret)
	2287	+ return ret;
2493	2288
2494		- wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2495		- in_len - sizeof cmd, cmd.wr_count,
2496		- cmd.sge_count, cmd.wqe_size);
	2289	+ wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
	2290	+ cmd.sge_count);
2497	2291	if (IS_ERR(wr))
2498	2292	return PTR_ERR(wr);
2499	2293
2500		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
2501		- if (!qp)
	2294	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
	2295	+ if (!qp) {
	2296	+ ret = -EINVAL;
2502	2297	goto out;
	2298	+ }
2503	2299
2504	2300	resp.bad_wr = 0;
2505		- ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr);
	2301	+ ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);
2506	2302
2507		- uobj_put_obj_read(qp);
	2303	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	2304	+ UVERBS_LOOKUP_READ);
2508	2305	if (ret) {
2509	2306	for (next = wr; next; next = next->next) {
2510	2307	++resp.bad_wr;
..	..	@@ -2513,9 +2310,9 @@
2513	2310	}
2514	2311	}
2515	2312
2516		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2517		- ret = -EFAULT;
2518		-
	2313	+ ret2 = uverbs_response(attrs, &resp, sizeof(resp));
	2314	+ if (ret2)
	2315	+ ret = ret2;
2519	2316	out:
2520	2317	while (wr) {
2521	2318	next = wr->next;
..	..	@@ -2523,38 +2320,39 @@
2523	2320	wr = next;
2524	2321	}
2525	2322
2526		- return ret ? ret : in_len;
	2323	+ return ret;
2527	2324	}
2528	2325
2529		-ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file,
2530		- const char __user *buf, int in_len,
2531		- int out_len)
	2326	+static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)
2532	2327	{
2533	2328	struct ib_uverbs_post_srq_recv cmd;
2534	2329	struct ib_uverbs_post_srq_recv_resp resp;
2535	2330	struct ib_recv_wr wr, next;
2536	2331	const struct ib_recv_wr *bad_wr;
2537	2332	struct ib_srq *srq;
2538		- ssize_t ret = -EINVAL;
	2333	+ int ret, ret2;
	2334	+ struct uverbs_req_iter iter;
2539	2335
2540		- if (copy_from_user(&cmd, buf, sizeof cmd))
2541		- return -EFAULT;
	2336	+ ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
	2337	+ if (ret)
	2338	+ return ret;
2542	2339
2543		- wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2544		- in_len - sizeof cmd, cmd.wr_count,
2545		- cmd.sge_count, cmd.wqe_size);
	2340	+ wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
	2341	+ cmd.sge_count);
2546	2342	if (IS_ERR(wr))
2547	2343	return PTR_ERR(wr);
2548	2344
2549		- srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file);
2550		- if (!srq)
	2345	+ srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
	2346	+ if (!srq) {
	2347	+ ret = -EINVAL;
2551	2348	goto out;
	2349	+ }
2552	2350
2553	2351	resp.bad_wr = 0;
2554		- ret = srq->device->post_srq_recv ?
2555		- srq->device->post_srq_recv(srq, wr, &bad_wr) : -EOPNOTSUPP;
	2352	+ ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);
2556	2353
2557		- uobj_put_obj_read(srq);
	2354	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	2355	+ UVERBS_LOOKUP_READ);
2558	2356
2559	2357	if (ret)
2560	2358	for (next = wr; next; next = next->next) {
..	..	@@ -2563,8 +2361,9 @@
2563	2361	break;
2564	2362	}
2565	2363
2566		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2567		- ret = -EFAULT;
	2364	+ ret2 = uverbs_response(attrs, &resp, sizeof(resp));
	2365	+ if (ret2)
	2366	+ ret = ret2;
2568	2367
2569	2368	out:
2570	2369	while (wr) {
..	..	@@ -2573,12 +2372,10 @@
2573	2372	wr = next;
2574	2373	}
2575	2374
2576		- return ret ? ret : in_len;
	2375	+ return ret;
2577	2376	}
2578	2377
2579		-ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
2580		- const char __user *buf, int in_len,
2581		- int out_len)
	2378	+static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
2582	2379	{
2583	2380	struct ib_uverbs_create_ah cmd;
2584	2381	struct ib_uverbs_create_ah_resp resp;
..	..	@@ -2587,21 +2384,13 @@
2587	2384	struct ib_ah *ah;
2588	2385	struct rdma_ah_attr attr = {};
2589	2386	int ret;
2590		- struct ib_udata udata;
2591	2387	struct ib_device *ib_dev;
2592	2388
2593		- if (out_len < sizeof resp)
2594		- return -ENOSPC;
	2389	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	2390	+ if (ret)
	2391	+ return ret;
2595	2392
2596		- if (copy_from_user(&cmd, buf, sizeof cmd))
2597		- return -EFAULT;
2598		-
2599		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
2600		- u64_to_user_ptr(cmd.response) + sizeof(resp),
2601		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
2602		- out_len - sizeof(resp));
2603		-
2604		- uobj = uobj_alloc(UVERBS_OBJECT_AH, file, &ib_dev);
	2393	+ uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);
2605	2394	if (IS_ERR(uobj))
2606	2395	return PTR_ERR(uobj);
2607	2396
..	..	@@ -2610,7 +2399,7 @@
2610	2399	goto err;
2611	2400	}
2612	2401
2613		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file);
	2402	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2614	2403	if (!pd) {
2615	2404	ret = -EINVAL;
2616	2405	goto err;
..	..	@@ -2634,7 +2423,7 @@
2634	2423	rdma_ah_set_ah_flags(&attr, 0);
2635	2424	}
2636	2425
2637		- ah = rdma_create_user_ah(pd, &attr, &udata);
	2426	+ ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);
2638	2427	if (IS_ERR(ah)) {
2639	2428	ret = PTR_ERR(ah);
2640	2429	goto err_put;
..	..	@@ -2643,43 +2432,32 @@
2643	2432	ah->uobject = uobj;
2644	2433	uobj->user_handle = cmd.user_handle;
2645	2434	uobj->object = ah;
	2435	+ uobj_put_obj_read(pd);
	2436	+ uobj_finalize_uobj_create(uobj, attrs);
2646	2437
2647	2438	resp.ah_handle = uobj->id;
2648		-
2649		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
2650		- ret = -EFAULT;
2651		- goto err_copy;
2652		- }
2653		-
2654		- uobj_put_obj_read(pd);
2655		- return uobj_alloc_commit(uobj, in_len);
2656		-
2657		-err_copy:
2658		- rdma_destroy_ah(ah);
	2439	+ return uverbs_response(attrs, &resp, sizeof(resp));
2659	2440
2660	2441	err_put:
2661	2442	uobj_put_obj_read(pd);
2662		-
2663	2443	err:
2664		- uobj_alloc_abort(uobj);
	2444	+ uobj_alloc_abort(uobj, attrs);
2665	2445	return ret;
2666	2446	}
2667	2447
2668		-ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file,
2669		- const char __user *buf, int in_len, int out_len)
	2448	+static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)
2670	2449	{
2671	2450	struct ib_uverbs_destroy_ah cmd;
	2451	+ int ret;
2672	2452
2673		- if (copy_from_user(&cmd, buf, sizeof cmd))
2674		- return -EFAULT;
	2453	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	2454	+ if (ret)
	2455	+ return ret;
2675	2456
2676		- return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, file,
2677		- in_len);
	2457	+ return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);
2678	2458	}
2679	2459
2680		-ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file,
2681		- const char __user *buf, int in_len,
2682		- int out_len)
	2460	+static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)
2683	2461	{
2684	2462	struct ib_uverbs_attach_mcast cmd;
2685	2463	struct ib_qp *qp;
..	..	@@ -2687,14 +2465,15 @@
2687	2465	struct ib_uverbs_mcast_entry *mcast;
2688	2466	int ret;
2689	2467
2690		- if (copy_from_user(&cmd, buf, sizeof cmd))
2691		- return -EFAULT;
	2468	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	2469	+ if (ret)
	2470	+ return ret;
2692	2471
2693		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
	2472	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2694	2473	if (!qp)
2695	2474	return -EINVAL;
2696	2475
2697		- obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
	2476	+ obj = qp->uobject;
2698	2477
2699	2478	mutex_lock(&obj->mcast_lock);
2700	2479	list_for_each_entry(mcast, &obj->mcast_list, list)
..	..	@@ -2721,30 +2500,30 @@
2721	2500
2722	2501	out_put:
2723	2502	mutex_unlock(&obj->mcast_lock);
2724		- uobj_put_obj_read(qp);
	2503	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	2504	+ UVERBS_LOOKUP_READ);
2725	2505
2726		- return ret ? ret : in_len;
	2506	+ return ret;
2727	2507	}
2728	2508
2729		-ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file,
2730		- const char __user *buf, int in_len,
2731		- int out_len)
	2509	+static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
2732	2510	{
2733	2511	struct ib_uverbs_detach_mcast cmd;
2734	2512	struct ib_uqp_object *obj;
2735	2513	struct ib_qp *qp;
2736	2514	struct ib_uverbs_mcast_entry *mcast;
2737		- int ret = -EINVAL;
	2515	+ int ret;
2738	2516	bool found = false;
2739	2517
2740		- if (copy_from_user(&cmd, buf, sizeof cmd))
2741		- return -EFAULT;
	2518	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	2519	+ if (ret)
	2520	+ return ret;
2742	2521
2743		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
	2522	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2744	2523	if (!qp)
2745	2524	return -EINVAL;
2746	2525
2747		- obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
	2526	+ obj = qp->uobject;
2748	2527	mutex_lock(&obj->mcast_lock);
2749	2528
2750	2529	list_for_each_entry(mcast, &obj->mcast_list, list)
..	..	@@ -2765,20 +2544,12 @@
2765	2544
2766	2545	out_put:
2767	2546	mutex_unlock(&obj->mcast_lock);
2768		- uobj_put_obj_read(qp);
2769		- return ret ? ret : in_len;
	2547	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	2548	+ UVERBS_LOOKUP_READ);
	2549	+ return ret;
2770	2550	}
2771	2551
2772		-struct ib_uflow_resources {
2773		- size_t max;
2774		- size_t num;
2775		- size_t collection_num;
2776		- size_t counters_num;
2777		- struct ib_counters **counters;
2778		- struct ib_flow_action **collection;
2779		-};
2780		-
2781		-static struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
	2552	+struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
2782	2553	{
2783	2554	struct ib_uflow_resources *resources;
2784	2555
..	..	@@ -2808,6 +2579,7 @@
2808	2579
2809	2580	return NULL;
2810	2581	}
	2582	+EXPORT_SYMBOL(flow_resources_alloc);
2811	2583
2812	2584	void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
2813	2585	{
..	..	@@ -2826,10 +2598,11 @@
2826	2598	kfree(uflow_res->counters);
2827	2599	kfree(uflow_res);
2828	2600	}
	2601	+EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
2829	2602
2830		-static void flow_resources_add(struct ib_uflow_resources *uflow_res,
2831		- enum ib_flow_spec_type type,
2832		- void *ibobj)
	2603	+void flow_resources_add(struct ib_uflow_resources *uflow_res,
	2604	+ enum ib_flow_spec_type type,
	2605	+ void *ibobj)
2833	2606	{
2834	2607	WARN_ON(uflow_res->num >= uflow_res->max);
2835	2608
..	..	@@ -2850,8 +2623,9 @@
2850	2623
2851	2624	uflow_res->num++;
2852	2625	}
	2626	+EXPORT_SYMBOL(flow_resources_add);
2853	2627
2854		-static int kern_spec_to_ib_spec_action(struct ib_uverbs_file *ufile,
	2628	+static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,
2855	2629	struct ib_uverbs_flow_spec *kern_spec,
2856	2630	union ib_flow_spec *ib_spec,
2857	2631	struct ib_uflow_resources *uflow_res)
..	..	@@ -2880,7 +2654,7 @@
2880	2654	ib_spec->action.act = uobj_get_obj_read(flow_action,
2881	2655	UVERBS_OBJECT_FLOW_ACTION,
2882	2656	kern_spec->action.handle,
2883		- ufile);
	2657	+ attrs);
2884	2658	if (!ib_spec->action.act)
2885	2659	return -EINVAL;
2886	2660	ib_spec->action.size =
..	..	@@ -2898,7 +2672,7 @@
2898	2672	uobj_get_obj_read(counters,
2899	2673	UVERBS_OBJECT_COUNTERS,
2900	2674	kern_spec->flow_count.handle,
2901		- ufile);
	2675	+ attrs);
2902	2676	if (!ib_spec->flow_count.counters)
2903	2677	return -EINVAL;
2904	2678	ib_spec->flow_count.size =
..	..	@@ -3078,7 +2852,7 @@
3078	2852	kern_filter_sz, ib_spec);
3079	2853	}
3080	2854
3081		-static int kern_spec_to_ib_spec(struct ib_uverbs_file *ufile,
	2855	+static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,
3082	2856	struct ib_uverbs_flow_spec *kern_spec,
3083	2857	union ib_flow_spec *ib_spec,
3084	2858	struct ib_uflow_resources *uflow_res)
..	..	@@ -3087,17 +2861,15 @@
3087	2861	return -EINVAL;
3088	2862
3089	2863	if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
3090		- return kern_spec_to_ib_spec_action(ufile, kern_spec, ib_spec,
	2864	+ return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,
3091	2865	uflow_res);
3092	2866	else
3093	2867	return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
3094	2868	}
3095	2869
3096		-int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file,
3097		- struct ib_udata *ucore,
3098		- struct ib_udata *uhw)
	2870	+static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
3099	2871	{
3100		- struct ib_uverbs_ex_create_wq cmd = {};
	2872	+ struct ib_uverbs_ex_create_wq cmd;
3101	2873	struct ib_uverbs_ex_create_wq_resp resp = {};
3102	2874	struct ib_uwq_object *obj;
3103	2875	int err = 0;
..	..	@@ -3105,43 +2877,27 @@
3105	2877	struct ib_pd *pd;
3106	2878	struct ib_wq *wq;
3107	2879	struct ib_wq_init_attr wq_init_attr = {};
3108		- size_t required_cmd_sz;
3109		- size_t required_resp_len;
3110	2880	struct ib_device *ib_dev;
3111	2881
3112		- required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge);
3113		- required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn);
3114		-
3115		- if (ucore->inlen < required_cmd_sz)
3116		- return -EINVAL;
3117		-
3118		- if (ucore->outlen < required_resp_len)
3119		- return -ENOSPC;
3120		-
3121		- if (ucore->inlen > sizeof(cmd) &&
3122		- !ib_is_udata_cleared(ucore, sizeof(cmd),
3123		- ucore->inlen - sizeof(cmd)))
3124		- return -EOPNOTSUPP;
3125		-
3126		- err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	2882	+ err = uverbs_request(attrs, &cmd, sizeof(cmd));
3127	2883	if (err)
3128	2884	return err;
3129	2885
3130	2886	if (cmd.comp_mask)
3131	2887	return -EOPNOTSUPP;
3132	2888
3133		- obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, file,
	2889	+ obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,
3134	2890	&ib_dev);
3135	2891	if (IS_ERR(obj))
3136	2892	return PTR_ERR(obj);
3137	2893
3138		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file);
	2894	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
3139	2895	if (!pd) {
3140	2896	err = -EINVAL;
3141	2897	goto err_uobj;
3142	2898	}
3143	2899
3144		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	2900	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
3145	2901	if (!cq) {
3146	2902	err = -EINVAL;
3147	2903	goto err_put_pd;
..	..	@@ -3150,100 +2906,71 @@
3150	2906	wq_init_attr.cq = cq;
3151	2907	wq_init_attr.max_sge = cmd.max_sge;
3152	2908	wq_init_attr.max_wr = cmd.max_wr;
3153		- wq_init_attr.wq_context = file;
3154	2909	wq_init_attr.wq_type = cmd.wq_type;
3155	2910	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
3156		- if (ucore->inlen >= (offsetof(typeof(cmd), create_flags) +
3157		- sizeof(cmd.create_flags)))
3158		- wq_init_attr.create_flags = cmd.create_flags;
3159		- obj->uevent.events_reported = 0;
	2911	+ wq_init_attr.create_flags = cmd.create_flags;
3160	2912	INIT_LIST_HEAD(&obj->uevent.event_list);
	2913	+ obj->uevent.uobject.user_handle = cmd.user_handle;
3161	2914
3162		- if (!pd->device->create_wq) {
3163		- err = -EOPNOTSUPP;
3164		- goto err_put_cq;
3165		- }
3166		- wq = pd->device->create_wq(pd, &wq_init_attr, uhw);
	2915	+ wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
3167	2916	if (IS_ERR(wq)) {
3168	2917	err = PTR_ERR(wq);
3169	2918	goto err_put_cq;
3170	2919	}
3171	2920
3172		- wq->uobject = &obj->uevent.uobject;
	2921	+ wq->uobject = obj;
3173	2922	obj->uevent.uobject.object = wq;
3174	2923	wq->wq_type = wq_init_attr.wq_type;
3175	2924	wq->cq = cq;
3176	2925	wq->pd = pd;
3177	2926	wq->device = pd->device;
3178		- wq->wq_context = wq_init_attr.wq_context;
3179	2927	atomic_set(&wq->usecnt, 0);
3180	2928	atomic_inc(&pd->usecnt);
3181	2929	atomic_inc(&cq->usecnt);
3182		- wq->uobject = &obj->uevent.uobject;
3183		- obj->uevent.uobject.object = wq;
	2930	+ obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
	2931	+ if (obj->uevent.event_file)
	2932	+ uverbs_uobject_get(&obj->uevent.event_file->uobj);
3184	2933
3185		- memset(&resp, 0, sizeof(resp));
	2934	+ uobj_put_obj_read(pd);
	2935	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	2936	+ UVERBS_LOOKUP_READ);
	2937	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
	2938	+
3186	2939	resp.wq_handle = obj->uevent.uobject.id;
3187	2940	resp.max_sge = wq_init_attr.max_sge;
3188	2941	resp.max_wr = wq_init_attr.max_wr;
3189	2942	resp.wqn = wq->wq_num;
3190		- resp.response_length = required_resp_len;
3191		- err = ib_copy_to_udata(ucore,
3192		- &resp, resp.response_length);
3193		- if (err)
3194		- goto err_copy;
	2943	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	2944	+ return uverbs_response(attrs, &resp, sizeof(resp));
3195	2945
3196		- uobj_put_obj_read(pd);
3197		- uobj_put_obj_read(cq);
3198		- return uobj_alloc_commit(&obj->uevent.uobject, 0);
3199		-
3200		-err_copy:
3201		- ib_destroy_wq(wq);
3202	2946	err_put_cq:
3203		- uobj_put_obj_read(cq);
	2947	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	2948	+ UVERBS_LOOKUP_READ);
3204	2949	err_put_pd:
3205	2950	uobj_put_obj_read(pd);
3206	2951	err_uobj:
3207		- uobj_alloc_abort(&obj->uevent.uobject);
	2952	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
3208	2953
3209	2954	return err;
3210	2955	}
3211	2956
3212		-int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file,
3213		- struct ib_udata *ucore,
3214		- struct ib_udata *uhw)
	2957	+static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)
3215	2958	{
3216		- struct ib_uverbs_ex_destroy_wq cmd = {};
	2959	+ struct ib_uverbs_ex_destroy_wq cmd;
3217	2960	struct ib_uverbs_ex_destroy_wq_resp resp = {};
3218	2961	struct ib_uobject *uobj;
3219	2962	struct ib_uwq_object *obj;
3220		- size_t required_cmd_sz;
3221		- size_t required_resp_len;
3222	2963	int ret;
3223	2964
3224		- required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle);
3225		- required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
3226		-
3227		- if (ucore->inlen < required_cmd_sz)
3228		- return -EINVAL;
3229		-
3230		- if (ucore->outlen < required_resp_len)
3231		- return -ENOSPC;
3232		-
3233		- if (ucore->inlen > sizeof(cmd) &&
3234		- !ib_is_udata_cleared(ucore, sizeof(cmd),
3235		- ucore->inlen - sizeof(cmd)))
3236		- return -EOPNOTSUPP;
3237		-
3238		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	2965	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3239	2966	if (ret)
3240	2967	return ret;
3241	2968
3242	2969	if (cmd.comp_mask)
3243	2970	return -EOPNOTSUPP;
3244	2971
3245		- resp.response_length = required_resp_len;
3246		- uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, file);
	2972	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	2973	+ uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3247	2974	if (IS_ERR(uobj))
3248	2975	return PTR_ERR(uobj);
3249	2976
..	..	@@ -3252,29 +2979,17 @@
3252	2979
3253	2980	uobj_put_destroy(uobj);
3254	2981
3255		- return ib_copy_to_udata(ucore, &resp, resp.response_length);
	2982	+ return uverbs_response(attrs, &resp, sizeof(resp));
3256	2983	}
3257	2984
3258		-int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file,
3259		- struct ib_udata *ucore,
3260		- struct ib_udata *uhw)
	2985	+static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)
3261	2986	{
3262		- struct ib_uverbs_ex_modify_wq cmd = {};
	2987	+ struct ib_uverbs_ex_modify_wq cmd;
3263	2988	struct ib_wq *wq;
3264	2989	struct ib_wq_attr wq_attr = {};
3265		- size_t required_cmd_sz;
3266	2990	int ret;
3267	2991
3268		- required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state);
3269		- if (ucore->inlen < required_cmd_sz)
3270		- return -EINVAL;
3271		-
3272		- if (ucore->inlen > sizeof(cmd) &&
3273		- !ib_is_udata_cleared(ucore, sizeof(cmd),
3274		- ucore->inlen - sizeof(cmd)))
3275		- return -EOPNOTSUPP;
3276		-
3277		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	2992	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3278	2993	if (ret)
3279	2994	return ret;
3280	2995
..	..	@@ -3284,61 +2999,59 @@
3284	2999	if (cmd.attr_mask > (IB_WQ_STATE \| IB_WQ_CUR_STATE \| IB_WQ_FLAGS))
3285	3000	return -EINVAL;
3286	3001
3287		- wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, file);
	3002	+ wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3288	3003	if (!wq)
3289	3004	return -EINVAL;
3290	3005
3291		- wq_attr.curr_wq_state = cmd.curr_wq_state;
3292		- wq_attr.wq_state = cmd.wq_state;
3293	3006	if (cmd.attr_mask & IB_WQ_FLAGS) {
3294	3007	wq_attr.flags = cmd.flags;
3295	3008	wq_attr.flags_mask = cmd.flags_mask;
3296	3009	}
3297		- if (!wq->device->modify_wq) {
3298		- ret = -EOPNOTSUPP;
3299		- goto out;
	3010	+
	3011	+ if (cmd.attr_mask & IB_WQ_CUR_STATE) {
	3012	+ if (cmd.curr_wq_state > IB_WQS_ERR)
	3013	+ return -EINVAL;
	3014	+
	3015	+ wq_attr.curr_wq_state = cmd.curr_wq_state;
	3016	+ } else {
	3017	+ wq_attr.curr_wq_state = wq->state;
3300	3018	}
3301		- ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw);
3302		-out:
3303		- uobj_put_obj_read(wq);
	3019	+
	3020	+ if (cmd.attr_mask & IB_WQ_STATE) {
	3021	+ if (cmd.wq_state > IB_WQS_ERR)
	3022	+ return -EINVAL;
	3023	+
	3024	+ wq_attr.wq_state = cmd.wq_state;
	3025	+ } else {
	3026	+ wq_attr.wq_state = wq_attr.curr_wq_state;
	3027	+ }
	3028	+
	3029	+ ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,
	3030	+ &attrs->driver_udata);
	3031	+ rdma_lookup_put_uobject(&wq->uobject->uevent.uobject,
	3032	+ UVERBS_LOOKUP_READ);
3304	3033	return ret;
3305	3034	}
3306	3035
3307		-int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file,
3308		- struct ib_udata *ucore,
3309		- struct ib_udata *uhw)
	3036	+static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3310	3037	{
3311		- struct ib_uverbs_ex_create_rwq_ind_table cmd = {};
	3038	+ struct ib_uverbs_ex_create_rwq_ind_table cmd;
3312	3039	struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {};
3313		- struct ib_uobject *uobj;
3314		- int err = 0;
	3040	+ struct ib_uobject *uobj;
	3041	+ int err;
3315	3042	struct ib_rwq_ind_table_init_attr init_attr = {};
3316	3043	struct ib_rwq_ind_table *rwq_ind_tbl;
3317		- struct ib_wq **wqs = NULL;
	3044	+ struct ib_wq **wqs = NULL;
3318	3045	u32 *wqs_handles = NULL;
3319	3046	struct ib_wq *wq = NULL;
3320		- int i, j, num_read_wqs;
	3047	+ int i, num_read_wqs;
3321	3048	u32 num_wq_handles;
3322		- u32 expected_in_size;
3323		- size_t required_cmd_sz_header;
3324		- size_t required_resp_len;
	3049	+ struct uverbs_req_iter iter;
3325	3050	struct ib_device *ib_dev;
3326	3051
3327		- required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size);
3328		- required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num);
3329		-
3330		- if (ucore->inlen < required_cmd_sz_header)
3331		- return -EINVAL;
3332		-
3333		- if (ucore->outlen < required_resp_len)
3334		- return -ENOSPC;
3335		-
3336		- err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header);
	3052	+ err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3337	3053	if (err)
3338	3054	return err;
3339		-
3340		- ucore->inbuf += required_cmd_sz_header;
3341		- ucore->inlen -= required_cmd_sz_header;
3342	3055
3343	3056	if (cmd.comp_mask)
3344	3057	return -EOPNOTSUPP;
..	..	@@ -3347,26 +3060,17 @@
3347	3060	return -EINVAL;
3348	3061
3349	3062	num_wq_handles = 1 << cmd.log_ind_tbl_size;
3350		- expected_in_size = num_wq_handles * sizeof(__u32);
3351		- if (num_wq_handles == 1)
3352		- /* input size for wq handles is u64 aligned */
3353		- expected_in_size += sizeof(__u32);
3354		-
3355		- if (ucore->inlen < expected_in_size)
3356		- return -EINVAL;
3357		-
3358		- if (ucore->inlen > expected_in_size &&
3359		- !ib_is_udata_cleared(ucore, expected_in_size,
3360		- ucore->inlen - expected_in_size))
3361		- return -EOPNOTSUPP;
3362		-
3363	3063	wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3364	3064	GFP_KERNEL);
3365	3065	if (!wqs_handles)
3366	3066	return -ENOMEM;
3367	3067
3368		- err = ib_copy_from_udata(wqs_handles, ucore,
3369		- num_wq_handles * sizeof(__u32));
	3068	+ err = uverbs_request_next(&iter, wqs_handles,
	3069	+ num_wq_handles * sizeof(__u32));
	3070	+ if (err)
	3071	+ goto err_free;
	3072	+
	3073	+ err = uverbs_request_finish(&iter);
3370	3074	if (err)
3371	3075	goto err_free;
3372	3076
..	..	@@ -3379,34 +3083,30 @@
3379	3083	for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3380	3084	num_read_wqs++) {
3381	3085	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,
3382		- wqs_handles[num_read_wqs], file);
	3086	+ wqs_handles[num_read_wqs], attrs);
3383	3087	if (!wq) {
3384	3088	err = -EINVAL;
3385	3089	goto put_wqs;
3386	3090	}
3387	3091
3388	3092	wqs[num_read_wqs] = wq;
	3093	+ atomic_inc(&wqs[num_read_wqs]->usecnt);
3389	3094	}
3390	3095
3391		- uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, file, &ib_dev);
	3096	+ uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
3392	3097	if (IS_ERR(uobj)) {
3393	3098	err = PTR_ERR(uobj);
3394	3099	goto put_wqs;
3395	3100	}
3396	3101
	3102	+ rwq_ind_tbl = rdma_zalloc_drv_obj(ib_dev, ib_rwq_ind_table);
	3103	+ if (!rwq_ind_tbl) {
	3104	+ err = -ENOMEM;
	3105	+ goto err_uobj;
	3106	+ }
	3107	+
3397	3108	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3398	3109	init_attr.ind_tbl = wqs;
3399		-
3400		- if (!ib_dev->create_rwq_ind_table) {
3401		- err = -EOPNOTSUPP;
3402		- goto err_uobj;
3403		- }
3404		- rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw);
3405		-
3406		- if (IS_ERR(rwq_ind_tbl)) {
3407		- err = PTR_ERR(rwq_ind_tbl);
3408		- goto err_uobj;
3409		- }
3410	3110
3411	3111	rwq_ind_tbl->ind_tbl = wqs;
3412	3112	rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
..	..	@@ -3415,57 +3115,44 @@
3415	3115	rwq_ind_tbl->device = ib_dev;
3416	3116	atomic_set(&rwq_ind_tbl->usecnt, 0);
3417	3117
	3118	+ err = ib_dev->ops.create_rwq_ind_table(rwq_ind_tbl, &init_attr,
	3119	+ &attrs->driver_udata);
	3120	+ if (err)
	3121	+ goto err_create;
	3122	+
3418	3123	for (i = 0; i < num_wq_handles; i++)
3419		- atomic_inc(&wqs[i]->usecnt);
	3124	+ rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
	3125	+ UVERBS_LOOKUP_READ);
	3126	+ kfree(wqs_handles);
	3127	+ uobj_finalize_uobj_create(uobj, attrs);
3420	3128
3421	3129	resp.ind_tbl_handle = uobj->id;
3422	3130	resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3423		- resp.response_length = required_resp_len;
	3131	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	3132	+ return uverbs_response(attrs, &resp, sizeof(resp));
3424	3133
3425		- err = ib_copy_to_udata(ucore,
3426		- &resp, resp.response_length);
3427		- if (err)
3428		- goto err_copy;
3429		-
3430		- kfree(wqs_handles);
3431		-
3432		- for (j = 0; j < num_read_wqs; j++)
3433		- uobj_put_obj_read(wqs[j]);
3434		-
3435		- return uobj_alloc_commit(uobj, 0);
3436		-
3437		-err_copy:
3438		- ib_destroy_rwq_ind_table(rwq_ind_tbl);
	3134	+err_create:
	3135	+ kfree(rwq_ind_tbl);
3439	3136	err_uobj:
3440		- uobj_alloc_abort(uobj);
	3137	+ uobj_alloc_abort(uobj, attrs);
3441	3138	put_wqs:
3442		- for (j = 0; j < num_read_wqs; j++)
3443		- uobj_put_obj_read(wqs[j]);
	3139	+ for (i = 0; i < num_read_wqs; i++) {
	3140	+ rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
	3141	+ UVERBS_LOOKUP_READ);
	3142	+ atomic_dec(&wqs[i]->usecnt);
	3143	+ }
3444	3144	err_free:
3445	3145	kfree(wqs_handles);
3446	3146	kfree(wqs);
3447	3147	return err;
3448	3148	}
3449	3149
3450		-int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file,
3451		- struct ib_udata *ucore,
3452		- struct ib_udata *uhw)
	3150	+static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3453	3151	{
3454		- struct ib_uverbs_ex_destroy_rwq_ind_table cmd = {};
3455		- int ret;
3456		- size_t required_cmd_sz;
	3152	+ struct ib_uverbs_ex_destroy_rwq_ind_table cmd;
	3153	+ int ret;
3457	3154
3458		- required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle);
3459		-
3460		- if (ucore->inlen < required_cmd_sz)
3461		- return -EINVAL;
3462		-
3463		- if (ucore->inlen > sizeof(cmd) &&
3464		- !ib_is_udata_cleared(ucore, sizeof(cmd),
3465		- ucore->inlen - sizeof(cmd)))
3466		- return -EOPNOTSUPP;
3467		-
3468		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	3155	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3469	3156	if (ret)
3470	3157	return ret;
3471	3158
..	..	@@ -3473,40 +3160,29 @@
3473	3160	return -EOPNOTSUPP;
3474	3161
3475	3162	return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,
3476		- cmd.ind_tbl_handle, file, 0);
	3163	+ cmd.ind_tbl_handle, attrs);
3477	3164	}
3478	3165
3479		-int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
3480		- struct ib_udata *ucore,
3481		- struct ib_udata *uhw)
	3166	+static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
3482	3167	{
3483	3168	struct ib_uverbs_create_flow cmd;
3484		- struct ib_uverbs_create_flow_resp resp;
	3169	+ struct ib_uverbs_create_flow_resp resp = {};
3485	3170	struct ib_uobject *uobj;
3486		- struct ib_uflow_object *uflow;
3487	3171	struct ib_flow *flow_id;
3488	3172	struct ib_uverbs_flow_attr *kern_flow_attr;
3489	3173	struct ib_flow_attr *flow_attr;
3490	3174	struct ib_qp *qp;
3491	3175	struct ib_uflow_resources *uflow_res;
3492	3176	struct ib_uverbs_flow_spec_hdr *kern_spec;
3493		- int err = 0;
	3177	+ struct uverbs_req_iter iter;
	3178	+ int err;
3494	3179	void *ib_spec;
3495	3180	int i;
3496	3181	struct ib_device *ib_dev;
3497	3182
3498		- if (ucore->inlen < sizeof(cmd))
3499		- return -EINVAL;
3500		-
3501		- if (ucore->outlen < sizeof(resp))
3502		- return -ENOSPC;
3503		-
3504		- err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
	3183	+ err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3505	3184	if (err)
3506	3185	return err;
3507		-
3508		- ucore->inbuf += sizeof(cmd);
3509		- ucore->inlen -= sizeof(cmd);
3510	3186
3511	3187	if (cmd.comp_mask)
3512	3188	return -EINVAL;
..	..	@@ -3525,8 +3201,7 @@
3525	3201	if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3526	3202	return -EINVAL;
3527	3203
3528		- if (cmd.flow_attr.size > ucore->inlen \|\|
3529		- cmd.flow_attr.size >
	3204	+ if (cmd.flow_attr.size >
3530	3205	(cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3531	3206	return -EINVAL;
3532	3207
..	..	@@ -3541,21 +3216,25 @@
3541	3216	return -ENOMEM;
3542	3217
3543	3218	*kern_flow_attr = cmd.flow_attr;
3544		- err = ib_copy_from_udata(&kern_flow_attr->flow_specs, ucore,
3545		- cmd.flow_attr.size);
	3219	+ err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs,
	3220	+ cmd.flow_attr.size);
3546	3221	if (err)
3547	3222	goto err_free_attr;
3548	3223	} else {
3549	3224	kern_flow_attr = &cmd.flow_attr;
3550	3225	}
3551	3226
3552		- uobj = uobj_alloc(UVERBS_OBJECT_FLOW, file, &ib_dev);
	3227	+ err = uverbs_request_finish(&iter);
	3228	+ if (err)
	3229	+ goto err_free_attr;
	3230	+
	3231	+ uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);
3553	3232	if (IS_ERR(uobj)) {
3554	3233	err = PTR_ERR(uobj);
3555	3234	goto err_free_attr;
3556	3235	}
3557	3236
3558		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
	3237	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
3559	3238	if (!qp) {
3560	3239	err = -EINVAL;
3561	3240	goto err_uobj;
..	..	@@ -3563,11 +3242,6 @@
3563	3242
3564	3243	if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3565	3244	err = -EINVAL;
3566		- goto err_put;
3567		- }
3568		-
3569		- if (!qp->device->create_flow) {
3570		- err = -EOPNOTSUPP;
3571	3245	goto err_put;
3572	3246	}
3573	3247
..	..	@@ -3597,7 +3271,7 @@
3597	3271	cmd.flow_attr.size >= kern_spec->size;
3598	3272	i++) {
3599	3273	err = kern_spec_to_ib_spec(
3600		- file, (struct ib_uverbs_flow_spec *)kern_spec,
	3274	+ attrs, (struct ib_uverbs_flow_spec *)kern_spec,
3601	3275	ib_spec, uflow_res);
3602	3276	if (err)
3603	3277	goto err_free;
..	..	@@ -3615,86 +3289,71 @@
3615	3289	goto err_free;
3616	3290	}
3617	3291
3618		- flow_id = qp->device->create_flow(qp, flow_attr,
3619		- IB_FLOW_DOMAIN_USER, uhw);
	3292	+ flow_id = qp->device->ops.create_flow(qp, flow_attr,
	3293	+ &attrs->driver_udata);
3620	3294
3621	3295	if (IS_ERR(flow_id)) {
3622	3296	err = PTR_ERR(flow_id);
3623	3297	goto err_free;
3624	3298	}
3625		- atomic_inc(&qp->usecnt);
3626		- flow_id->qp = qp;
3627		- flow_id->device = qp->device;
3628		- flow_id->uobject = uobj;
3629		- uobj->object = flow_id;
3630		- uflow = container_of(uobj, typeof(*uflow), uobject);
3631		- uflow->resources = uflow_res;
3632	3299
3633		- memset(&resp, 0, sizeof(resp));
3634		- resp.flow_handle = uobj->id;
	3300	+ ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
3635	3301
3636		- err = ib_copy_to_udata(ucore,
3637		- &resp, sizeof(resp));
3638		- if (err)
3639		- goto err_copy;
3640		-
3641		- uobj_put_obj_read(qp);
	3302	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	3303	+ UVERBS_LOOKUP_READ);
3642	3304	kfree(flow_attr);
	3305	+
3643	3306	if (cmd.flow_attr.num_of_specs)
3644	3307	kfree(kern_flow_attr);
3645		- return uobj_alloc_commit(uobj, 0);
3646		-err_copy:
3647		- if (!qp->device->destroy_flow(flow_id))
3648		- atomic_dec(&qp->usecnt);
	3308	+ uobj_finalize_uobj_create(uobj, attrs);
	3309	+
	3310	+ resp.flow_handle = uobj->id;
	3311	+ return uverbs_response(attrs, &resp, sizeof(resp));
	3312	+
3649	3313	err_free:
3650	3314	ib_uverbs_flow_resources_free(uflow_res);
3651	3315	err_free_flow_attr:
3652	3316	kfree(flow_attr);
3653	3317	err_put:
3654		- uobj_put_obj_read(qp);
	3318	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	3319	+ UVERBS_LOOKUP_READ);
3655	3320	err_uobj:
3656		- uobj_alloc_abort(uobj);
	3321	+ uobj_alloc_abort(uobj, attrs);
3657	3322	err_free_attr:
3658	3323	if (cmd.flow_attr.num_of_specs)
3659	3324	kfree(kern_flow_attr);
3660	3325	return err;
3661	3326	}
3662	3327
3663		-int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file,
3664		- struct ib_udata *ucore,
3665		- struct ib_udata *uhw)
	3328	+static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)
3666	3329	{
3667	3330	struct ib_uverbs_destroy_flow cmd;
3668	3331	int ret;
3669	3332
3670		- if (ucore->inlen < sizeof(cmd))
3671		- return -EINVAL;
3672		-
3673		- ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
	3333	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3674	3334	if (ret)
3675	3335	return ret;
3676	3336
3677	3337	if (cmd.comp_mask)
3678	3338	return -EINVAL;
3679	3339
3680		- return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, file,
3681		- 0);
	3340	+ return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);
3682	3341	}
3683	3342
3684		-static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
	3343	+static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
3685	3344	struct ib_uverbs_create_xsrq *cmd,
3686	3345	struct ib_udata *udata)
3687	3346	{
3688		- struct ib_uverbs_create_srq_resp resp;
	3347	+ struct ib_uverbs_create_srq_resp resp = {};
3689	3348	struct ib_usrq_object *obj;
3690	3349	struct ib_pd *pd;
3691	3350	struct ib_srq *srq;
3692		- struct ib_uobject *uninitialized_var(xrcd_uobj);
3693	3351	struct ib_srq_init_attr attr;
3694	3352	int ret;
	3353	+ struct ib_uobject *xrcd_uobj;
3695	3354	struct ib_device *ib_dev;
3696	3355
3697		- obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, file,
	3356	+ obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
3698	3357	&ib_dev);
3699	3358	if (IS_ERR(obj))
3700	3359	return PTR_ERR(obj);
..	..	@@ -3704,7 +3363,7 @@
3704	3363
3705	3364	if (cmd->srq_type == IB_SRQT_XRC) {
3706	3365	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,
3707		- file);
	3366	+ attrs);
3708	3367	if (IS_ERR(xrcd_uobj)) {
3709	3368	ret = -EINVAL;
3710	3369	goto err;
..	..	@@ -3722,89 +3381,64 @@
3722	3381
3723	3382	if (ib_srq_has_cq(cmd->srq_type)) {
3724	3383	attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
3725		- cmd->cq_handle, file);
	3384	+ cmd->cq_handle, attrs);
3726	3385	if (!attr.ext.cq) {
3727	3386	ret = -EINVAL;
3728	3387	goto err_put_xrcd;
3729	3388	}
3730	3389	}
3731	3390
3732		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, file);
	3391	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);
3733	3392	if (!pd) {
3734	3393	ret = -EINVAL;
3735	3394	goto err_put_cq;
3736	3395	}
3737	3396
3738	3397	attr.event_handler = ib_uverbs_srq_event_handler;
3739		- attr.srq_context = file;
3740	3398	attr.srq_type = cmd->srq_type;
3741	3399	attr.attr.max_wr = cmd->max_wr;
3742	3400	attr.attr.max_sge = cmd->max_sge;
3743	3401	attr.attr.srq_limit = cmd->srq_limit;
3744	3402
3745		- obj->uevent.events_reported = 0;
3746	3403	INIT_LIST_HEAD(&obj->uevent.event_list);
	3404	+ obj->uevent.uobject.user_handle = cmd->user_handle;
3747	3405
3748		- srq = pd->device->create_srq(pd, &attr, udata);
	3406	+ srq = ib_create_srq_user(pd, &attr, obj, udata);
3749	3407	if (IS_ERR(srq)) {
3750	3408	ret = PTR_ERR(srq);
3751		- goto err_put;
	3409	+ goto err_put_pd;
3752	3410	}
3753		-
3754		- srq->device = pd->device;
3755		- srq->pd = pd;
3756		- srq->srq_type = cmd->srq_type;
3757		- srq->uobject = &obj->uevent.uobject;
3758		- srq->event_handler = attr.event_handler;
3759		- srq->srq_context = attr.srq_context;
3760		-
3761		- if (ib_srq_has_cq(cmd->srq_type)) {
3762		- srq->ext.cq = attr.ext.cq;
3763		- atomic_inc(&attr.ext.cq->usecnt);
3764		- }
3765		-
3766		- if (cmd->srq_type == IB_SRQT_XRC) {
3767		- srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
3768		- atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3769		- }
3770		-
3771		- atomic_inc(&pd->usecnt);
3772		- atomic_set(&srq->usecnt, 0);
3773	3411
3774	3412	obj->uevent.uobject.object = srq;
3775	3413	obj->uevent.uobject.user_handle = cmd->user_handle;
	3414	+ obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
	3415	+ if (obj->uevent.event_file)
	3416	+ uverbs_uobject_get(&obj->uevent.event_file->uobj);
3776	3417
3777		- memset(&resp, 0, sizeof resp);
3778		- resp.srq_handle = obj->uevent.uobject.id;
3779		- resp.max_wr = attr.attr.max_wr;
3780		- resp.max_sge = attr.attr.max_sge;
3781	3418	if (cmd->srq_type == IB_SRQT_XRC)
3782	3419	resp.srqn = srq->ext.xrc.srq_num;
3783		-
3784		- if (copy_to_user(u64_to_user_ptr(cmd->response),
3785		- &resp, sizeof resp)) {
3786		- ret = -EFAULT;
3787		- goto err_copy;
3788		- }
3789	3420
3790	3421	if (cmd->srq_type == IB_SRQT_XRC)
3791	3422	uobj_put_read(xrcd_uobj);
3792	3423
3793	3424	if (ib_srq_has_cq(cmd->srq_type))
3794		- uobj_put_obj_read(attr.ext.cq);
	3425	+ rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,
	3426	+ UVERBS_LOOKUP_READ);
3795	3427
3796	3428	uobj_put_obj_read(pd);
3797		- return uobj_alloc_commit(&obj->uevent.uobject, 0);
	3429	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
3798	3430
3799		-err_copy:
3800		- ib_destroy_srq(srq);
	3431	+ resp.srq_handle = obj->uevent.uobject.id;
	3432	+ resp.max_wr = attr.attr.max_wr;
	3433	+ resp.max_sge = attr.attr.max_sge;
	3434	+ return uverbs_response(attrs, &resp, sizeof(resp));
3801	3435
3802		-err_put:
	3436	+err_put_pd:
3803	3437	uobj_put_obj_read(pd);
3804		-
3805	3438	err_put_cq:
3806	3439	if (ib_srq_has_cq(cmd->srq_type))
3807		- uobj_put_obj_read(attr.ext.cq);
	3440	+ rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,
	3441	+ UVERBS_LOOKUP_READ);
3808	3442
3809	3443	err_put_xrcd:
3810	3444	if (cmd->srq_type == IB_SRQT_XRC) {
..	..	@@ -3813,25 +3447,19 @@
3813	3447	}
3814	3448
3815	3449	err:
3816		- uobj_alloc_abort(&obj->uevent.uobject);
	3450	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
3817	3451	return ret;
3818	3452	}
3819	3453
3820		-ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file,
3821		- const char __user *buf, int in_len,
3822		- int out_len)
	3454	+static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)
3823	3455	{
3824	3456	struct ib_uverbs_create_srq cmd;
3825	3457	struct ib_uverbs_create_xsrq xcmd;
3826		- struct ib_uverbs_create_srq_resp resp;
3827		- struct ib_udata udata;
3828	3458	int ret;
3829	3459
3830		- if (out_len < sizeof resp)
3831		- return -ENOSPC;
3832		-
3833		- if (copy_from_user(&cmd, buf, sizeof cmd))
3834		- return -EFAULT;
	3460	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	3461	+ if (ret)
	3462	+ return ret;
3835	3463
3836	3464	memset(&xcmd, 0, sizeof(xcmd));
3837	3465	xcmd.response = cmd.response;
..	..	@@ -3842,77 +3470,49 @@
3842	3470	xcmd.max_sge = cmd.max_sge;
3843	3471	xcmd.srq_limit = cmd.srq_limit;
3844	3472
3845		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
3846		- u64_to_user_ptr(cmd.response) + sizeof(resp),
3847		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
3848		- out_len - sizeof(resp));
3849		-
3850		- ret = __uverbs_create_xsrq(file, &xcmd, &udata);
3851		- if (ret)
3852		- return ret;
3853		-
3854		- return in_len;
	3473	+ return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);
3855	3474	}
3856	3475
3857		-ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file,
3858		- const char __user *buf, int in_len, int out_len)
	3476	+static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)
3859	3477	{
3860	3478	struct ib_uverbs_create_xsrq cmd;
3861		- struct ib_uverbs_create_srq_resp resp;
3862		- struct ib_udata udata;
3863	3479	int ret;
3864	3480
3865		- if (out_len < sizeof resp)
3866		- return -ENOSPC;
3867		-
3868		- if (copy_from_user(&cmd, buf, sizeof cmd))
3869		- return -EFAULT;
3870		-
3871		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
3872		- u64_to_user_ptr(cmd.response) + sizeof(resp),
3873		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
3874		- out_len - sizeof(resp));
3875		-
3876		- ret = __uverbs_create_xsrq(file, &cmd, &udata);
	3481	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3877	3482	if (ret)
3878	3483	return ret;
3879	3484
3880		- return in_len;
	3485	+ return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);
3881	3486	}
3882	3487
3883		-ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
3884		- const char __user *buf, int in_len,
3885		- int out_len)
	3488	+static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)
3886	3489	{
3887	3490	struct ib_uverbs_modify_srq cmd;
3888		- struct ib_udata udata;
3889	3491	struct ib_srq *srq;
3890	3492	struct ib_srq_attr attr;
3891	3493	int ret;
3892	3494
3893		- if (copy_from_user(&cmd, buf, sizeof cmd))
3894		- return -EFAULT;
	3495	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	3496	+ if (ret)
	3497	+ return ret;
3895	3498
3896		- ib_uverbs_init_udata(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
3897		- out_len);
3898		-
3899		- srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file);
	3499	+ srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3900	3500	if (!srq)
3901	3501	return -EINVAL;
3902	3502
3903	3503	attr.max_wr = cmd.max_wr;
3904	3504	attr.srq_limit = cmd.srq_limit;
3905	3505
3906		- ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata);
	3506	+ ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,
	3507	+ &attrs->driver_udata);
3907	3508
3908		- uobj_put_obj_read(srq);
	3509	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	3510	+ UVERBS_LOOKUP_READ);
3909	3511
3910		- return ret ? ret : in_len;
	3512	+ return ret;
3911	3513	}
3912	3514
3913		-ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file,
3914		- const char __user *buf,
3915		- int in_len, int out_len)
	3515	+static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)
3916	3516	{
3917	3517	struct ib_uverbs_query_srq cmd;
3918	3518	struct ib_uverbs_query_srq_resp resp;
..	..	@@ -3920,19 +3520,18 @@
3920	3520	struct ib_srq *srq;
3921	3521	int ret;
3922	3522
3923		- if (out_len < sizeof resp)
3924		- return -ENOSPC;
	3523	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	3524	+ if (ret)
	3525	+ return ret;
3925	3526
3926		- if (copy_from_user(&cmd, buf, sizeof cmd))
3927		- return -EFAULT;
3928		-
3929		- srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file);
	3527	+ srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3930	3528	if (!srq)
3931	3529	return -EINVAL;
3932	3530
3933	3531	ret = ib_query_srq(srq, &attr);
3934	3532
3935		- uobj_put_obj_read(srq);
	3533	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	3534	+ UVERBS_LOOKUP_READ);
3936	3535
3937	3536	if (ret)
3938	3537	return ret;
..	..	@@ -3943,25 +3542,22 @@
3943	3542	resp.max_sge = attr.max_sge;
3944	3543	resp.srq_limit = attr.srq_limit;
3945	3544
3946		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
3947		- return -EFAULT;
3948		-
3949		- return in_len;
	3545	+ return uverbs_response(attrs, &resp, sizeof(resp));
3950	3546	}
3951	3547
3952		-ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3953		- const char __user *buf, int in_len,
3954		- int out_len)
	3548	+static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)
3955	3549	{
3956	3550	struct ib_uverbs_destroy_srq cmd;
3957	3551	struct ib_uverbs_destroy_srq_resp resp;
3958	3552	struct ib_uobject *uobj;
3959	3553	struct ib_uevent_object *obj;
	3554	+ int ret;
3960	3555
3961		- if (copy_from_user(&cmd, buf, sizeof cmd))
3962		- return -EFAULT;
	3556	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	3557	+ if (ret)
	3558	+ return ret;
3963	3559
3964		- uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, file);
	3560	+ uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3965	3561	if (IS_ERR(uobj))
3966	3562	return PTR_ERR(uobj);
3967	3563
..	..	@@ -3971,35 +3567,24 @@
3971	3567
3972	3568	uobj_put_destroy(uobj);
3973	3569
3974		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp)))
3975		- return -EFAULT;
3976		-
3977		- return in_len;
	3570	+ return uverbs_response(attrs, &resp, sizeof(resp));
3978	3571	}
3979	3572
3980		-int ib_uverbs_ex_query_device(struct ib_uverbs_file *file,
3981		- struct ib_udata *ucore,
3982		- struct ib_udata *uhw)
	3573	+static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
3983	3574	{
3984		- struct ib_uverbs_ex_query_device_resp resp = { {0} };
	3575	+ struct ib_uverbs_ex_query_device_resp resp = {};
3985	3576	struct ib_uverbs_ex_query_device cmd;
3986	3577	struct ib_device_attr attr = {0};
3987	3578	struct ib_ucontext *ucontext;
3988	3579	struct ib_device *ib_dev;
3989	3580	int err;
3990	3581
3991		- ucontext = ib_uverbs_get_ucontext(file);
	3582	+ ucontext = ib_uverbs_get_ucontext(attrs);
3992	3583	if (IS_ERR(ucontext))
3993	3584	return PTR_ERR(ucontext);
3994	3585	ib_dev = ucontext->device;
3995	3586
3996		- if (!ib_dev->query_device)
3997		- return -EOPNOTSUPP;
3998		-
3999		- if (ucore->inlen < sizeof(cmd))
4000		- return -EINVAL;
4001		-
4002		- err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
	3587	+ err = uverbs_request(attrs, &cmd, sizeof(cmd));
4003	3588	if (err)
4004	3589	return err;
4005	3590
..	..	@@ -4009,21 +3594,12 @@
4009	3594	if (cmd.reserved)
4010	3595	return -EINVAL;
4011	3596
4012		- resp.response_length = offsetof(typeof(resp), odp_caps);
4013		-
4014		- if (ucore->outlen < resp.response_length)
4015		- return -ENOSPC;
4016		-
4017		- err = ib_dev->query_device(ib_dev, &attr, uhw);
	3597	+ err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);
4018	3598	if (err)
4019	3599	return err;
4020	3600
4021	3601	copy_query_dev_fields(ucontext, &resp.base, &attr);
4022	3602
4023		- if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps))
4024		- goto end;
4025		-
4026		-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
4027	3603	resp.odp_caps.general_caps = attr.odp_caps.general_caps;
4028	3604	resp.odp_caps.per_transport_caps.rc_odp_caps =
4029	3605	attr.odp_caps.per_transport_caps.rc_odp_caps;
..	..	@@ -4031,100 +3607,40 @@
4031	3607	attr.odp_caps.per_transport_caps.uc_odp_caps;
4032	3608	resp.odp_caps.per_transport_caps.ud_odp_caps =
4033	3609	attr.odp_caps.per_transport_caps.ud_odp_caps;
4034		-#endif
4035		- resp.response_length += sizeof(resp.odp_caps);
4036		-
4037		- if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask))
4038		- goto end;
	3610	+ resp.xrc_odp_caps = attr.odp_caps.per_transport_caps.xrc_odp_caps;
4039	3611
4040	3612	resp.timestamp_mask = attr.timestamp_mask;
4041		- resp.response_length += sizeof(resp.timestamp_mask);
4042		-
4043		- if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock))
4044		- goto end;
4045		-
4046	3613	resp.hca_core_clock = attr.hca_core_clock;
4047		- resp.response_length += sizeof(resp.hca_core_clock);
4048		-
4049		- if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex))
4050		- goto end;
4051		-
4052	3614	resp.device_cap_flags_ex = attr.device_cap_flags;
4053		- resp.response_length += sizeof(resp.device_cap_flags_ex);
4054		-
4055		- if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps))
4056		- goto end;
4057		-
4058	3615	resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
4059	3616	resp.rss_caps.max_rwq_indirection_tables =
4060	3617	attr.rss_caps.max_rwq_indirection_tables;
4061	3618	resp.rss_caps.max_rwq_indirection_table_size =
4062	3619	attr.rss_caps.max_rwq_indirection_table_size;
4063		-
4064		- resp.response_length += sizeof(resp.rss_caps);
4065		-
4066		- if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq))
4067		- goto end;
4068		-
4069	3620	resp.max_wq_type_rq = attr.max_wq_type_rq;
4070		- resp.response_length += sizeof(resp.max_wq_type_rq);
4071		-
4072		- if (ucore->outlen < resp.response_length + sizeof(resp.raw_packet_caps))
4073		- goto end;
4074		-
4075	3621	resp.raw_packet_caps = attr.raw_packet_caps;
4076		- resp.response_length += sizeof(resp.raw_packet_caps);
4077		-
4078		- if (ucore->outlen < resp.response_length + sizeof(resp.tm_caps))
4079		- goto end;
4080		-
4081	3622	resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size;
4082	3623	resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags;
4083	3624	resp.tm_caps.max_ops = attr.tm_caps.max_ops;
4084	3625	resp.tm_caps.max_sge = attr.tm_caps.max_sge;
4085	3626	resp.tm_caps.flags = attr.tm_caps.flags;
4086		- resp.response_length += sizeof(resp.tm_caps);
4087		-
4088		- if (ucore->outlen < resp.response_length + sizeof(resp.cq_moderation_caps))
4089		- goto end;
4090		-
4091	3627	resp.cq_moderation_caps.max_cq_moderation_count =
4092	3628	attr.cq_caps.max_cq_moderation_count;
4093	3629	resp.cq_moderation_caps.max_cq_moderation_period =
4094	3630	attr.cq_caps.max_cq_moderation_period;
4095		- resp.response_length += sizeof(resp.cq_moderation_caps);
4096		-
4097		- if (ucore->outlen < resp.response_length + sizeof(resp.max_dm_size))
4098		- goto end;
4099		-
4100	3631	resp.max_dm_size = attr.max_dm_size;
4101		- resp.response_length += sizeof(resp.max_dm_size);
4102		-end:
4103		- err = ib_copy_to_udata(ucore, &resp, resp.response_length);
4104		- return err;
	3632	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	3633	+
	3634	+ return uverbs_response(attrs, &resp, sizeof(resp));
4105	3635	}
4106	3636
4107		-int ib_uverbs_ex_modify_cq(struct ib_uverbs_file *file,
4108		- struct ib_udata *ucore,
4109		- struct ib_udata *uhw)
	3637	+static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
4110	3638	{
4111		- struct ib_uverbs_ex_modify_cq cmd = {};
	3639	+ struct ib_uverbs_ex_modify_cq cmd;
4112	3640	struct ib_cq *cq;
4113		- size_t required_cmd_sz;
4114	3641	int ret;
4115	3642
4116		- required_cmd_sz = offsetof(typeof(cmd), reserved) +
4117		- sizeof(cmd.reserved);
4118		- if (ucore->inlen < required_cmd_sz)
4119		- return -EINVAL;
4120		-
4121		- /* sanity checks */
4122		- if (ucore->inlen > sizeof(cmd) &&
4123		- !ib_is_udata_cleared(ucore, sizeof(cmd),
4124		- ucore->inlen - sizeof(cmd)))
4125		- return -EOPNOTSUPP;
4126		-
4127		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	3643	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
4128	3644	if (ret)
4129	3645	return ret;
4130	3646
..	..	@@ -4134,13 +3650,388 @@
4134	3650	if (cmd.attr_mask > IB_CQ_MODERATE)
4135	3651	return -EOPNOTSUPP;
4136	3652
4137		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	3653	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
4138	3654	if (!cq)
4139	3655	return -EINVAL;
4140	3656
4141	3657	ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);
4142	3658
4143		- uobj_put_obj_read(cq);
4144		-
	3659	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	3660	+ UVERBS_LOOKUP_READ);
4145	3661	return ret;
4146	3662	}
	3663	+
	3664	+/*
	3665	+ * Describe the input structs for write(). Some write methods have an input
	3666	+ * only struct, most have an input and output. If the struct has an output then
	3667	+ * the 'response' u64 must be the first field in the request structure.
	3668	+ *
	3669	+ * If udata is present then both the request and response structs have a
	3670	+ * trailing driver_data flex array. In this case the size of the base struct
	3671	+ * cannot be changed.
	3672	+ */
	3673	+#define UAPI_DEF_WRITE_IO(req, resp) \
	3674	+ .write.has_resp = 1 + \
	3675	+ BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \
	3676	+ BUILD_BUG_ON_ZERO(sizeof_field(req, response) != \
	3677	+ sizeof(u64)), \
	3678	+ .write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
	3679	+
	3680	+#define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
	3681	+
	3682	+#define UAPI_DEF_WRITE_UDATA_IO(req, resp) \
	3683	+ UAPI_DEF_WRITE_IO(req, resp), \
	3684	+ .write.has_udata = \
	3685	+ 1 + \
	3686	+ BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
	3687	+ sizeof(req)) + \
	3688	+ BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) != \
	3689	+ sizeof(resp))
	3690	+
	3691	+#define UAPI_DEF_WRITE_UDATA_I(req) \
	3692	+ UAPI_DEF_WRITE_I(req), \
	3693	+ .write.has_udata = \
	3694	+ 1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
	3695	+ sizeof(req))
	3696	+
	3697	+/*
	3698	+ * The _EX versions are for use with WRITE_EX and allow the last struct member
	3699	+ * to be specified. Buffers that do not include that member will be rejected.
	3700	+ */
	3701	+#define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \
	3702	+ .write.has_resp = 1, \
	3703	+ .write.req_size = offsetofend(req, req_last_member), \
	3704	+ .write.resp_size = offsetofend(resp, resp_last_member)
	3705	+
	3706	+#define UAPI_DEF_WRITE_I_EX(req, req_last_member) \
	3707	+ .write.req_size = offsetofend(req, req_last_member)
	3708	+
	3709	+const struct uapi_definition uverbs_def_write_intf[] = {
	3710	+ DECLARE_UVERBS_OBJECT(
	3711	+ UVERBS_OBJECT_AH,
	3712	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,
	3713	+ ib_uverbs_create_ah,
	3714	+ UAPI_DEF_WRITE_UDATA_IO(
	3715	+ struct ib_uverbs_create_ah,
	3716	+ struct ib_uverbs_create_ah_resp),
	3717	+ UAPI_DEF_METHOD_NEEDS_FN(create_ah)),
	3718	+ DECLARE_UVERBS_WRITE(
	3719	+ IB_USER_VERBS_CMD_DESTROY_AH,
	3720	+ ib_uverbs_destroy_ah,
	3721	+ UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah),
	3722	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))),
	3723	+
	3724	+ DECLARE_UVERBS_OBJECT(
	3725	+ UVERBS_OBJECT_COMP_CHANNEL,
	3726	+ DECLARE_UVERBS_WRITE(
	3727	+ IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
	3728	+ ib_uverbs_create_comp_channel,
	3729	+ UAPI_DEF_WRITE_IO(
	3730	+ struct ib_uverbs_create_comp_channel,
	3731	+ struct ib_uverbs_create_comp_channel_resp))),
	3732	+
	3733	+ DECLARE_UVERBS_OBJECT(
	3734	+ UVERBS_OBJECT_CQ,
	3735	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,
	3736	+ ib_uverbs_create_cq,
	3737	+ UAPI_DEF_WRITE_UDATA_IO(
	3738	+ struct ib_uverbs_create_cq,
	3739	+ struct ib_uverbs_create_cq_resp),
	3740	+ UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
	3741	+ DECLARE_UVERBS_WRITE(
	3742	+ IB_USER_VERBS_CMD_DESTROY_CQ,
	3743	+ ib_uverbs_destroy_cq,
	3744	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,
	3745	+ struct ib_uverbs_destroy_cq_resp),
	3746	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),
	3747	+ DECLARE_UVERBS_WRITE(
	3748	+ IB_USER_VERBS_CMD_POLL_CQ,
	3749	+ ib_uverbs_poll_cq,
	3750	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,
	3751	+ struct ib_uverbs_poll_cq_resp),
	3752	+ UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),
	3753	+ DECLARE_UVERBS_WRITE(
	3754	+ IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,
	3755	+ ib_uverbs_req_notify_cq,
	3756	+ UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),
	3757	+ UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),
	3758	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,
	3759	+ ib_uverbs_resize_cq,
	3760	+ UAPI_DEF_WRITE_UDATA_IO(
	3761	+ struct ib_uverbs_resize_cq,
	3762	+ struct ib_uverbs_resize_cq_resp),
	3763	+ UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),
	3764	+ DECLARE_UVERBS_WRITE_EX(
	3765	+ IB_USER_VERBS_EX_CMD_CREATE_CQ,
	3766	+ ib_uverbs_ex_create_cq,
	3767	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,
	3768	+ reserved,
	3769	+ struct ib_uverbs_ex_create_cq_resp,
	3770	+ response_length),
	3771	+ UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
	3772	+ DECLARE_UVERBS_WRITE_EX(
	3773	+ IB_USER_VERBS_EX_CMD_MODIFY_CQ,
	3774	+ ib_uverbs_ex_modify_cq,
	3775	+ UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
	3776	+ UAPI_DEF_METHOD_NEEDS_FN(create_cq))),
	3777	+
	3778	+ DECLARE_UVERBS_OBJECT(
	3779	+ UVERBS_OBJECT_DEVICE,
	3780	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,
	3781	+ ib_uverbs_get_context,
	3782	+ UAPI_DEF_WRITE_UDATA_IO(
	3783	+ struct ib_uverbs_get_context,
	3784	+ struct ib_uverbs_get_context_resp)),
	3785	+ DECLARE_UVERBS_WRITE(
	3786	+ IB_USER_VERBS_CMD_QUERY_DEVICE,
	3787	+ ib_uverbs_query_device,
	3788	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,
	3789	+ struct ib_uverbs_query_device_resp)),
	3790	+ DECLARE_UVERBS_WRITE(
	3791	+ IB_USER_VERBS_CMD_QUERY_PORT,
	3792	+ ib_uverbs_query_port,
	3793	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,
	3794	+ struct ib_uverbs_query_port_resp),
	3795	+ UAPI_DEF_METHOD_NEEDS_FN(query_port)),
	3796	+ DECLARE_UVERBS_WRITE_EX(
	3797	+ IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
	3798	+ ib_uverbs_ex_query_device,
	3799	+ UAPI_DEF_WRITE_IO_EX(
	3800	+ struct ib_uverbs_ex_query_device,
	3801	+ reserved,
	3802	+ struct ib_uverbs_ex_query_device_resp,
	3803	+ response_length),
	3804	+ UAPI_DEF_METHOD_NEEDS_FN(query_device)),
	3805	+ UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),
	3806	+ UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),
	3807	+
	3808	+ DECLARE_UVERBS_OBJECT(
	3809	+ UVERBS_OBJECT_FLOW,
	3810	+ DECLARE_UVERBS_WRITE_EX(
	3811	+ IB_USER_VERBS_EX_CMD_CREATE_FLOW,
	3812	+ ib_uverbs_ex_create_flow,
	3813	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,
	3814	+ flow_attr,
	3815	+ struct ib_uverbs_create_flow_resp,
	3816	+ flow_handle),
	3817	+ UAPI_DEF_METHOD_NEEDS_FN(create_flow)),
	3818	+ DECLARE_UVERBS_WRITE_EX(
	3819	+ IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
	3820	+ ib_uverbs_ex_destroy_flow,
	3821	+ UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),
	3822	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),
	3823	+
	3824	+ DECLARE_UVERBS_OBJECT(
	3825	+ UVERBS_OBJECT_MR,
	3826	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,
	3827	+ ib_uverbs_dereg_mr,
	3828	+ UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),
	3829	+ UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),
	3830	+ DECLARE_UVERBS_WRITE(
	3831	+ IB_USER_VERBS_CMD_REG_MR,
	3832	+ ib_uverbs_reg_mr,
	3833	+ UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,
	3834	+ struct ib_uverbs_reg_mr_resp),
	3835	+ UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),
	3836	+ DECLARE_UVERBS_WRITE(
	3837	+ IB_USER_VERBS_CMD_REREG_MR,
	3838	+ ib_uverbs_rereg_mr,
	3839	+ UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,
	3840	+ struct ib_uverbs_rereg_mr_resp),
	3841	+ UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),
	3842	+
	3843	+ DECLARE_UVERBS_OBJECT(
	3844	+ UVERBS_OBJECT_MW,
	3845	+ DECLARE_UVERBS_WRITE(
	3846	+ IB_USER_VERBS_CMD_ALLOC_MW,
	3847	+ ib_uverbs_alloc_mw,
	3848	+ UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,
	3849	+ struct ib_uverbs_alloc_mw_resp),
	3850	+ UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),
	3851	+ DECLARE_UVERBS_WRITE(
	3852	+ IB_USER_VERBS_CMD_DEALLOC_MW,
	3853	+ ib_uverbs_dealloc_mw,
	3854	+ UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),
	3855	+ UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),
	3856	+
	3857	+ DECLARE_UVERBS_OBJECT(
	3858	+ UVERBS_OBJECT_PD,
	3859	+ DECLARE_UVERBS_WRITE(
	3860	+ IB_USER_VERBS_CMD_ALLOC_PD,
	3861	+ ib_uverbs_alloc_pd,
	3862	+ UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,
	3863	+ struct ib_uverbs_alloc_pd_resp),
	3864	+ UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),
	3865	+ DECLARE_UVERBS_WRITE(
	3866	+ IB_USER_VERBS_CMD_DEALLOC_PD,
	3867	+ ib_uverbs_dealloc_pd,
	3868	+ UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),
	3869	+ UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),
	3870	+
	3871	+ DECLARE_UVERBS_OBJECT(
	3872	+ UVERBS_OBJECT_QP,
	3873	+ DECLARE_UVERBS_WRITE(
	3874	+ IB_USER_VERBS_CMD_ATTACH_MCAST,
	3875	+ ib_uverbs_attach_mcast,
	3876	+ UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),
	3877	+ UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),
	3878	+ UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
	3879	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,
	3880	+ ib_uverbs_create_qp,
	3881	+ UAPI_DEF_WRITE_UDATA_IO(
	3882	+ struct ib_uverbs_create_qp,
	3883	+ struct ib_uverbs_create_qp_resp),
	3884	+ UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
	3885	+ DECLARE_UVERBS_WRITE(
	3886	+ IB_USER_VERBS_CMD_DESTROY_QP,
	3887	+ ib_uverbs_destroy_qp,
	3888	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,
	3889	+ struct ib_uverbs_destroy_qp_resp),
	3890	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),
	3891	+ DECLARE_UVERBS_WRITE(
	3892	+ IB_USER_VERBS_CMD_DETACH_MCAST,
	3893	+ ib_uverbs_detach_mcast,
	3894	+ UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),
	3895	+ UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
	3896	+ DECLARE_UVERBS_WRITE(
	3897	+ IB_USER_VERBS_CMD_MODIFY_QP,
	3898	+ ib_uverbs_modify_qp,
	3899	+ UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),
	3900	+ UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),
	3901	+ DECLARE_UVERBS_WRITE(
	3902	+ IB_USER_VERBS_CMD_POST_RECV,
	3903	+ ib_uverbs_post_recv,
	3904	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,
	3905	+ struct ib_uverbs_post_recv_resp),
	3906	+ UAPI_DEF_METHOD_NEEDS_FN(post_recv)),
	3907	+ DECLARE_UVERBS_WRITE(
	3908	+ IB_USER_VERBS_CMD_POST_SEND,
	3909	+ ib_uverbs_post_send,
	3910	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,
	3911	+ struct ib_uverbs_post_send_resp),
	3912	+ UAPI_DEF_METHOD_NEEDS_FN(post_send)),
	3913	+ DECLARE_UVERBS_WRITE(
	3914	+ IB_USER_VERBS_CMD_QUERY_QP,
	3915	+ ib_uverbs_query_qp,
	3916	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,
	3917	+ struct ib_uverbs_query_qp_resp),
	3918	+ UAPI_DEF_METHOD_NEEDS_FN(query_qp)),
	3919	+ DECLARE_UVERBS_WRITE_EX(
	3920	+ IB_USER_VERBS_EX_CMD_CREATE_QP,
	3921	+ ib_uverbs_ex_create_qp,
	3922	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,
	3923	+ comp_mask,
	3924	+ struct ib_uverbs_ex_create_qp_resp,
	3925	+ response_length),
	3926	+ UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
	3927	+ DECLARE_UVERBS_WRITE_EX(
	3928	+ IB_USER_VERBS_EX_CMD_MODIFY_QP,
	3929	+ ib_uverbs_ex_modify_qp,
	3930	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,
	3931	+ base,
	3932	+ struct ib_uverbs_ex_modify_qp_resp,
	3933	+ response_length),
	3934	+ UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),
	3935	+
	3936	+ DECLARE_UVERBS_OBJECT(
	3937	+ UVERBS_OBJECT_RWQ_IND_TBL,
	3938	+ DECLARE_UVERBS_WRITE_EX(
	3939	+ IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
	3940	+ ib_uverbs_ex_create_rwq_ind_table,
	3941	+ UAPI_DEF_WRITE_IO_EX(
	3942	+ struct ib_uverbs_ex_create_rwq_ind_table,
	3943	+ log_ind_tbl_size,
	3944	+ struct ib_uverbs_ex_create_rwq_ind_table_resp,
	3945	+ ind_tbl_num),
	3946	+ UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),
	3947	+ DECLARE_UVERBS_WRITE_EX(
	3948	+ IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
	3949	+ ib_uverbs_ex_destroy_rwq_ind_table,
	3950	+ UAPI_DEF_WRITE_I(
	3951	+ struct ib_uverbs_ex_destroy_rwq_ind_table),
	3952	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),
	3953	+
	3954	+ DECLARE_UVERBS_OBJECT(
	3955	+ UVERBS_OBJECT_WQ,
	3956	+ DECLARE_UVERBS_WRITE_EX(
	3957	+ IB_USER_VERBS_EX_CMD_CREATE_WQ,
	3958	+ ib_uverbs_ex_create_wq,
	3959	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,
	3960	+ max_sge,
	3961	+ struct ib_uverbs_ex_create_wq_resp,
	3962	+ wqn),
	3963	+ UAPI_DEF_METHOD_NEEDS_FN(create_wq)),
	3964	+ DECLARE_UVERBS_WRITE_EX(
	3965	+ IB_USER_VERBS_EX_CMD_DESTROY_WQ,
	3966	+ ib_uverbs_ex_destroy_wq,
	3967	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,
	3968	+ wq_handle,
	3969	+ struct ib_uverbs_ex_destroy_wq_resp,
	3970	+ reserved),
	3971	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),
	3972	+ DECLARE_UVERBS_WRITE_EX(
	3973	+ IB_USER_VERBS_EX_CMD_MODIFY_WQ,
	3974	+ ib_uverbs_ex_modify_wq,
	3975	+ UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,
	3976	+ curr_wq_state),
	3977	+ UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),
	3978	+
	3979	+ DECLARE_UVERBS_OBJECT(
	3980	+ UVERBS_OBJECT_SRQ,
	3981	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,
	3982	+ ib_uverbs_create_srq,
	3983	+ UAPI_DEF_WRITE_UDATA_IO(
	3984	+ struct ib_uverbs_create_srq,
	3985	+ struct ib_uverbs_create_srq_resp),
	3986	+ UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
	3987	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,
	3988	+ ib_uverbs_create_xsrq,
	3989	+ UAPI_DEF_WRITE_UDATA_IO(
	3990	+ struct ib_uverbs_create_xsrq,
	3991	+ struct ib_uverbs_create_srq_resp),
	3992	+ UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
	3993	+ DECLARE_UVERBS_WRITE(
	3994	+ IB_USER_VERBS_CMD_DESTROY_SRQ,
	3995	+ ib_uverbs_destroy_srq,
	3996	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,
	3997	+ struct ib_uverbs_destroy_srq_resp),
	3998	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),
	3999	+ DECLARE_UVERBS_WRITE(
	4000	+ IB_USER_VERBS_CMD_MODIFY_SRQ,
	4001	+ ib_uverbs_modify_srq,
	4002	+ UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),
	4003	+ UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),
	4004	+ DECLARE_UVERBS_WRITE(
	4005	+ IB_USER_VERBS_CMD_POST_SRQ_RECV,
	4006	+ ib_uverbs_post_srq_recv,
	4007	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,
	4008	+ struct ib_uverbs_post_srq_recv_resp),
	4009	+ UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),
	4010	+ DECLARE_UVERBS_WRITE(
	4011	+ IB_USER_VERBS_CMD_QUERY_SRQ,
	4012	+ ib_uverbs_query_srq,
	4013	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,
	4014	+ struct ib_uverbs_query_srq_resp),
	4015	+ UAPI_DEF_METHOD_NEEDS_FN(query_srq))),
	4016	+
	4017	+ DECLARE_UVERBS_OBJECT(
	4018	+ UVERBS_OBJECT_XRCD,
	4019	+ DECLARE_UVERBS_WRITE(
	4020	+ IB_USER_VERBS_CMD_CLOSE_XRCD,
	4021	+ ib_uverbs_close_xrcd,
	4022	+ UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd),
	4023	+ UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)),
	4024	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
	4025	+ ib_uverbs_open_qp,
	4026	+ UAPI_DEF_WRITE_UDATA_IO(
	4027	+ struct ib_uverbs_open_qp,
	4028	+ struct ib_uverbs_create_qp_resp)),
	4029	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,
	4030	+ ib_uverbs_open_xrcd,
	4031	+ UAPI_DEF_WRITE_UDATA_IO(
	4032	+ struct ib_uverbs_open_xrcd,
	4033	+ struct ib_uverbs_open_xrcd_resp),
	4034	+ UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))),
	4035	+
	4036	+ {},
	4037	+};