~hc/RK356X_SDK_RELEASE.git

..	..	@@ -47,11 +47,149 @@
47	47	#include "uverbs.h"
48	48	#include "core_priv.h"
49	49
	50	+/*
	51	+ * Copy a response to userspace. If the provided 'resp' is larger than the
	52	+ * user buffer it is silently truncated. If the user provided a larger buffer
	53	+ * then the trailing portion is zero filled.
	54	+ *
	55	+ * These semantics are intended to support future extension of the output
	56	+ * structures.
	57	+ */
	58	+static int uverbs_response(struct uverbs_attr_bundle attrs, const void resp,
	59	+ size_t resp_len)
	60	+{
	61	+ int ret;
	62	+
	63	+ if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
	64	+ return uverbs_copy_to_struct_or_zero(
	65	+ attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len);
	66	+
	67	+ if (copy_to_user(attrs->ucore.outbuf, resp,
	68	+ min(attrs->ucore.outlen, resp_len)))
	69	+ return -EFAULT;
	70	+
	71	+ if (resp_len < attrs->ucore.outlen) {
	72	+ /*
	73	+ * Zero fill any extra memory that user
	74	+ * space might have provided.
	75	+ */
	76	+ ret = clear_user(attrs->ucore.outbuf + resp_len,
	77	+ attrs->ucore.outlen - resp_len);
	78	+ if (ret)
	79	+ return -EFAULT;
	80	+ }
	81	+
	82	+ return 0;
	83	+}
	84	+
	85	+/*
	86	+ * Copy a request from userspace. If the provided 'req' is larger than the
	87	+ * user buffer then the user buffer is zero extended into the 'req'. If 'req'
	88	+ * is smaller than the user buffer then the uncopied bytes in the user buffer
	89	+ * must be zero.
	90	+ */
	91	+static int uverbs_request(struct uverbs_attr_bundle attrs, void req,
	92	+ size_t req_len)
	93	+{
	94	+ if (copy_from_user(req, attrs->ucore.inbuf,
	95	+ min(attrs->ucore.inlen, req_len)))
	96	+ return -EFAULT;
	97	+
	98	+ if (attrs->ucore.inlen < req_len) {
	99	+ memset(req + attrs->ucore.inlen, 0,
	100	+ req_len - attrs->ucore.inlen);
	101	+ } else if (attrs->ucore.inlen > req_len) {
	102	+ if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len,
	103	+ attrs->ucore.inlen - req_len))
	104	+ return -EOPNOTSUPP;
	105	+ }
	106	+ return 0;
	107	+}
	108	+
	109	+/*
	110	+ * Generate the value for the 'response_length' protocol used by write_ex.
	111	+ * This is the number of bytes the kernel actually wrote. Userspace can use
	112	+ * this to detect what structure members in the response the kernel
	113	+ * understood.
	114	+ */
	115	+static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,
	116	+ size_t resp_len)
	117	+{
	118	+ return min_t(size_t, attrs->ucore.outlen, resp_len);
	119	+}
	120	+
	121	+/*
	122	+ * The iterator version of the request interface is for handlers that need to
	123	+ * step over a flex array at the end of a command header.
	124	+ */
	125	+struct uverbs_req_iter {
	126	+ const void __user *cur;
	127	+ const void __user *end;
	128	+};
	129	+
	130	+static int uverbs_request_start(struct uverbs_attr_bundle *attrs,
	131	+ struct uverbs_req_iter *iter,
	132	+ void *req,
	133	+ size_t req_len)
	134	+{
	135	+ if (attrs->ucore.inlen < req_len)
	136	+ return -ENOSPC;
	137	+
	138	+ if (copy_from_user(req, attrs->ucore.inbuf, req_len))
	139	+ return -EFAULT;
	140	+
	141	+ iter->cur = attrs->ucore.inbuf + req_len;
	142	+ iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;
	143	+ return 0;
	144	+}
	145	+
	146	+static int uverbs_request_next(struct uverbs_req_iter iter, void val,
	147	+ size_t len)
	148	+{
	149	+ if (iter->cur + len > iter->end)
	150	+ return -ENOSPC;
	151	+
	152	+ if (copy_from_user(val, iter->cur, len))
	153	+ return -EFAULT;
	154	+
	155	+ iter->cur += len;
	156	+ return 0;
	157	+}
	158	+
	159	+static const void __user uverbs_request_next_ptr(struct uverbs_req_iter iter,
	160	+ size_t len)
	161	+{
	162	+ const void __user *res = iter->cur;
	163	+
	164	+ if (iter->cur + len > iter->end)
	165	+ return (void __force __user *)ERR_PTR(-ENOSPC);
	166	+ iter->cur += len;
	167	+ return res;
	168	+}
	169	+
	170	+static int uverbs_request_finish(struct uverbs_req_iter *iter)
	171	+{
	172	+ if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur))
	173	+ return -EOPNOTSUPP;
	174	+ return 0;
	175	+}
	176	+
	177	+/*
	178	+ * When calling a destroy function during an error unwind we need to pass in
	179	+ * the udata that is sanitized of all user arguments. Ie from the driver
	180	+ * perspective it looks like no udata was passed.
	181	+ */
	182	+struct ib_udata uverbs_get_cleared_udata(struct uverbs_attr_bundle attrs)
	183	+{
	184	+ attrs->driver_udata = (struct ib_udata){};
	185	+ return &attrs->driver_udata;
	186	+}
	187	+
50	188	static struct ib_uverbs_completion_event_file *
51		-_ib_uverbs_lookup_comp_file(s32 fd, struct ib_uverbs_file *ufile)
	189	+_ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)
52	190	{
53	191	struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
54		- fd, ufile);
	192	+ fd, attrs);
55	193
56	194	if (IS_ERR(uobj))
57	195	return (void *)uobj;
..	..	@@ -65,94 +203,56 @@
65	203	#define ib_uverbs_lookup_comp_file(_fd, _ufile) \
66	204	_ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
67	205
68		-ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
69		- const char __user *buf,
70		- int in_len, int out_len)
	206	+int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)
71	207	{
72		- struct ib_uverbs_get_context cmd;
73		- struct ib_uverbs_get_context_resp resp;
74		- struct ib_udata udata;
75		- struct ib_ucontext *ucontext;
76		- struct file *filp;
77		- struct ib_rdmacg_object cg_obj;
	208	+ struct ib_uverbs_file *ufile = attrs->ufile;
	209	+ struct ib_ucontext *ucontext;
78	210	struct ib_device *ib_dev;
	211	+
	212	+ ib_dev = srcu_dereference(ufile->device->ib_dev,
	213	+ &ufile->device->disassociate_srcu);
	214	+ if (!ib_dev)
	215	+ return -EIO;
	216	+
	217	+ ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);
	218	+ if (!ucontext)
	219	+ return -ENOMEM;
	220	+
	221	+ ucontext->device = ib_dev;
	222	+ ucontext->ufile = ufile;
	223	+ xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC);
	224	+
	225	+ rdma_restrack_new(&ucontext->res, RDMA_RESTRACK_CTX);
	226	+ rdma_restrack_set_name(&ucontext->res, NULL);
	227	+ attrs->context = ucontext;
	228	+ return 0;
	229	+}
	230	+
	231	+int ib_init_ucontext(struct uverbs_attr_bundle *attrs)
	232	+{
	233	+ struct ib_ucontext *ucontext = attrs->context;
	234	+ struct ib_uverbs_file *file = attrs->ufile;
79	235	int ret;
80	236
81		- if (out_len < sizeof resp)
82		- return -ENOSPC;
83		-
84		- if (copy_from_user(&cmd, buf, sizeof cmd))
85		- return -EFAULT;
86		-
	237	+ if (!down_read_trylock(&file->hw_destroy_rwsem))
	238	+ return -EIO;
87	239	mutex_lock(&file->ucontext_lock);
88		- ib_dev = srcu_dereference(file->device->ib_dev,
89		- &file->device->disassociate_srcu);
90		- if (!ib_dev) {
91		- ret = -EIO;
92		- goto err;
93		- }
94		-
95	240	if (file->ucontext) {
96	241	ret = -EINVAL;
97	242	goto err;
98	243	}
99	244
100		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
101		- u64_to_user_ptr(cmd.response) + sizeof(resp),
102		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
103		- out_len - sizeof(resp));
104		-
105		- ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
	245	+ ret = ib_rdmacg_try_charge(&ucontext->cg_obj, ucontext->device,
	246	+ RDMACG_RESOURCE_HCA_HANDLE);
106	247	if (ret)
107	248	goto err;
108	249
109		- ucontext = ib_dev->alloc_ucontext(ib_dev, &udata);
110		- if (IS_ERR(ucontext)) {
111		- ret = PTR_ERR(ucontext);
112		- goto err_alloc;
113		- }
	250	+ ret = ucontext->device->ops.alloc_ucontext(ucontext,
	251	+ &attrs->driver_udata);
	252	+ if (ret)
	253	+ goto err_uncharge;
114	254
115		- ucontext->device = ib_dev;
116		- ucontext->cg_obj = cg_obj;
117		- /* ufile is required when some objects are released */
118		- ucontext->ufile = file;
119		-
120		- rcu_read_lock();
121		- ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
122		- rcu_read_unlock();
123		- ucontext->closing = 0;
124		- ucontext->cleanup_retryable = false;
125		-
126		-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
127		- ucontext->umem_tree = RB_ROOT_CACHED;
128		- init_rwsem(&ucontext->umem_rwsem);
129		- ucontext->odp_mrs_count = 0;
130		- INIT_LIST_HEAD(&ucontext->no_private_counters);
131		-
132		- if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
133		- ucontext->invalidate_range = NULL;
134		-
135		-#endif
136		-
137		- resp.num_comp_vectors = file->device->num_comp_vectors;
138		-
139		- ret = get_unused_fd_flags(O_CLOEXEC);
140		- if (ret < 0)
141		- goto err_free;
142		- resp.async_fd = ret;
143		-
144		- filp = ib_uverbs_alloc_async_event_file(file, ib_dev);
145		- if (IS_ERR(filp)) {
146		- ret = PTR_ERR(filp);
147		- goto err_fd;
148		- }
149		-
150		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
151		- ret = -EFAULT;
152		- goto err_file;
153		- }
154		-
155		- fd_install(resp.async_fd, filp);
	255	+ rdma_restrack_add(&ucontext->res);
156	256
157	257	/*
158	258	* Make sure that ib_uverbs_get_ucontext() sees the pointer update
..	..	@@ -161,25 +261,63 @@
161	261	smp_store_release(&file->ucontext, ucontext);
162	262
163	263	mutex_unlock(&file->ucontext_lock);
	264	+ up_read(&file->hw_destroy_rwsem);
	265	+ return 0;
164	266
165		- return in_len;
166		-
167		-err_file:
168		- ib_uverbs_free_async_event_file(file);
169		- fput(filp);
170		-
171		-err_fd:
172		- put_unused_fd(resp.async_fd);
173		-
174		-err_free:
175		- put_pid(ucontext->tgid);
176		- ib_dev->dealloc_ucontext(ucontext);
177		-
178		-err_alloc:
179		- ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
180		-
	267	+err_uncharge:
	268	+ ib_rdmacg_uncharge(&ucontext->cg_obj, ucontext->device,
	269	+ RDMACG_RESOURCE_HCA_HANDLE);
181	270	err:
182	271	mutex_unlock(&file->ucontext_lock);
	272	+ up_read(&file->hw_destroy_rwsem);
	273	+ return ret;
	274	+}
	275	+
	276	+static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
	277	+{
	278	+ struct ib_uverbs_get_context_resp resp;
	279	+ struct ib_uverbs_get_context cmd;
	280	+ struct ib_device *ib_dev;
	281	+ struct ib_uobject *uobj;
	282	+ int ret;
	283	+
	284	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	285	+ if (ret)
	286	+ return ret;
	287	+
	288	+ ret = ib_alloc_ucontext(attrs);
	289	+ if (ret)
	290	+ return ret;
	291	+
	292	+ uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev);
	293	+ if (IS_ERR(uobj)) {
	294	+ ret = PTR_ERR(uobj);
	295	+ goto err_ucontext;
	296	+ }
	297	+
	298	+ resp = (struct ib_uverbs_get_context_resp){
	299	+ .num_comp_vectors = attrs->ufile->device->num_comp_vectors,
	300	+ .async_fd = uobj->id,
	301	+ };
	302	+ ret = uverbs_response(attrs, &resp, sizeof(resp));
	303	+ if (ret)
	304	+ goto err_uobj;
	305	+
	306	+ ret = ib_init_ucontext(attrs);
	307	+ if (ret)
	308	+ goto err_uobj;
	309	+
	310	+ ib_uverbs_init_async_event_file(
	311	+ container_of(uobj, struct ib_uverbs_async_event_file, uobj));
	312	+ rdma_alloc_commit_uobject(uobj, attrs);
	313	+ return 0;
	314	+
	315	+err_uobj:
	316	+ rdma_alloc_abort_uobject(uobj, attrs, false);
	317	+err_ucontext:
	318	+ rdma_restrack_put(&attrs->context->res);
	319	+ kfree(attrs->context);
	320	+ attrs->context = NULL;
183	321	return ret;
184	322	}
185	323
..	..	@@ -221,8 +359,6 @@
221	359	resp->max_mcast_qp_attach = attr->max_mcast_qp_attach;
222	360	resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
223	361	resp->max_ah = attr->max_ah;
224		- resp->max_fmr = attr->max_fmr;
225		- resp->max_map_per_fmr = attr->max_map_per_fmr;
226	362	resp->max_srq = attr->max_srq;
227	363	resp->max_srq_wr = attr->max_srq_wr;
228	364	resp->max_srq_sge = attr->max_srq_sge;
..	..	@@ -231,57 +367,28 @@
231	367	resp->phys_port_cnt = ib_dev->phys_port_cnt;
232	368	}
233	369
234		-ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
235		- const char __user *buf,
236		- int in_len, int out_len)
	370	+static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)
237	371	{
238	372	struct ib_uverbs_query_device cmd;
239	373	struct ib_uverbs_query_device_resp resp;
240	374	struct ib_ucontext *ucontext;
	375	+ int ret;
241	376
242		- ucontext = ib_uverbs_get_ucontext(file);
	377	+ ucontext = ib_uverbs_get_ucontext(attrs);
243	378	if (IS_ERR(ucontext))
244	379	return PTR_ERR(ucontext);
245	380
246		- if (out_len < sizeof resp)
247		- return -ENOSPC;
248		-
249		- if (copy_from_user(&cmd, buf, sizeof cmd))
250		- return -EFAULT;
	381	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	382	+ if (ret)
	383	+ return ret;
251	384
252	385	memset(&resp, 0, sizeof resp);
253	386	copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs);
254	387
255		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
256		- return -EFAULT;
257		-
258		- return in_len;
	388	+ return uverbs_response(attrs, &resp, sizeof(resp));
259	389	}
260	390
261		-/*
262		- * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the
263		- * PortInfo CapabilityMask, but was extended with unique bits.
264		- */
265		-static u32 make_port_cap_flags(const struct ib_port_attr *attr)
266		-{
267		- u32 res;
268		-
269		- /* All IBA CapabilityMask bits are passed through here, except bit 26,
270		- * which is overridden with IP_BASED_GIDS. This is due to a historical
271		- * mistake in the implementation of IP_BASED_GIDS. Otherwise all other
272		- * bits match the IBA definition across all kernel versions.
273		- */
274		- res = attr->port_cap_flags & ~(u32)IB_UVERBS_PCF_IP_BASED_GIDS;
275		-
276		- if (attr->ip_gids)
277		- res \|= IB_UVERBS_PCF_IP_BASED_GIDS;
278		-
279		- return res;
280		-}
281		-
282		-ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
283		- const char __user *buf,
284		- int in_len, int out_len)
	391	+static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)
285	392	{
286	393	struct ib_uverbs_query_port cmd;
287	394	struct ib_uverbs_query_port_resp resp;
..	..	@@ -290,130 +397,84 @@
290	397	struct ib_ucontext *ucontext;
291	398	struct ib_device *ib_dev;
292	399
293		- ucontext = ib_uverbs_get_ucontext(file);
	400	+ ucontext = ib_uverbs_get_ucontext(attrs);
294	401	if (IS_ERR(ucontext))
295	402	return PTR_ERR(ucontext);
296	403	ib_dev = ucontext->device;
297	404
298		- if (out_len < sizeof resp)
299		- return -ENOSPC;
300		-
301		- if (copy_from_user(&cmd, buf, sizeof cmd))
302		- return -EFAULT;
	405	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	406	+ if (ret)
	407	+ return ret;
303	408
304	409	ret = ib_query_port(ib_dev, cmd.port_num, &attr);
305	410	if (ret)
306	411	return ret;
307	412
308	413	memset(&resp, 0, sizeof resp);
	414	+ copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num);
309	415
310		- resp.state = attr.state;
311		- resp.max_mtu = attr.max_mtu;
312		- resp.active_mtu = attr.active_mtu;
313		- resp.gid_tbl_len = attr.gid_tbl_len;
314		- resp.port_cap_flags = make_port_cap_flags(&attr);
315		- resp.max_msg_sz = attr.max_msg_sz;
316		- resp.bad_pkey_cntr = attr.bad_pkey_cntr;
317		- resp.qkey_viol_cntr = attr.qkey_viol_cntr;
318		- resp.pkey_tbl_len = attr.pkey_tbl_len;
319		-
320		- if (rdma_is_grh_required(ib_dev, cmd.port_num))
321		- resp.flags \|= IB_UVERBS_QPF_GRH_REQUIRED;
322		-
323		- if (rdma_cap_opa_ah(ib_dev, cmd.port_num)) {
324		- resp.lid = OPA_TO_IB_UCAST_LID(attr.lid);
325		- resp.sm_lid = OPA_TO_IB_UCAST_LID(attr.sm_lid);
326		- } else {
327		- resp.lid = ib_lid_cpu16(attr.lid);
328		- resp.sm_lid = ib_lid_cpu16(attr.sm_lid);
329		- }
330		- resp.lmc = attr.lmc;
331		- resp.max_vl_num = attr.max_vl_num;
332		- resp.sm_sl = attr.sm_sl;
333		- resp.subnet_timeout = attr.subnet_timeout;
334		- resp.init_type_reply = attr.init_type_reply;
335		- resp.active_width = attr.active_width;
336		- resp.active_speed = attr.active_speed;
337		- resp.phys_state = attr.phys_state;
338		- resp.link_layer = rdma_port_get_link_layer(ib_dev,
339		- cmd.port_num);
340		-
341		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
342		- return -EFAULT;
343		-
344		- return in_len;
	416	+ return uverbs_response(attrs, &resp, sizeof(resp));
345	417	}
346	418
347		-ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
348		- const char __user *buf,
349		- int in_len, int out_len)
	419	+static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
350	420	{
	421	+ struct ib_uverbs_alloc_pd_resp resp = {};
351	422	struct ib_uverbs_alloc_pd cmd;
352		- struct ib_uverbs_alloc_pd_resp resp;
353		- struct ib_udata udata;
354	423	struct ib_uobject *uobj;
355	424	struct ib_pd *pd;
356	425	int ret;
357	426	struct ib_device *ib_dev;
358	427
359		- if (out_len < sizeof resp)
360		- return -ENOSPC;
	428	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	429	+ if (ret)
	430	+ return ret;
361	431
362		- if (copy_from_user(&cmd, buf, sizeof cmd))
363		- return -EFAULT;
364		-
365		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
366		- u64_to_user_ptr(cmd.response) + sizeof(resp),
367		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
368		- out_len - sizeof(resp));
369		-
370		- uobj = uobj_alloc(UVERBS_OBJECT_PD, file, &ib_dev);
	432	+ uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);
371	433	if (IS_ERR(uobj))
372	434	return PTR_ERR(uobj);
373	435
374		- pd = ib_dev->alloc_pd(ib_dev, uobj->context, &udata);
375		- if (IS_ERR(pd)) {
376		- ret = PTR_ERR(pd);
	436	+ pd = rdma_zalloc_drv_obj(ib_dev, ib_pd);
	437	+ if (!pd) {
	438	+ ret = -ENOMEM;
377	439	goto err;
378	440	}
379	441
380	442	pd->device = ib_dev;
381	443	pd->uobject = uobj;
382		- pd->__internal_mr = NULL;
383	444	atomic_set(&pd->usecnt, 0);
384	445
385		- uobj->object = pd;
386		- memset(&resp, 0, sizeof resp);
387		- resp.pd_handle = uobj->id;
388		- pd->res.type = RDMA_RESTRACK_PD;
	446	+ rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
	447	+ rdma_restrack_set_name(&pd->res, NULL);
	448	+
	449	+ ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
	450	+ if (ret)
	451	+ goto err_alloc;
389	452	rdma_restrack_add(&pd->res);
390	453
391		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
392		- ret = -EFAULT;
393		- goto err_copy;
394		- }
	454	+ uobj->object = pd;
	455	+ uobj_finalize_uobj_create(uobj, attrs);
395	456
396		- return uobj_alloc_commit(uobj, in_len);
	457	+ resp.pd_handle = uobj->id;
	458	+ return uverbs_response(attrs, &resp, sizeof(resp));
397	459
398		-err_copy:
399		- ib_dealloc_pd(pd);
400		-
	460	+err_alloc:
	461	+ rdma_restrack_put(&pd->res);
	462	+ kfree(pd);
401	463	err:
402		- uobj_alloc_abort(uobj);
	464	+ uobj_alloc_abort(uobj, attrs);
403	465	return ret;
404	466	}
405	467
406		-ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file,
407		- const char __user *buf,
408		- int in_len, int out_len)
	468	+static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)
409	469	{
410	470	struct ib_uverbs_dealloc_pd cmd;
	471	+ int ret;
411	472
412		- if (copy_from_user(&cmd, buf, sizeof cmd))
413		- return -EFAULT;
	473	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	474	+ if (ret)
	475	+ return ret;
414	476
415		- return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, file,
416		- in_len);
	477	+ return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
417	478	}
418	479
419	480	struct xrcd_table_entry {
..	..	@@ -501,33 +562,24 @@
501	562	}
502	563	}
503	564
504		-ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
505		- const char __user *buf, int in_len,
506		- int out_len)
	565	+static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
507	566	{
	567	+ struct ib_uverbs_device *ibudev = attrs->ufile->device;
	568	+ struct ib_uverbs_open_xrcd_resp resp = {};
508	569	struct ib_uverbs_open_xrcd cmd;
509		- struct ib_uverbs_open_xrcd_resp resp;
510		- struct ib_udata udata;
511	570	struct ib_uxrcd_object *obj;
512	571	struct ib_xrcd *xrcd = NULL;
513		- struct fd f = {NULL, 0};
514	572	struct inode *inode = NULL;
515		- int ret = 0;
516	573	int new_xrcd = 0;
517	574	struct ib_device *ib_dev;
	575	+ struct fd f = {};
	576	+ int ret;
518	577
519		- if (out_len < sizeof resp)
520		- return -ENOSPC;
	578	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	579	+ if (ret)
	580	+ return ret;
521	581
522		- if (copy_from_user(&cmd, buf, sizeof cmd))
523		- return -EFAULT;
524		-
525		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
526		- u64_to_user_ptr(cmd.response) + sizeof(resp),
527		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
528		- out_len - sizeof(resp));
529		-
530		- mutex_lock(&file->device->xrcd_tree_mutex);
	582	+ mutex_lock(&ibudev->xrcd_tree_mutex);
531	583
532	584	if (cmd.fd != -1) {
533	585	/* search for file descriptor */
..	..	@@ -538,7 +590,7 @@
538	590	}
539	591
540	592	inode = file_inode(f.file);
541		- xrcd = find_xrcd(file->device, inode);
	593	+ xrcd = find_xrcd(ibudev, inode);
542	594	if (!xrcd && !(cmd.oflags & O_CREAT)) {
543	595	/* no file descriptor. Need CREATE flag */
544	596	ret = -EAGAIN;
..	..	@@ -551,7 +603,7 @@
551	603	}
552	604	}
553	605
554		- obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, file,
	606	+ obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,
555	607	&ib_dev);
556	608	if (IS_ERR(obj)) {
557	609	ret = PTR_ERR(obj);
..	..	@@ -559,95 +611,76 @@
559	611	}
560	612
561	613	if (!xrcd) {
562		- xrcd = ib_dev->alloc_xrcd(ib_dev, obj->uobject.context, &udata);
	614	+ xrcd = ib_alloc_xrcd_user(ib_dev, inode, &attrs->driver_udata);
563	615	if (IS_ERR(xrcd)) {
564	616	ret = PTR_ERR(xrcd);
565	617	goto err;
566	618	}
567		-
568		- xrcd->inode = inode;
569		- xrcd->device = ib_dev;
570		- atomic_set(&xrcd->usecnt, 0);
571		- mutex_init(&xrcd->tgt_qp_mutex);
572		- INIT_LIST_HEAD(&xrcd->tgt_qp_list);
573	619	new_xrcd = 1;
574	620	}
575	621
576	622	atomic_set(&obj->refcnt, 0);
577	623	obj->uobject.object = xrcd;
578		- memset(&resp, 0, sizeof resp);
579		- resp.xrcd_handle = obj->uobject.id;
580	624
581	625	if (inode) {
582	626	if (new_xrcd) {
583	627	/* create new inode/xrcd table entry */
584		- ret = xrcd_table_insert(file->device, inode, xrcd);
	628	+ ret = xrcd_table_insert(ibudev, inode, xrcd);
585	629	if (ret)
586	630	goto err_dealloc_xrcd;
587	631	}
588	632	atomic_inc(&xrcd->usecnt);
589	633	}
590	634
591		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
592		- ret = -EFAULT;
593		- goto err_copy;
594		- }
595		-
596	635	if (f.file)
597	636	fdput(f);
598	637
599		- mutex_unlock(&file->device->xrcd_tree_mutex);
	638	+ mutex_unlock(&ibudev->xrcd_tree_mutex);
	639	+ uobj_finalize_uobj_create(&obj->uobject, attrs);
600	640
601		- return uobj_alloc_commit(&obj->uobject, in_len);
602		-
603		-err_copy:
604		- if (inode) {
605		- if (new_xrcd)
606		- xrcd_table_delete(file->device, inode);
607		- atomic_dec(&xrcd->usecnt);
608		- }
	641	+ resp.xrcd_handle = obj->uobject.id;
	642	+ return uverbs_response(attrs, &resp, sizeof(resp));
609	643
610	644	err_dealloc_xrcd:
611		- ib_dealloc_xrcd(xrcd);
	645	+ ib_dealloc_xrcd_user(xrcd, uverbs_get_cleared_udata(attrs));
612	646
613	647	err:
614		- uobj_alloc_abort(&obj->uobject);
	648	+ uobj_alloc_abort(&obj->uobject, attrs);
615	649
616	650	err_tree_mutex_unlock:
617	651	if (f.file)
618	652	fdput(f);
619	653
620		- mutex_unlock(&file->device->xrcd_tree_mutex);
	654	+ mutex_unlock(&ibudev->xrcd_tree_mutex);
621	655
622	656	return ret;
623	657	}
624	658
625		-ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file,
626		- const char __user *buf, int in_len,
627		- int out_len)
	659	+static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
628	660	{
629	661	struct ib_uverbs_close_xrcd cmd;
	662	+ int ret;
630	663
631		- if (copy_from_user(&cmd, buf, sizeof cmd))
632		- return -EFAULT;
	664	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	665	+ if (ret)
	666	+ return ret;
633	667
634		- return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, file,
635		- in_len);
	668	+ return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
636	669	}
637	670
638		-int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject,
639		- struct ib_xrcd *xrcd,
640		- enum rdma_remove_reason why)
	671	+int ib_uverbs_dealloc_xrcd(struct ib_uobject uobject, struct ib_xrcd xrcd,
	672	+ enum rdma_remove_reason why,
	673	+ struct uverbs_attr_bundle *attrs)
641	674	{
642	675	struct inode *inode;
643	676	int ret;
644		- struct ib_uverbs_device *dev = uobject->context->ufile->device;
	677	+ struct ib_uverbs_device *dev = attrs->ufile->device;
645	678
646	679	inode = xrcd->inode;
647	680	if (inode && !atomic_dec_and_test(&xrcd->usecnt))
648	681	return 0;
649	682
650		- ret = ib_dealloc_xrcd(xrcd);
	683	+ ret = ib_dealloc_xrcd_user(xrcd, &attrs->driver_udata);
651	684
652	685	if (ib_is_destroy_retryable(ret, why, uobject)) {
653	686	atomic_inc(&xrcd->usecnt);
..	..	@@ -660,29 +693,19 @@
660	693	return ret;
661	694	}
662	695
663		-ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
664		- const char __user *buf, int in_len,
665		- int out_len)
	696	+static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
666	697	{
	698	+ struct ib_uverbs_reg_mr_resp resp = {};
667	699	struct ib_uverbs_reg_mr cmd;
668		- struct ib_uverbs_reg_mr_resp resp;
669		- struct ib_udata udata;
670	700	struct ib_uobject *uobj;
671	701	struct ib_pd *pd;
672	702	struct ib_mr *mr;
673	703	int ret;
674	704	struct ib_device *ib_dev;
675	705
676		- if (out_len < sizeof resp)
677		- return -ENOSPC;
678		-
679		- if (copy_from_user(&cmd, buf, sizeof cmd))
680		- return -EFAULT;
681		-
682		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
683		- u64_to_user_ptr(cmd.response) + sizeof(resp),
684		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
685		- out_len - sizeof(resp));
	706	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	707	+ if (ret)
	708	+ return ret;
686	709
687	710	if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
688	711	return -EINVAL;
..	..	@@ -691,11 +714,11 @@
691	714	if (ret)
692	715	return ret;
693	716
694		- uobj = uobj_alloc(UVERBS_OBJECT_MR, file, &ib_dev);
	717	+ uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
695	718	if (IS_ERR(uobj))
696	719	return PTR_ERR(uobj);
697	720
698		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file);
	721	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
699	722	if (!pd) {
700	723	ret = -EINVAL;
701	724	goto err_free;
..	..	@@ -710,8 +733,9 @@
710	733	}
711	734	}
712	735
713		- mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
714		- cmd.access_flags, &udata);
	736	+ mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
	737	+ cmd.access_flags,
	738	+ &attrs->driver_udata);
715	739	if (IS_ERR(mr)) {
716	740	ret = PTR_ERR(mr);
717	741	goto err_put;
..	..	@@ -719,62 +743,47 @@
719	743
720	744	mr->device = pd->device;
721	745	mr->pd = pd;
	746	+ mr->type = IB_MR_TYPE_USER;
722	747	mr->dm = NULL;
	748	+ mr->sig_attrs = NULL;
723	749	mr->uobject = uobj;
724	750	atomic_inc(&pd->usecnt);
725		- mr->res.type = RDMA_RESTRACK_MR;
	751	+ mr->iova = cmd.hca_va;
	752	+ mr->length = cmd.length;
	753	+
	754	+ rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
	755	+ rdma_restrack_set_name(&mr->res, NULL);
726	756	rdma_restrack_add(&mr->res);
727	757
728	758	uobj->object = mr;
729		-
730		- memset(&resp, 0, sizeof resp);
731		- resp.lkey = mr->lkey;
732		- resp.rkey = mr->rkey;
733		- resp.mr_handle = uobj->id;
734		-
735		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
736		- ret = -EFAULT;
737		- goto err_copy;
738		- }
739		-
740	759	uobj_put_obj_read(pd);
	760	+ uobj_finalize_uobj_create(uobj, attrs);
741	761
742		- return uobj_alloc_commit(uobj, in_len);
743		-
744		-err_copy:
745		- ib_dereg_mr(mr);
	762	+ resp.lkey = mr->lkey;
	763	+ resp.rkey = mr->rkey;
	764	+ resp.mr_handle = uobj->id;
	765	+ return uverbs_response(attrs, &resp, sizeof(resp));
746	766
747	767	err_put:
748	768	uobj_put_obj_read(pd);
749		-
750	769	err_free:
751		- uobj_alloc_abort(uobj);
	770	+ uobj_alloc_abort(uobj, attrs);
752	771	return ret;
753	772	}
754	773
755		-ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file,
756		- const char __user *buf, int in_len,
757		- int out_len)
	774	+static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
758	775	{
759	776	struct ib_uverbs_rereg_mr cmd;
760	777	struct ib_uverbs_rereg_mr_resp resp;
761		- struct ib_udata udata;
762	778	struct ib_pd *pd = NULL;
763	779	struct ib_mr *mr;
764	780	struct ib_pd *old_pd;
765	781	int ret;
766	782	struct ib_uobject *uobj;
767	783
768		- if (out_len < sizeof(resp))
769		- return -ENOSPC;
770		-
771		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
772		- return -EFAULT;
773		-
774		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
775		- u64_to_user_ptr(cmd.response) + sizeof(resp),
776		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
777		- out_len - sizeof(resp));
	784	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	785	+ if (ret)
	786	+ return ret;
778	787
779	788	if (cmd.flags & ~IB_MR_REREG_SUPPORTED \|\| !cmd.flags)
780	789	return -EINVAL;
..	..	@@ -784,7 +793,7 @@
784	793	(cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
785	794	return -EINVAL;
786	795
787		- uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, file);
	796	+ uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
788	797	if (IS_ERR(uobj))
789	798	return PTR_ERR(uobj);
790	799
..	..	@@ -803,7 +812,7 @@
803	812
804	813	if (cmd.flags & IB_MR_REREG_PD) {
805	814	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
806		- file);
	815	+ attrs);
807	816	if (!pd) {
808	817	ret = -EINVAL;
809	818	goto put_uobjs;
..	..	@@ -811,27 +820,29 @@
811	820	}
812	821
813	822	old_pd = mr->pd;
814		- ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start,
815		- cmd.length, cmd.hca_va,
816		- cmd.access_flags, pd, &udata);
817		- if (!ret) {
818		- if (cmd.flags & IB_MR_REREG_PD) {
819		- atomic_inc(&pd->usecnt);
820		- mr->pd = pd;
821		- atomic_dec(&old_pd->usecnt);
822		- }
823		- } else {
	823	+ ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start,
	824	+ cmd.length, cmd.hca_va,
	825	+ cmd.access_flags, pd,
	826	+ &attrs->driver_udata);
	827	+ if (ret)
824	828	goto put_uobj_pd;
	829	+
	830	+ if (cmd.flags & IB_MR_REREG_PD) {
	831	+ atomic_inc(&pd->usecnt);
	832	+ mr->pd = pd;
	833	+ atomic_dec(&old_pd->usecnt);
	834	+ }
	835	+
	836	+ if (cmd.flags & IB_MR_REREG_TRANS) {
	837	+ mr->iova = cmd.hca_va;
	838	+ mr->length = cmd.length;
825	839	}
826	840
827	841	memset(&resp, 0, sizeof(resp));
828	842	resp.lkey = mr->lkey;
829	843	resp.rkey = mr->rkey;
830	844
831		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp)))
832		- ret = -EFAULT;
833		- else
834		- ret = in_len;
	845	+ ret = uverbs_response(attrs, &resp, sizeof(resp));
835	846
836	847	put_uobj_pd:
837	848	if (cmd.flags & IB_MR_REREG_PD)
..	..	@@ -843,268 +854,205 @@
843	854	return ret;
844	855	}
845	856
846		-ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file,
847		- const char __user *buf, int in_len,
848		- int out_len)
	857	+static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)
849	858	{
850	859	struct ib_uverbs_dereg_mr cmd;
	860	+ int ret;
851	861
852		- if (copy_from_user(&cmd, buf, sizeof cmd))
853		- return -EFAULT;
	862	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	863	+ if (ret)
	864	+ return ret;
854	865
855		- return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, file,
856		- in_len);
	866	+ return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
857	867	}
858	868
859		-ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file,
860		- const char __user *buf, int in_len,
861		- int out_len)
	869	+static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
862	870	{
863	871	struct ib_uverbs_alloc_mw cmd;
864		- struct ib_uverbs_alloc_mw_resp resp;
	872	+ struct ib_uverbs_alloc_mw_resp resp = {};
865	873	struct ib_uobject *uobj;
866	874	struct ib_pd *pd;
867	875	struct ib_mw *mw;
868		- struct ib_udata udata;
869	876	int ret;
870	877	struct ib_device *ib_dev;
871	878
872		- if (out_len < sizeof(resp))
873		- return -ENOSPC;
	879	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	880	+ if (ret)
	881	+ return ret;
874	882
875		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
876		- return -EFAULT;
877		-
878		- uobj = uobj_alloc(UVERBS_OBJECT_MW, file, &ib_dev);
	883	+ uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);
879	884	if (IS_ERR(uobj))
880	885	return PTR_ERR(uobj);
881	886
882		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file);
	887	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
883	888	if (!pd) {
884	889	ret = -EINVAL;
885	890	goto err_free;
886	891	}
887	892
888		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
889		- u64_to_user_ptr(cmd.response) + sizeof(resp),
890		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
891		- out_len - sizeof(resp));
892		-
893		- mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata);
894		- if (IS_ERR(mw)) {
895		- ret = PTR_ERR(mw);
	893	+ if (cmd.mw_type != IB_MW_TYPE_1 && cmd.mw_type != IB_MW_TYPE_2) {
	894	+ ret = -EINVAL;
896	895	goto err_put;
897	896	}
898	897
899		- mw->device = pd->device;
900		- mw->pd = pd;
	898	+ mw = rdma_zalloc_drv_obj(ib_dev, ib_mw);
	899	+ if (!mw) {
	900	+ ret = -ENOMEM;
	901	+ goto err_put;
	902	+ }
	903	+
	904	+ mw->device = ib_dev;
	905	+ mw->pd = pd;
901	906	mw->uobject = uobj;
	907	+ mw->type = cmd.mw_type;
	908	+
	909	+ ret = pd->device->ops.alloc_mw(mw, &attrs->driver_udata);
	910	+ if (ret)
	911	+ goto err_alloc;
	912	+
902	913	atomic_inc(&pd->usecnt);
903	914
904	915	uobj->object = mw;
905		-
906		- memset(&resp, 0, sizeof(resp));
907		- resp.rkey = mw->rkey;
908		- resp.mw_handle = uobj->id;
909		-
910		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) {
911		- ret = -EFAULT;
912		- goto err_copy;
913		- }
914		-
915	916	uobj_put_obj_read(pd);
916		- return uobj_alloc_commit(uobj, in_len);
	917	+ uobj_finalize_uobj_create(uobj, attrs);
917	918
918		-err_copy:
919		- uverbs_dealloc_mw(mw);
	919	+ resp.rkey = mw->rkey;
	920	+ resp.mw_handle = uobj->id;
	921	+ return uverbs_response(attrs, &resp, sizeof(resp));
	922	+
	923	+err_alloc:
	924	+ kfree(mw);
920	925	err_put:
921	926	uobj_put_obj_read(pd);
922	927	err_free:
923		- uobj_alloc_abort(uobj);
	928	+ uobj_alloc_abort(uobj, attrs);
924	929	return ret;
925	930	}
926	931
927		-ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file,
928		- const char __user *buf, int in_len,
929		- int out_len)
	932	+static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)
930	933	{
931	934	struct ib_uverbs_dealloc_mw cmd;
	935	+ int ret;
932	936
933		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
934		- return -EFAULT;
	937	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	938	+ if (ret)
	939	+ return ret;
935	940
936		- return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, file,
937		- in_len);
	941	+ return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);
938	942	}
939	943
940		-ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file,
941		- const char __user *buf, int in_len,
942		- int out_len)
	944	+static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
943	945	{
944	946	struct ib_uverbs_create_comp_channel cmd;
945	947	struct ib_uverbs_create_comp_channel_resp resp;
946	948	struct ib_uobject *uobj;
947	949	struct ib_uverbs_completion_event_file *ev_file;
948	950	struct ib_device *ib_dev;
	951	+ int ret;
949	952
950		- if (out_len < sizeof resp)
951		- return -ENOSPC;
	953	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	954	+ if (ret)
	955	+ return ret;
952	956
953		- if (copy_from_user(&cmd, buf, sizeof cmd))
954		- return -EFAULT;
955		-
956		- uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, file, &ib_dev);
	957	+ uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);
957	958	if (IS_ERR(uobj))
958	959	return PTR_ERR(uobj);
959		-
960		- resp.fd = uobj->id;
961	960
962	961	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
963	962	uobj);
964	963	ib_uverbs_init_event_queue(&ev_file->ev_queue);
	964	+ uobj_finalize_uobj_create(uobj, attrs);
965	965
966		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
967		- uobj_alloc_abort(uobj);
968		- return -EFAULT;
969		- }
970		-
971		- return uobj_alloc_commit(uobj, in_len);
	966	+ resp.fd = uobj->id;
	967	+ return uverbs_response(attrs, &resp, sizeof(resp));
972	968	}
973	969
974		-static struct ib_ucq_object create_cq(struct ib_uverbs_file file,
975		- struct ib_udata *ucore,
976		- struct ib_udata *uhw,
977		- struct ib_uverbs_ex_create_cq *cmd,
978		- size_t cmd_sz,
979		- int (cb)(struct ib_uverbs_file file,
980		- struct ib_ucq_object *obj,
981		- struct ib_uverbs_ex_create_cq_resp *resp,
982		- struct ib_udata *udata,
983		- void *context),
984		- void *context)
	970	+static int create_cq(struct uverbs_attr_bundle *attrs,
	971	+ struct ib_uverbs_ex_create_cq *cmd)
985	972	{
986	973	struct ib_ucq_object *obj;
987	974	struct ib_uverbs_completion_event_file *ev_file = NULL;
988	975	struct ib_cq *cq;
989	976	int ret;
990		- struct ib_uverbs_ex_create_cq_resp resp;
	977	+ struct ib_uverbs_ex_create_cq_resp resp = {};
991	978	struct ib_cq_init_attr attr = {};
992	979	struct ib_device *ib_dev;
993	980
994		- if (cmd->comp_vector >= file->device->num_comp_vectors)
995		- return ERR_PTR(-EINVAL);
	981	+ if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
	982	+ return -EINVAL;
996	983
997		- obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, file,
	984	+ obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
998	985	&ib_dev);
999	986	if (IS_ERR(obj))
1000		- return obj;
1001		-
1002		- if (!ib_dev->create_cq) {
1003		- ret = -EOPNOTSUPP;
1004		- goto err;
1005		- }
	987	+ return PTR_ERR(obj);
1006	988
1007	989	if (cmd->comp_channel >= 0) {
1008		- ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, file);
	990	+ ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
1009	991	if (IS_ERR(ev_file)) {
1010	992	ret = PTR_ERR(ev_file);
1011	993	goto err;
1012	994	}
1013	995	}
1014	996
1015		- obj->uobject.user_handle = cmd->user_handle;
1016		- obj->comp_events_reported = 0;
1017		- obj->async_events_reported = 0;
	997	+ obj->uevent.uobject.user_handle = cmd->user_handle;
1018	998	INIT_LIST_HEAD(&obj->comp_list);
1019		- INIT_LIST_HEAD(&obj->async_list);
	999	+ INIT_LIST_HEAD(&obj->uevent.event_list);
1020	1000
1021	1001	attr.cqe = cmd->cqe;
1022	1002	attr.comp_vector = cmd->comp_vector;
	1003	+ attr.flags = cmd->flags;
1023	1004
1024		- if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags))
1025		- attr.flags = cmd->flags;
1026		-
1027		- cq = ib_dev->create_cq(ib_dev, &attr, obj->uobject.context, uhw);
1028		- if (IS_ERR(cq)) {
1029		- ret = PTR_ERR(cq);
	1005	+ cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
	1006	+ if (!cq) {
	1007	+ ret = -ENOMEM;
1030	1008	goto err_file;
1031	1009	}
1032		-
1033	1010	cq->device = ib_dev;
1034		- cq->uobject = &obj->uobject;
	1011	+ cq->uobject = obj;
1035	1012	cq->comp_handler = ib_uverbs_comp_handler;
1036	1013	cq->event_handler = ib_uverbs_cq_event_handler;
1037	1014	cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
1038	1015	atomic_set(&cq->usecnt, 0);
1039	1016
1040		- obj->uobject.object = cq;
1041		- memset(&resp, 0, sizeof resp);
1042		- resp.base.cq_handle = obj->uobject.id;
1043		- resp.base.cqe = cq->cqe;
	1017	+ rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
	1018	+ rdma_restrack_set_name(&cq->res, NULL);
1044	1019
1045		- resp.response_length = offsetof(typeof(resp), response_length) +
1046		- sizeof(resp.response_length);
1047		-
1048		- cq->res.type = RDMA_RESTRACK_CQ;
	1020	+ ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
	1021	+ if (ret)
	1022	+ goto err_free;
1049	1023	rdma_restrack_add(&cq->res);
1050	1024
1051		- ret = cb(file, obj, &resp, ucore, context);
1052		- if (ret)
1053		- goto err_cb;
	1025	+ obj->uevent.uobject.object = cq;
	1026	+ obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
	1027	+ if (obj->uevent.event_file)
	1028	+ uverbs_uobject_get(&obj->uevent.event_file->uobj);
	1029	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1054	1030
1055		- ret = uobj_alloc_commit(&obj->uobject, 0);
1056		- if (ret)
1057		- return ERR_PTR(ret);
1058		- return obj;
	1031	+ resp.base.cq_handle = obj->uevent.uobject.id;
	1032	+ resp.base.cqe = cq->cqe;
	1033	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	1034	+ return uverbs_response(attrs, &resp, sizeof(resp));
1059	1035
1060		-err_cb:
1061		- ib_destroy_cq(cq);
1062		-
	1036	+err_free:
	1037	+ rdma_restrack_put(&cq->res);
	1038	+ kfree(cq);
1063	1039	err_file:
1064	1040	if (ev_file)
1065		- ib_uverbs_release_ucq(file, ev_file, obj);
1066		-
	1041	+ ib_uverbs_release_ucq(ev_file, obj);
1067	1042	err:
1068		- uobj_alloc_abort(&obj->uobject);
1069		-
1070		- return ERR_PTR(ret);
	1043	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
	1044	+ return ret;
1071	1045	}
1072	1046
1073		-static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file,
1074		- struct ib_ucq_object *obj,
1075		- struct ib_uverbs_ex_create_cq_resp *resp,
1076		- struct ib_udata ucore, void context)
1077		-{
1078		- if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1079		- return -EFAULT;
1080		-
1081		- return 0;
1082		-}
1083		-
1084		-ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
1085		- const char __user *buf, int in_len,
1086		- int out_len)
	1047	+static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
1087	1048	{
1088	1049	struct ib_uverbs_create_cq cmd;
1089	1050	struct ib_uverbs_ex_create_cq cmd_ex;
1090		- struct ib_uverbs_create_cq_resp resp;
1091		- struct ib_udata ucore;
1092		- struct ib_udata uhw;
1093		- struct ib_ucq_object *obj;
	1051	+ int ret;
1094	1052
1095		- if (out_len < sizeof(resp))
1096		- return -ENOSPC;
1097		-
1098		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
1099		- return -EFAULT;
1100		-
1101		- ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response),
1102		- sizeof(cmd), sizeof(resp));
1103		-
1104		- ib_uverbs_init_udata(&uhw, buf + sizeof(cmd),
1105		- u64_to_user_ptr(cmd.response) + sizeof(resp),
1106		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1107		- out_len - sizeof(resp));
	1053	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1054	+ if (ret)
	1055	+ return ret;
1108	1056
1109	1057	memset(&cmd_ex, 0, sizeof(cmd_ex));
1110	1058	cmd_ex.user_handle = cmd.user_handle;
..	..	@@ -1112,43 +1060,17 @@
1112	1060	cmd_ex.comp_vector = cmd.comp_vector;
1113	1061	cmd_ex.comp_channel = cmd.comp_channel;
1114	1062
1115		- obj = create_cq(file, &ucore, &uhw, &cmd_ex,
1116		- offsetof(typeof(cmd_ex), comp_channel) +
1117		- sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb,
1118		- NULL);
1119		-
1120		- if (IS_ERR(obj))
1121		- return PTR_ERR(obj);
1122		-
1123		- return in_len;
	1063	+ return create_cq(attrs, &cmd_ex);
1124	1064	}
1125	1065
1126		-static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file,
1127		- struct ib_ucq_object *obj,
1128		- struct ib_uverbs_ex_create_cq_resp *resp,
1129		- struct ib_udata ucore, void context)
	1066	+static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
1130	1067	{
1131		- if (ib_copy_to_udata(ucore, resp, resp->response_length))
1132		- return -EFAULT;
1133		-
1134		- return 0;
1135		-}
1136		-
1137		-int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file,
1138		- struct ib_udata *ucore,
1139		- struct ib_udata *uhw)
1140		-{
1141		- struct ib_uverbs_ex_create_cq_resp resp;
1142	1068	struct ib_uverbs_ex_create_cq cmd;
1143		- struct ib_ucq_object *obj;
1144		- int err;
	1069	+ int ret;
1145	1070
1146		- if (ucore->inlen < sizeof(cmd))
1147		- return -EINVAL;
1148		-
1149		- err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
1150		- if (err)
1151		- return err;
	1071	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1072	+ if (ret)
	1073	+ return ret;
1152	1074
1153	1075	if (cmd.comp_mask)
1154	1076	return -EINVAL;
..	..	@@ -1156,52 +1078,36 @@
1156	1078	if (cmd.reserved)
1157	1079	return -EINVAL;
1158	1080
1159		- if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1160		- sizeof(resp.response_length)))
1161		- return -ENOSPC;
1162		-
1163		- obj = create_cq(file, ucore, uhw, &cmd,
1164		- min(ucore->inlen, sizeof(cmd)),
1165		- ib_uverbs_ex_create_cq_cb, NULL);
1166		-
1167		- return PTR_ERR_OR_ZERO(obj);
	1081	+ return create_cq(attrs, &cmd);
1168	1082	}
1169	1083
1170		-ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
1171		- const char __user *buf, int in_len,
1172		- int out_len)
	1084	+static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
1173	1085	{
1174	1086	struct ib_uverbs_resize_cq cmd;
1175	1087	struct ib_uverbs_resize_cq_resp resp = {};
1176		- struct ib_udata udata;
1177	1088	struct ib_cq *cq;
1178		- int ret = -EINVAL;
	1089	+ int ret;
1179	1090
1180		- if (copy_from_user(&cmd, buf, sizeof cmd))
1181		- return -EFAULT;
	1091	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1092	+ if (ret)
	1093	+ return ret;
1182	1094
1183		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
1184		- u64_to_user_ptr(cmd.response) + sizeof(resp),
1185		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1186		- out_len - sizeof(resp));
1187		-
1188		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	1095	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1189	1096	if (!cq)
1190	1097	return -EINVAL;
1191	1098
1192		- ret = cq->device->resize_cq(cq, cmd.cqe, &udata);
	1099	+ ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);
1193	1100	if (ret)
1194	1101	goto out;
1195	1102
1196	1103	resp.cqe = cq->cqe;
1197	1104
1198		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp.cqe))
1199		- ret = -EFAULT;
1200		-
	1105	+ ret = uverbs_response(attrs, &resp, sizeof(resp));
1201	1106	out:
1202		- uobj_put_obj_read(cq);
	1107	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	1108	+ UVERBS_LOOKUP_READ);
1203	1109
1204		- return ret ? ret : in_len;
	1110	+ return ret;
1205	1111	}
1206	1112
1207	1113	static int copy_wc_to_user(struct ib_device ib_dev, void __user dest,
..	..	@@ -1234,9 +1140,7 @@
1234	1140	return 0;
1235	1141	}
1236	1142
1237		-ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
1238		- const char __user *buf, int in_len,
1239		- int out_len)
	1143	+static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
1240	1144	{
1241	1145	struct ib_uverbs_poll_cq cmd;
1242	1146	struct ib_uverbs_poll_cq_resp resp;
..	..	@@ -1246,15 +1150,16 @@
1246	1150	struct ib_wc wc;
1247	1151	int ret;
1248	1152
1249		- if (copy_from_user(&cmd, buf, sizeof cmd))
1250		- return -EFAULT;
	1153	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1154	+ if (ret)
	1155	+ return ret;
1251	1156
1252		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	1157	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1253	1158	if (!cq)
1254	1159	return -EINVAL;
1255	1160
1256	1161	/* we copy a struct ib_uverbs_poll_cq_resp to user space */
1257		- header_ptr = u64_to_user_ptr(cmd.response);
	1162	+ header_ptr = attrs->ucore.outbuf;
1258	1163	data_ptr = header_ptr + sizeof resp;
1259	1164
1260	1165	memset(&resp, 0, sizeof resp);
..	..	@@ -1277,74 +1182,67 @@
1277	1182	ret = -EFAULT;
1278	1183	goto out_put;
1279	1184	}
	1185	+ ret = 0;
1280	1186
1281		- ret = in_len;
	1187	+ if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT))
	1188	+ ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT);
1282	1189
1283	1190	out_put:
1284		- uobj_put_obj_read(cq);
	1191	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	1192	+ UVERBS_LOOKUP_READ);
1285	1193	return ret;
1286	1194	}
1287	1195
1288		-ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file,
1289		- const char __user *buf, int in_len,
1290		- int out_len)
	1196	+static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)
1291	1197	{
1292	1198	struct ib_uverbs_req_notify_cq cmd;
1293	1199	struct ib_cq *cq;
	1200	+ int ret;
1294	1201
1295		- if (copy_from_user(&cmd, buf, sizeof cmd))
1296		- return -EFAULT;
	1202	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1203	+ if (ret)
	1204	+ return ret;
1297	1205
1298		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	1206	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1299	1207	if (!cq)
1300	1208	return -EINVAL;
1301	1209
1302	1210	ib_req_notify_cq(cq, cmd.solicited_only ?
1303	1211	IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1304	1212
1305		- uobj_put_obj_read(cq);
1306		-
1307		- return in_len;
	1213	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	1214	+ UVERBS_LOOKUP_READ);
	1215	+ return 0;
1308	1216	}
1309	1217
1310		-ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
1311		- const char __user *buf, int in_len,
1312		- int out_len)
	1218	+static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)
1313	1219	{
1314	1220	struct ib_uverbs_destroy_cq cmd;
1315	1221	struct ib_uverbs_destroy_cq_resp resp;
1316	1222	struct ib_uobject *uobj;
1317	1223	struct ib_ucq_object *obj;
	1224	+ int ret;
1318	1225
1319		- if (copy_from_user(&cmd, buf, sizeof cmd))
1320		- return -EFAULT;
	1226	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1227	+ if (ret)
	1228	+ return ret;
1321	1229
1322		- uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	1230	+ uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1323	1231	if (IS_ERR(uobj))
1324	1232	return PTR_ERR(uobj);
1325	1233
1326		- obj = container_of(uobj, struct ib_ucq_object, uobject);
	1234	+ obj = container_of(uobj, struct ib_ucq_object, uevent.uobject);
1327	1235	memset(&resp, 0, sizeof(resp));
1328	1236	resp.comp_events_reported = obj->comp_events_reported;
1329		- resp.async_events_reported = obj->async_events_reported;
	1237	+ resp.async_events_reported = obj->uevent.events_reported;
1330	1238
1331	1239	uobj_put_destroy(uobj);
1332	1240
1333		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
1334		- return -EFAULT;
1335		-
1336		- return in_len;
	1241	+ return uverbs_response(attrs, &resp, sizeof(resp));
1337	1242	}
1338	1243
1339		-static int create_qp(struct ib_uverbs_file *file,
1340		- struct ib_udata *ucore,
1341		- struct ib_udata *uhw,
1342		- struct ib_uverbs_ex_create_qp *cmd,
1343		- size_t cmd_sz,
1344		- int (cb)(struct ib_uverbs_file file,
1345		- struct ib_uverbs_ex_create_qp_resp *resp,
1346		- struct ib_udata *udata),
1347		- void *context)
	1244	+static int create_qp(struct uverbs_attr_bundle *attrs,
	1245	+ struct ib_uverbs_ex_create_qp *cmd)
1348	1246	{
1349	1247	struct ib_uqp_object *obj;
1350	1248	struct ib_device *device;
..	..	@@ -1354,18 +1252,30 @@
1354	1252	struct ib_cq scq = NULL, rcq = NULL;
1355	1253	struct ib_srq *srq = NULL;
1356	1254	struct ib_qp *qp;
1357		- char *buf;
1358	1255	struct ib_qp_init_attr attr = {};
1359		- struct ib_uverbs_ex_create_qp_resp resp;
	1256	+ struct ib_uverbs_ex_create_qp_resp resp = {};
1360	1257	int ret;
1361	1258	struct ib_rwq_ind_table *ind_tbl = NULL;
1362	1259	bool has_sq = true;
1363	1260	struct ib_device *ib_dev;
1364	1261
1365		- if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
1366		- return -EPERM;
	1262	+ switch (cmd->qp_type) {
	1263	+ case IB_QPT_RAW_PACKET:
	1264	+ if (!capable(CAP_NET_RAW))
	1265	+ return -EPERM;
	1266	+ break;
	1267	+ case IB_QPT_RC:
	1268	+ case IB_QPT_UC:
	1269	+ case IB_QPT_UD:
	1270	+ case IB_QPT_XRC_INI:
	1271	+ case IB_QPT_XRC_TGT:
	1272	+ case IB_QPT_DRIVER:
	1273	+ break;
	1274	+ default:
	1275	+ return -EINVAL;
	1276	+ }
1367	1277
1368		- obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, file,
	1278	+ obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1369	1279	&ib_dev);
1370	1280	if (IS_ERR(obj))
1371	1281	return PTR_ERR(obj);
..	..	@@ -1373,25 +1283,16 @@
1373	1283	obj->uevent.uobject.user_handle = cmd->user_handle;
1374	1284	mutex_init(&obj->mcast_lock);
1375	1285
1376		- if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) +
1377		- sizeof(cmd->rwq_ind_tbl_handle) &&
1378		- (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) {
	1286	+ if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {
1379	1287	ind_tbl = uobj_get_obj_read(rwq_ind_table,
1380	1288	UVERBS_OBJECT_RWQ_IND_TBL,
1381		- cmd->rwq_ind_tbl_handle, file);
	1289	+ cmd->rwq_ind_tbl_handle, attrs);
1382	1290	if (!ind_tbl) {
1383	1291	ret = -EINVAL;
1384	1292	goto err_put;
1385	1293	}
1386	1294
1387	1295	attr.rwq_ind_tbl = ind_tbl;
1388		- }
1389		-
1390		- if (cmd_sz > sizeof(*cmd) &&
1391		- !ib_is_udata_cleared(ucore, sizeof(*cmd),
1392		- cmd_sz - sizeof(*cmd))) {
1393		- ret = -EOPNOTSUPP;
1394		- goto err_put;
1395	1296	}
1396	1297
1397	1298	if (ind_tbl && (cmd->max_recv_wr \|\| cmd->max_recv_sge \|\| cmd->is_srq)) {
..	..	@@ -1404,7 +1305,7 @@
1404	1305
1405	1306	if (cmd->qp_type == IB_QPT_XRC_TGT) {
1406	1307	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,
1407		- file);
	1308	+ attrs);
1408	1309
1409	1310	if (IS_ERR(xrcd_uobj)) {
1410	1311	ret = -EINVAL;
..	..	@@ -1424,7 +1325,7 @@
1424	1325	} else {
1425	1326	if (cmd->is_srq) {
1426	1327	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,
1427		- cmd->srq_handle, file);
	1328	+ cmd->srq_handle, attrs);
1428	1329	if (!srq \|\| srq->srq_type == IB_SRQT_XRC) {
1429	1330	ret = -EINVAL;
1430	1331	goto err_put;
..	..	@@ -1435,7 +1336,7 @@
1435	1336	if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1436	1337	rcq = uobj_get_obj_read(
1437	1338	cq, UVERBS_OBJECT_CQ,
1438		- cmd->recv_cq_handle, file);
	1339	+ cmd->recv_cq_handle, attrs);
1439	1340	if (!rcq) {
1440	1341	ret = -EINVAL;
1441	1342	goto err_put;
..	..	@@ -1446,11 +1347,11 @@
1446	1347
1447	1348	if (has_sq)
1448	1349	scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
1449		- cmd->send_cq_handle, file);
	1350	+ cmd->send_cq_handle, attrs);
1450	1351	if (!ind_tbl)
1451	1352	rcq = rcq ?: scq;
1452	1353	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,
1453		- file);
	1354	+ attrs);
1454	1355	if (!pd \|\| (!scq && has_sq)) {
1455	1356	ret = -EINVAL;
1456	1357	goto err_put;
..	..	@@ -1460,7 +1361,6 @@
1460	1361	}
1461	1362
1462	1363	attr.event_handler = ib_uverbs_qp_event_handler;
1463		- attr.qp_context = file;
1464	1364	attr.send_cq = scq;
1465	1365	attr.recv_cq = rcq;
1466	1366	attr.srq = srq;
..	..	@@ -1476,14 +1376,10 @@
1476	1376	attr.cap.max_recv_sge = cmd->max_recv_sge;
1477	1377	attr.cap.max_inline_data = cmd->max_inline_data;
1478	1378
1479		- obj->uevent.events_reported = 0;
1480	1379	INIT_LIST_HEAD(&obj->uevent.event_list);
1481	1380	INIT_LIST_HEAD(&obj->mcast_list);
1482	1381
1483		- if (cmd_sz >= offsetof(typeof(*cmd), create_flags) +
1484		- sizeof(cmd->create_flags))
1485		- attr.create_flags = cmd->create_flags;
1486		-
	1382	+ attr.create_flags = cmd->create_flags;
1487	1383	if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK \|
1488	1384	IB_QP_CREATE_CROSS_CHANNEL \|
1489	1385	IB_QP_CREATE_MANAGED_SEND \|
..	..	@@ -1505,19 +1401,11 @@
1505	1401	attr.source_qpn = cmd->source_qpn;
1506	1402	}
1507	1403
1508		- buf = (void )cmd + sizeof(cmd);
1509		- if (cmd_sz > sizeof(*cmd))
1510		- if (!(buf[0] == 0 && !memcmp(buf, buf + 1,
1511		- cmd_sz - sizeof(*cmd) - 1))) {
1512		- ret = -EINVAL;
1513		- goto err_put;
1514		- }
1515		-
1516	1404	if (cmd->qp_type == IB_QPT_XRC_TGT)
1517	1405	qp = ib_create_qp(pd, &attr);
1518	1406	else
1519		- qp = _ib_create_qp(device, pd, &attr, uhw,
1520		- &obj->uevent.uobject);
	1407	+ qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
	1408	+ obj);
1521	1409
1522	1410	if (IS_ERR(qp)) {
1523	1411	ret = PTR_ERR(qp);
..	..	@@ -1529,18 +1417,7 @@
1529	1417	if (ret)
1530	1418	goto err_cb;
1531	1419
1532		- qp->real_qp = qp;
1533		- qp->pd = pd;
1534		- qp->send_cq = attr.send_cq;
1535		- qp->recv_cq = attr.recv_cq;
1536		- qp->srq = attr.srq;
1537		- qp->rwq_ind_tbl = ind_tbl;
1538		- qp->event_handler = attr.event_handler;
1539		- qp->qp_context = attr.qp_context;
1540		- qp->qp_type = attr.qp_type;
1541		- atomic_set(&qp->usecnt, 0);
1542	1420	atomic_inc(&pd->usecnt);
1543		- qp->port = 0;
1544	1421	if (attr.send_cq)
1545	1422	atomic_inc(&attr.send_cq->usecnt);
1546	1423	if (attr.recv_cq)
..	..	@@ -1551,26 +1428,13 @@
1551	1428	atomic_inc(&ind_tbl->usecnt);
1552	1429	} else {
1553	1430	/* It is done in _ib_create_qp for other QP types */
1554		- qp->uobject = &obj->uevent.uobject;
	1431	+ qp->uobject = obj;
1555	1432	}
1556	1433
1557	1434	obj->uevent.uobject.object = qp;
1558		-
1559		- memset(&resp, 0, sizeof resp);
1560		- resp.base.qpn = qp->qp_num;
1561		- resp.base.qp_handle = obj->uevent.uobject.id;
1562		- resp.base.max_recv_sge = attr.cap.max_recv_sge;
1563		- resp.base.max_send_sge = attr.cap.max_send_sge;
1564		- resp.base.max_recv_wr = attr.cap.max_recv_wr;
1565		- resp.base.max_send_wr = attr.cap.max_send_wr;
1566		- resp.base.max_inline_data = attr.cap.max_inline_data;
1567		-
1568		- resp.response_length = offsetof(typeof(resp), response_length) +
1569		- sizeof(resp.response_length);
1570		-
1571		- ret = cb(file, &resp, ucore);
1572		- if (ret)
1573		- goto err_cb;
	1435	+ obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
	1436	+ if (obj->uevent.event_file)
	1437	+ uverbs_uobject_get(&obj->uevent.event_file->uobj);
1574	1438
1575	1439	if (xrcd) {
1576	1440	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
..	..	@@ -1582,17 +1446,30 @@
1582	1446	if (pd)
1583	1447	uobj_put_obj_read(pd);
1584	1448	if (scq)
1585		- uobj_put_obj_read(scq);
	1449	+ rdma_lookup_put_uobject(&scq->uobject->uevent.uobject,
	1450	+ UVERBS_LOOKUP_READ);
1586	1451	if (rcq && rcq != scq)
1587		- uobj_put_obj_read(rcq);
	1452	+ rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject,
	1453	+ UVERBS_LOOKUP_READ);
1588	1454	if (srq)
1589		- uobj_put_obj_read(srq);
	1455	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	1456	+ UVERBS_LOOKUP_READ);
1590	1457	if (ind_tbl)
1591	1458	uobj_put_obj_read(ind_tbl);
	1459	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1592	1460
1593		- return uobj_alloc_commit(&obj->uevent.uobject, 0);
	1461	+ resp.base.qpn = qp->qp_num;
	1462	+ resp.base.qp_handle = obj->uevent.uobject.id;
	1463	+ resp.base.max_recv_sge = attr.cap.max_recv_sge;
	1464	+ resp.base.max_send_sge = attr.cap.max_send_sge;
	1465	+ resp.base.max_recv_wr = attr.cap.max_recv_wr;
	1466	+ resp.base.max_send_wr = attr.cap.max_send_wr;
	1467	+ resp.base.max_inline_data = attr.cap.max_inline_data;
	1468	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	1469	+ return uverbs_response(attrs, &resp, sizeof(resp));
	1470	+
1594	1471	err_cb:
1595		- ib_destroy_qp(qp);
	1472	+ ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
1596	1473
1597	1474	err_put:
1598	1475	if (!IS_ERR(xrcd_uobj))
..	..	@@ -1600,51 +1477,30 @@
1600	1477	if (pd)
1601	1478	uobj_put_obj_read(pd);
1602	1479	if (scq)
1603		- uobj_put_obj_read(scq);
	1480	+ rdma_lookup_put_uobject(&scq->uobject->uevent.uobject,
	1481	+ UVERBS_LOOKUP_READ);
1604	1482	if (rcq && rcq != scq)
1605		- uobj_put_obj_read(rcq);
	1483	+ rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject,
	1484	+ UVERBS_LOOKUP_READ);
1606	1485	if (srq)
1607		- uobj_put_obj_read(srq);
	1486	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	1487	+ UVERBS_LOOKUP_READ);
1608	1488	if (ind_tbl)
1609	1489	uobj_put_obj_read(ind_tbl);
1610	1490
1611		- uobj_alloc_abort(&obj->uevent.uobject);
	1491	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
1612	1492	return ret;
1613	1493	}
1614	1494
1615		-static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file,
1616		- struct ib_uverbs_ex_create_qp_resp *resp,
1617		- struct ib_udata *ucore)
1618		-{
1619		- if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1620		- return -EFAULT;
1621		-
1622		- return 0;
1623		-}
1624		-
1625		-ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
1626		- const char __user *buf, int in_len,
1627		- int out_len)
	1495	+static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)
1628	1496	{
1629	1497	struct ib_uverbs_create_qp cmd;
1630	1498	struct ib_uverbs_ex_create_qp cmd_ex;
1631		- struct ib_udata ucore;
1632		- struct ib_udata uhw;
1633		- ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp);
1634		- int err;
	1499	+ int ret;
1635	1500
1636		- if (out_len < resp_size)
1637		- return -ENOSPC;
1638		-
1639		- if (copy_from_user(&cmd, buf, sizeof(cmd)))
1640		- return -EFAULT;
1641		-
1642		- ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response),
1643		- sizeof(cmd), resp_size);
1644		- ib_uverbs_init_udata(&uhw, buf + sizeof(cmd),
1645		- u64_to_user_ptr(cmd.response) + resp_size,
1646		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1647		- out_len - resp_size);
	1501	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1502	+ if (ret)
	1503	+ return ret;
1648	1504
1649	1505	memset(&cmd_ex, 0, sizeof(cmd_ex));
1650	1506	cmd_ex.user_handle = cmd.user_handle;
..	..	@@ -1661,42 +1517,17 @@
1661	1517	cmd_ex.qp_type = cmd.qp_type;
1662	1518	cmd_ex.is_srq = cmd.is_srq;
1663	1519
1664		- err = create_qp(file, &ucore, &uhw, &cmd_ex,
1665		- offsetof(typeof(cmd_ex), is_srq) +
1666		- sizeof(cmd.is_srq), ib_uverbs_create_qp_cb,
1667		- NULL);
1668		-
1669		- if (err)
1670		- return err;
1671		-
1672		- return in_len;
	1520	+ return create_qp(attrs, &cmd_ex);
1673	1521	}
1674	1522
1675		-static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file,
1676		- struct ib_uverbs_ex_create_qp_resp *resp,
1677		- struct ib_udata *ucore)
	1523	+static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)
1678	1524	{
1679		- if (ib_copy_to_udata(ucore, resp, resp->response_length))
1680		- return -EFAULT;
	1525	+ struct ib_uverbs_ex_create_qp cmd;
	1526	+ int ret;
1681	1527
1682		- return 0;
1683		-}
1684		-
1685		-int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file,
1686		- struct ib_udata *ucore,
1687		- struct ib_udata *uhw)
1688		-{
1689		- struct ib_uverbs_ex_create_qp_resp resp;
1690		- struct ib_uverbs_ex_create_qp cmd = {0};
1691		- int err;
1692		-
1693		- if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) +
1694		- sizeof(cmd.comp_mask)))
1695		- return -EINVAL;
1696		-
1697		- err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
1698		- if (err)
1699		- return err;
	1528	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1529	+ if (ret)
	1530	+ return ret;
1700	1531
1701	1532	if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1702	1533	return -EINVAL;
..	..	@@ -1704,51 +1535,31 @@
1704	1535	if (cmd.reserved)
1705	1536	return -EINVAL;
1706	1537
1707		- if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1708		- sizeof(resp.response_length)))
1709		- return -ENOSPC;
1710		-
1711		- err = create_qp(file, ucore, uhw, &cmd,
1712		- min(ucore->inlen, sizeof(cmd)),
1713		- ib_uverbs_ex_create_qp_cb, NULL);
1714		-
1715		- if (err)
1716		- return err;
1717		-
1718		- return 0;
	1538	+ return create_qp(attrs, &cmd);
1719	1539	}
1720	1540
1721		-ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
1722		- const char __user *buf, int in_len, int out_len)
	1541	+static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
1723	1542	{
	1543	+ struct ib_uverbs_create_qp_resp resp = {};
1724	1544	struct ib_uverbs_open_qp cmd;
1725		- struct ib_uverbs_create_qp_resp resp;
1726		- struct ib_udata udata;
1727	1545	struct ib_uqp_object *obj;
1728	1546	struct ib_xrcd *xrcd;
1729		- struct ib_uobject *uninitialized_var(xrcd_uobj);
1730	1547	struct ib_qp *qp;
1731		- struct ib_qp_open_attr attr;
	1548	+ struct ib_qp_open_attr attr = {};
1732	1549	int ret;
	1550	+ struct ib_uobject *xrcd_uobj;
1733	1551	struct ib_device *ib_dev;
1734	1552
1735		- if (out_len < sizeof resp)
1736		- return -ENOSPC;
	1553	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1554	+ if (ret)
	1555	+ return ret;
1737	1556
1738		- if (copy_from_user(&cmd, buf, sizeof cmd))
1739		- return -EFAULT;
1740		-
1741		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
1742		- u64_to_user_ptr(cmd.response) + sizeof(resp),
1743		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1744		- out_len - sizeof(resp));
1745		-
1746		- obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, file,
	1557	+ obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1747	1558	&ib_dev);
1748	1559	if (IS_ERR(obj))
1749	1560	return PTR_ERR(obj);
1750	1561
1751		- xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, file);
	1562	+ xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);
1752	1563	if (IS_ERR(xrcd_uobj)) {
1753	1564	ret = -EINVAL;
1754	1565	goto err_put;
..	..	@@ -1761,11 +1572,9 @@
1761	1572	}
1762	1573
1763	1574	attr.event_handler = ib_uverbs_qp_event_handler;
1764		- attr.qp_context = file;
1765	1575	attr.qp_num = cmd.qpn;
1766	1576	attr.qp_type = cmd.qp_type;
1767	1577
1768		- obj->uevent.events_reported = 0;
1769	1578	INIT_LIST_HEAD(&obj->uevent.event_list);
1770	1579	INIT_LIST_HEAD(&obj->mcast_list);
1771	1580
..	..	@@ -1778,28 +1587,20 @@
1778	1587	obj->uevent.uobject.object = qp;
1779	1588	obj->uevent.uobject.user_handle = cmd.user_handle;
1780	1589
1781		- memset(&resp, 0, sizeof resp);
1782		- resp.qpn = qp->qp_num;
1783		- resp.qp_handle = obj->uevent.uobject.id;
1784		-
1785		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
1786		- ret = -EFAULT;
1787		- goto err_destroy;
1788		- }
1789		-
1790	1590	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1791	1591	atomic_inc(&obj->uxrcd->refcnt);
1792		- qp->uobject = &obj->uevent.uobject;
	1592	+ qp->uobject = obj;
1793	1593	uobj_put_read(xrcd_uobj);
	1594	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
1794	1595
1795		- return uobj_alloc_commit(&obj->uevent.uobject, in_len);
	1596	+ resp.qpn = qp->qp_num;
	1597	+ resp.qp_handle = obj->uevent.uobject.id;
	1598	+ return uverbs_response(attrs, &resp, sizeof(resp));
1796	1599
1797		-err_destroy:
1798		- ib_destroy_qp(qp);
1799	1600	err_xrcd:
1800	1601	uobj_put_read(xrcd_uobj);
1801	1602	err_put:
1802		- uobj_alloc_abort(&obj->uevent.uobject);
	1603	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
1803	1604	return ret;
1804	1605	}
1805	1606
..	..	@@ -1825,9 +1626,7 @@
1825	1626	uverb_attr->port_num = rdma_ah_get_port_num(rdma_attr);
1826	1627	}
1827	1628
1828		-ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
1829		- const char __user *buf, int in_len,
1830		- int out_len)
	1629	+static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)
1831	1630	{
1832	1631	struct ib_uverbs_query_qp cmd;
1833	1632	struct ib_uverbs_query_qp_resp resp;
..	..	@@ -1836,8 +1635,9 @@
1836	1635	struct ib_qp_init_attr *init_attr;
1837	1636	int ret;
1838	1637
1839		- if (copy_from_user(&cmd, buf, sizeof cmd))
1840		- return -EFAULT;
	1638	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1639	+ if (ret)
	1640	+ return ret;
1841	1641
1842	1642	attr = kmalloc(sizeof *attr, GFP_KERNEL);
1843	1643	init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
..	..	@@ -1846,7 +1646,7 @@
1846	1646	goto out;
1847	1647	}
1848	1648
1849		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
	1649	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1850	1650	if (!qp) {
1851	1651	ret = -EINVAL;
1852	1652	goto out;
..	..	@@ -1854,7 +1654,8 @@
1854	1654
1855	1655	ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1856	1656
1857		- uobj_put_obj_read(qp);
	1657	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	1658	+ UVERBS_LOOKUP_READ);
1858	1659
1859	1660	if (ret)
1860	1661	goto out;
..	..	@@ -1893,14 +1694,13 @@
1893	1694	resp.max_inline_data = init_attr->cap.max_inline_data;
1894	1695	resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1895	1696
1896		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
1897		- ret = -EFAULT;
	1697	+ ret = uverbs_response(attrs, &resp, sizeof(resp));
1898	1698
1899	1699	out:
1900	1700	kfree(attr);
1901	1701	kfree(init_attr);
1902	1702
1903		- return ret ? ret : in_len;
	1703	+ return ret;
1904	1704	}
1905	1705
1906	1706	/* Remove ignored fields set in the attribute mask */
..	..	@@ -1940,8 +1740,8 @@
1940	1740	rdma_ah_set_make_grd(rdma_attr, false);
1941	1741	}
1942	1742
1943		-static int modify_qp(struct ib_uverbs_file *file,
1944		- struct ib_uverbs_ex_modify_qp cmd, struct ib_udata udata)
	1743	+static int modify_qp(struct uverbs_attr_bundle *attrs,
	1744	+ struct ib_uverbs_ex_modify_qp *cmd)
1945	1745	{
1946	1746	struct ib_qp_attr *attr;
1947	1747	struct ib_qp *qp;
..	..	@@ -1951,7 +1751,8 @@
1951	1751	if (!attr)
1952	1752	return -ENOMEM;
1953	1753
1954		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle, file);
	1754	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,
	1755	+ attrs);
1955	1756	if (!qp) {
1956	1757	ret = -EINVAL;
1957	1758	goto out;
..	..	@@ -2041,8 +1842,13 @@
2041	1842	attr->path_mtu = cmd->base.path_mtu;
2042	1843	if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE)
2043	1844	attr->path_mig_state = cmd->base.path_mig_state;
2044		- if (cmd->base.attr_mask & IB_QP_QKEY)
	1845	+ if (cmd->base.attr_mask & IB_QP_QKEY) {
	1846	+ if (cmd->base.qkey & IB_QP_SET_QKEY && !capable(CAP_NET_RAW)) {
	1847	+ ret = -EPERM;
	1848	+ goto release_qp;
	1849	+ }
2045	1850	attr->qkey = cmd->base.qkey;
	1851	+ }
2046	1852	if (cmd->base.attr_mask & IB_QP_RQ_PSN)
2047	1853	attr->rq_psn = cmd->base.rq_psn;
2048	1854	if (cmd->base.attr_mask & IB_QP_SQ_PSN)
..	..	@@ -2088,90 +1894,75 @@
2088	1894	ret = ib_modify_qp_with_udata(qp, attr,
2089	1895	modify_qp_mask(qp->qp_type,
2090	1896	cmd->base.attr_mask),
2091		- udata);
	1897	+ &attrs->driver_udata);
2092	1898
2093	1899	release_qp:
2094		- uobj_put_obj_read(qp);
	1900	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	1901	+ UVERBS_LOOKUP_READ);
2095	1902	out:
2096	1903	kfree(attr);
2097	1904
2098	1905	return ret;
2099	1906	}
2100	1907
2101		-ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
2102		- const char __user *buf, int in_len,
2103		- int out_len)
	1908	+static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)
2104	1909	{
2105		- struct ib_uverbs_ex_modify_qp cmd = {};
2106		- struct ib_udata udata;
	1910	+ struct ib_uverbs_ex_modify_qp cmd;
2107	1911	int ret;
2108	1912
2109		- if (copy_from_user(&cmd.base, buf, sizeof(cmd.base)))
2110		- return -EFAULT;
	1913	+ ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base));
	1914	+ if (ret)
	1915	+ return ret;
2111	1916
2112	1917	if (cmd.base.attr_mask &
2113	1918	~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
2114	1919	return -EOPNOTSUPP;
2115	1920
2116		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd.base), NULL,
2117		- in_len - sizeof(cmd.base) - sizeof(struct ib_uverbs_cmd_hdr),
2118		- out_len);
2119		-
2120		- ret = modify_qp(file, &cmd, &udata);
2121		- if (ret)
2122		- return ret;
2123		-
2124		- return in_len;
	1921	+ return modify_qp(attrs, &cmd);
2125	1922	}
2126	1923
2127		-int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file,
2128		- struct ib_udata *ucore,
2129		- struct ib_udata *uhw)
	1924	+static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)
2130	1925	{
2131		- struct ib_uverbs_ex_modify_qp cmd = {};
	1926	+ struct ib_uverbs_ex_modify_qp cmd;
	1927	+ struct ib_uverbs_ex_modify_qp_resp resp = {
	1928	+ .response_length = uverbs_response_length(attrs, sizeof(resp))
	1929	+ };
2132	1930	int ret;
	1931	+
	1932	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1933	+ if (ret)
	1934	+ return ret;
2133	1935
2134	1936	/*
2135	1937	* Last bit is reserved for extending the attr_mask by
2136	1938	* using another field.
2137	1939	*/
2138		- BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
2139		-
2140		- if (ucore->inlen < sizeof(cmd.base))
2141		- return -EINVAL;
2142		-
2143		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
2144		- if (ret)
2145		- return ret;
	1940	+ BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1ULL << 31));
2146	1941
2147	1942	if (cmd.base.attr_mask &
2148	1943	~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
2149	1944	return -EOPNOTSUPP;
2150	1945
2151		- if (ucore->inlen > sizeof(cmd)) {
2152		- if (!ib_is_udata_cleared(ucore, sizeof(cmd),
2153		- ucore->inlen - sizeof(cmd)))
2154		- return -EOPNOTSUPP;
2155		- }
	1946	+ ret = modify_qp(attrs, &cmd);
	1947	+ if (ret)
	1948	+ return ret;
2156	1949
2157		- ret = modify_qp(file, &cmd, uhw);
2158		-
2159		- return ret;
	1950	+ return uverbs_response(attrs, &resp, sizeof(resp));
2160	1951	}
2161	1952
2162		-ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
2163		- const char __user *buf, int in_len,
2164		- int out_len)
	1953	+static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)
2165	1954	{
2166	1955	struct ib_uverbs_destroy_qp cmd;
2167	1956	struct ib_uverbs_destroy_qp_resp resp;
2168	1957	struct ib_uobject *uobj;
2169	1958	struct ib_uqp_object *obj;
	1959	+ int ret;
2170	1960
2171		- if (copy_from_user(&cmd, buf, sizeof cmd))
2172		- return -EFAULT;
	1961	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	1962	+ if (ret)
	1963	+ return ret;
2173	1964
2174		- uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, file);
	1965	+ uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2175	1966	if (IS_ERR(uobj))
2176	1967	return PTR_ERR(uobj);
2177	1968
..	..	@@ -2181,10 +1972,7 @@
2181	1972
2182	1973	uobj_put_destroy(uobj);
2183	1974
2184		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2185		- return -EFAULT;
2186		-
2187		- return in_len;
	1975	+ return uverbs_response(attrs, &resp, sizeof(resp));
2188	1976	}
2189	1977
2190	1978	static void *alloc_wr(size_t wr_size, __u32 num_sge)
..	..	@@ -2197,9 +1985,7 @@
2197	1985	num_sge * sizeof (struct ib_sge), GFP_KERNEL);
2198	1986	}
2199	1987
2200		-ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
2201		- const char __user *buf, int in_len,
2202		- int out_len)
	1988	+static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)
2203	1989	{
2204	1990	struct ib_uverbs_post_send cmd;
2205	1991	struct ib_uverbs_post_send_resp resp;
..	..	@@ -2209,33 +1995,41 @@
2209	1995	struct ib_qp *qp;
2210	1996	int i, sg_ind;
2211	1997	int is_ud;
2212		- ssize_t ret = -EINVAL;
	1998	+ int ret, ret2;
2213	1999	size_t next_size;
	2000	+ const struct ib_sge __user *sgls;
	2001	+ const void __user *wqes;
	2002	+ struct uverbs_req_iter iter;
2214	2003
2215		- if (copy_from_user(&cmd, buf, sizeof cmd))
2216		- return -EFAULT;
2217		-
2218		- if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count +
2219		- cmd.sge_count * sizeof (struct ib_uverbs_sge))
2220		- return -EINVAL;
2221		-
2222		- if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr))
2223		- return -EINVAL;
	2004	+ ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
	2005	+ if (ret)
	2006	+ return ret;
	2007	+ wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count);
	2008	+ if (IS_ERR(wqes))
	2009	+ return PTR_ERR(wqes);
	2010	+ sgls = uverbs_request_next_ptr(
	2011	+ &iter, cmd.sge_count * sizeof(struct ib_uverbs_sge));
	2012	+ if (IS_ERR(sgls))
	2013	+ return PTR_ERR(sgls);
	2014	+ ret = uverbs_request_finish(&iter);
	2015	+ if (ret)
	2016	+ return ret;
2224	2017
2225	2018	user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2226	2019	if (!user_wr)
2227	2020	return -ENOMEM;
2228	2021
2229		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
2230		- if (!qp)
	2022	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
	2023	+ if (!qp) {
	2024	+ ret = -EINVAL;
2231	2025	goto out;
	2026	+ }
2232	2027
2233	2028	is_ud = qp->qp_type == IB_QPT_UD;
2234	2029	sg_ind = 0;
2235	2030	last = NULL;
2236	2031	for (i = 0; i < cmd.wr_count; ++i) {
2237		- if (copy_from_user(user_wr,
2238		- buf + sizeof cmd + i * cmd.wqe_size,
	2032	+ if (copy_from_user(user_wr, wqes + i * cmd.wqe_size,
2239	2033	cmd.wqe_size)) {
2240	2034	ret = -EFAULT;
2241	2035	goto out_put;
..	..	@@ -2263,7 +2057,7 @@
2263	2057	}
2264	2058
2265	2059	ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,
2266		- user_wr->wr.ud.ah, file);
	2060	+ user_wr->wr.ud.ah, attrs);
2267	2061	if (!ud->ah) {
2268	2062	kfree(ud);
2269	2063	ret = -EINVAL;
..	..	@@ -2343,11 +2137,9 @@
2343	2137	if (next->num_sge) {
2344	2138	next->sg_list = (void *) next +
2345	2139	ALIGN(next_size, sizeof(struct ib_sge));
2346		- if (copy_from_user(next->sg_list,
2347		- buf + sizeof cmd +
2348		- cmd.wr_count * cmd.wqe_size +
2349		- sg_ind * sizeof (struct ib_sge),
2350		- next->num_sge * sizeof (struct ib_sge))) {
	2140	+ if (copy_from_user(next->sg_list, sgls + sg_ind,
	2141	+ next->num_sge *
	2142	+ sizeof(struct ib_sge))) {
2351	2143	ret = -EFAULT;
2352	2144	goto out_put;
2353	2145	}
..	..	@@ -2357,7 +2149,7 @@
2357	2149	}
2358	2150
2359	2151	resp.bad_wr = 0;
2360		- ret = qp->device->post_send(qp->real_qp, wr, &bad_wr);
	2152	+ ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);
2361	2153	if (ret)
2362	2154	for (next = wr; next; next = next->next) {
2363	2155	++resp.bad_wr;
..	..	@@ -2365,11 +2157,13 @@
2365	2157	break;
2366	2158	}
2367	2159
2368		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2369		- ret = -EFAULT;
	2160	+ ret2 = uverbs_response(attrs, &resp, sizeof(resp));
	2161	+ if (ret2)
	2162	+ ret = ret2;
2370	2163
2371	2164	out_put:
2372		- uobj_put_obj_read(qp);
	2165	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	2166	+ UVERBS_LOOKUP_READ);
2373	2167
2374	2168	while (wr) {
2375	2169	if (is_ud && ud_wr(wr)->ah)
..	..	@@ -2382,27 +2176,34 @@
2382	2176	out:
2383	2177	kfree(user_wr);
2384	2178
2385		- return ret ? ret : in_len;
	2179	+ return ret;
2386	2180	}
2387	2181
2388		-static struct ib_recv_wr ib_uverbs_unmarshall_recv(const char __user buf,
2389		- int in_len,
2390		- u32 wr_count,
2391		- u32 sge_count,
2392		- u32 wqe_size)
	2182	+static struct ib_recv_wr *
	2183	+ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,
	2184	+ u32 wqe_size, u32 sge_count)
2393	2185	{
2394	2186	struct ib_uverbs_recv_wr *user_wr;
2395	2187	struct ib_recv_wr wr = NULL, last, *next;
2396	2188	int sg_ind;
2397	2189	int i;
2398	2190	int ret;
2399		-
2400		- if (in_len < wqe_size * wr_count +
2401		- sge_count * sizeof (struct ib_uverbs_sge))
2402		- return ERR_PTR(-EINVAL);
	2191	+ const struct ib_sge __user *sgls;
	2192	+ const void __user *wqes;
2403	2193
2404	2194	if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2405	2195	return ERR_PTR(-EINVAL);
	2196	+
	2197	+ wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count);
	2198	+ if (IS_ERR(wqes))
	2199	+ return ERR_CAST(wqes);
	2200	+ sgls = uverbs_request_next_ptr(
	2201	+ iter, sge_count * sizeof(struct ib_uverbs_sge));
	2202	+ if (IS_ERR(sgls))
	2203	+ return ERR_CAST(sgls);
	2204	+ ret = uverbs_request_finish(iter);
	2205	+ if (ret)
	2206	+ return ERR_PTR(ret);
2406	2207
2407	2208	user_wr = kmalloc(wqe_size, GFP_KERNEL);
2408	2209	if (!user_wr)
..	..	@@ -2411,7 +2212,7 @@
2411	2212	sg_ind = 0;
2412	2213	last = NULL;
2413	2214	for (i = 0; i < wr_count; ++i) {
2414		- if (copy_from_user(user_wr, buf + i * wqe_size,
	2215	+ if (copy_from_user(user_wr, wqes + i * wqe_size,
2415	2216	wqe_size)) {
2416	2217	ret = -EFAULT;
2417	2218	goto err;
..	..	@@ -2450,10 +2251,9 @@
2450	2251	if (next->num_sge) {
2451	2252	next->sg_list = (void *) next +
2452	2253	ALIGN(sizeof *next, sizeof (struct ib_sge));
2453		- if (copy_from_user(next->sg_list,
2454		- buf + wr_count * wqe_size +
2455		- sg_ind * sizeof (struct ib_sge),
2456		- next->num_sge * sizeof (struct ib_sge))) {
	2254	+ if (copy_from_user(next->sg_list, sgls + sg_ind,
	2255	+ next->num_sge *
	2256	+ sizeof(struct ib_sge))) {
2457	2257	ret = -EFAULT;
2458	2258	goto err;
2459	2259	}
..	..	@@ -2477,34 +2277,36 @@
2477	2277	return ERR_PTR(ret);
2478	2278	}
2479	2279
2480		-ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file,
2481		- const char __user *buf, int in_len,
2482		- int out_len)
	2280	+static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)
2483	2281	{
2484	2282	struct ib_uverbs_post_recv cmd;
2485	2283	struct ib_uverbs_post_recv_resp resp;
2486	2284	struct ib_recv_wr wr, next;
2487	2285	const struct ib_recv_wr *bad_wr;
2488	2286	struct ib_qp *qp;
2489		- ssize_t ret = -EINVAL;
	2287	+ int ret, ret2;
	2288	+ struct uverbs_req_iter iter;
2490	2289
2491		- if (copy_from_user(&cmd, buf, sizeof cmd))
2492		- return -EFAULT;
	2290	+ ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
	2291	+ if (ret)
	2292	+ return ret;
2493	2293
2494		- wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2495		- in_len - sizeof cmd, cmd.wr_count,
2496		- cmd.sge_count, cmd.wqe_size);
	2294	+ wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
	2295	+ cmd.sge_count);
2497	2296	if (IS_ERR(wr))
2498	2297	return PTR_ERR(wr);
2499	2298
2500		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
2501		- if (!qp)
	2299	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
	2300	+ if (!qp) {
	2301	+ ret = -EINVAL;
2502	2302	goto out;
	2303	+ }
2503	2304
2504	2305	resp.bad_wr = 0;
2505		- ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr);
	2306	+ ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);
2506	2307
2507		- uobj_put_obj_read(qp);
	2308	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	2309	+ UVERBS_LOOKUP_READ);
2508	2310	if (ret) {
2509	2311	for (next = wr; next; next = next->next) {
2510	2312	++resp.bad_wr;
..	..	@@ -2513,9 +2315,9 @@
2513	2315	}
2514	2316	}
2515	2317
2516		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2517		- ret = -EFAULT;
2518		-
	2318	+ ret2 = uverbs_response(attrs, &resp, sizeof(resp));
	2319	+ if (ret2)
	2320	+ ret = ret2;
2519	2321	out:
2520	2322	while (wr) {
2521	2323	next = wr->next;
..	..	@@ -2523,38 +2325,39 @@
2523	2325	wr = next;
2524	2326	}
2525	2327
2526		- return ret ? ret : in_len;
	2328	+ return ret;
2527	2329	}
2528	2330
2529		-ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file,
2530		- const char __user *buf, int in_len,
2531		- int out_len)
	2331	+static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)
2532	2332	{
2533	2333	struct ib_uverbs_post_srq_recv cmd;
2534	2334	struct ib_uverbs_post_srq_recv_resp resp;
2535	2335	struct ib_recv_wr wr, next;
2536	2336	const struct ib_recv_wr *bad_wr;
2537	2337	struct ib_srq *srq;
2538		- ssize_t ret = -EINVAL;
	2338	+ int ret, ret2;
	2339	+ struct uverbs_req_iter iter;
2539	2340
2540		- if (copy_from_user(&cmd, buf, sizeof cmd))
2541		- return -EFAULT;
	2341	+ ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
	2342	+ if (ret)
	2343	+ return ret;
2542	2344
2543		- wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2544		- in_len - sizeof cmd, cmd.wr_count,
2545		- cmd.sge_count, cmd.wqe_size);
	2345	+ wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size,
	2346	+ cmd.sge_count);
2546	2347	if (IS_ERR(wr))
2547	2348	return PTR_ERR(wr);
2548	2349
2549		- srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file);
2550		- if (!srq)
	2350	+ srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
	2351	+ if (!srq) {
	2352	+ ret = -EINVAL;
2551	2353	goto out;
	2354	+ }
2552	2355
2553	2356	resp.bad_wr = 0;
2554		- ret = srq->device->post_srq_recv ?
2555		- srq->device->post_srq_recv(srq, wr, &bad_wr) : -EOPNOTSUPP;
	2357	+ ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);
2556	2358
2557		- uobj_put_obj_read(srq);
	2359	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	2360	+ UVERBS_LOOKUP_READ);
2558	2361
2559	2362	if (ret)
2560	2363	for (next = wr; next; next = next->next) {
..	..	@@ -2563,8 +2366,9 @@
2563	2366	break;
2564	2367	}
2565	2368
2566		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
2567		- ret = -EFAULT;
	2369	+ ret2 = uverbs_response(attrs, &resp, sizeof(resp));
	2370	+ if (ret2)
	2371	+ ret = ret2;
2568	2372
2569	2373	out:
2570	2374	while (wr) {
..	..	@@ -2573,12 +2377,10 @@
2573	2377	wr = next;
2574	2378	}
2575	2379
2576		- return ret ? ret : in_len;
	2380	+ return ret;
2577	2381	}
2578	2382
2579		-ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
2580		- const char __user *buf, int in_len,
2581		- int out_len)
	2383	+static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
2582	2384	{
2583	2385	struct ib_uverbs_create_ah cmd;
2584	2386	struct ib_uverbs_create_ah_resp resp;
..	..	@@ -2587,21 +2389,13 @@
2587	2389	struct ib_ah *ah;
2588	2390	struct rdma_ah_attr attr = {};
2589	2391	int ret;
2590		- struct ib_udata udata;
2591	2392	struct ib_device *ib_dev;
2592	2393
2593		- if (out_len < sizeof resp)
2594		- return -ENOSPC;
	2394	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	2395	+ if (ret)
	2396	+ return ret;
2595	2397
2596		- if (copy_from_user(&cmd, buf, sizeof cmd))
2597		- return -EFAULT;
2598		-
2599		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
2600		- u64_to_user_ptr(cmd.response) + sizeof(resp),
2601		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
2602		- out_len - sizeof(resp));
2603		-
2604		- uobj = uobj_alloc(UVERBS_OBJECT_AH, file, &ib_dev);
	2398	+ uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);
2605	2399	if (IS_ERR(uobj))
2606	2400	return PTR_ERR(uobj);
2607	2401
..	..	@@ -2610,7 +2404,7 @@
2610	2404	goto err;
2611	2405	}
2612	2406
2613		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file);
	2407	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2614	2408	if (!pd) {
2615	2409	ret = -EINVAL;
2616	2410	goto err;
..	..	@@ -2634,7 +2428,7 @@
2634	2428	rdma_ah_set_ah_flags(&attr, 0);
2635	2429	}
2636	2430
2637		- ah = rdma_create_user_ah(pd, &attr, &udata);
	2431	+ ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata);
2638	2432	if (IS_ERR(ah)) {
2639	2433	ret = PTR_ERR(ah);
2640	2434	goto err_put;
..	..	@@ -2643,43 +2437,32 @@
2643	2437	ah->uobject = uobj;
2644	2438	uobj->user_handle = cmd.user_handle;
2645	2439	uobj->object = ah;
	2440	+ uobj_put_obj_read(pd);
	2441	+ uobj_finalize_uobj_create(uobj, attrs);
2646	2442
2647	2443	resp.ah_handle = uobj->id;
2648		-
2649		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
2650		- ret = -EFAULT;
2651		- goto err_copy;
2652		- }
2653		-
2654		- uobj_put_obj_read(pd);
2655		- return uobj_alloc_commit(uobj, in_len);
2656		-
2657		-err_copy:
2658		- rdma_destroy_ah(ah);
	2444	+ return uverbs_response(attrs, &resp, sizeof(resp));
2659	2445
2660	2446	err_put:
2661	2447	uobj_put_obj_read(pd);
2662		-
2663	2448	err:
2664		- uobj_alloc_abort(uobj);
	2449	+ uobj_alloc_abort(uobj, attrs);
2665	2450	return ret;
2666	2451	}
2667	2452
2668		-ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file,
2669		- const char __user *buf, int in_len, int out_len)
	2453	+static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)
2670	2454	{
2671	2455	struct ib_uverbs_destroy_ah cmd;
	2456	+ int ret;
2672	2457
2673		- if (copy_from_user(&cmd, buf, sizeof cmd))
2674		- return -EFAULT;
	2458	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	2459	+ if (ret)
	2460	+ return ret;
2675	2461
2676		- return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, file,
2677		- in_len);
	2462	+ return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);
2678	2463	}
2679	2464
2680		-ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file,
2681		- const char __user *buf, int in_len,
2682		- int out_len)
	2465	+static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)
2683	2466	{
2684	2467	struct ib_uverbs_attach_mcast cmd;
2685	2468	struct ib_qp *qp;
..	..	@@ -2687,14 +2470,15 @@
2687	2470	struct ib_uverbs_mcast_entry *mcast;
2688	2471	int ret;
2689	2472
2690		- if (copy_from_user(&cmd, buf, sizeof cmd))
2691		- return -EFAULT;
	2473	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	2474	+ if (ret)
	2475	+ return ret;
2692	2476
2693		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
	2477	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2694	2478	if (!qp)
2695	2479	return -EINVAL;
2696	2480
2697		- obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
	2481	+ obj = qp->uobject;
2698	2482
2699	2483	mutex_lock(&obj->mcast_lock);
2700	2484	list_for_each_entry(mcast, &obj->mcast_list, list)
..	..	@@ -2721,30 +2505,30 @@
2721	2505
2722	2506	out_put:
2723	2507	mutex_unlock(&obj->mcast_lock);
2724		- uobj_put_obj_read(qp);
	2508	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	2509	+ UVERBS_LOOKUP_READ);
2725	2510
2726		- return ret ? ret : in_len;
	2511	+ return ret;
2727	2512	}
2728	2513
2729		-ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file,
2730		- const char __user *buf, int in_len,
2731		- int out_len)
	2514	+static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
2732	2515	{
2733	2516	struct ib_uverbs_detach_mcast cmd;
2734	2517	struct ib_uqp_object *obj;
2735	2518	struct ib_qp *qp;
2736	2519	struct ib_uverbs_mcast_entry *mcast;
2737		- int ret = -EINVAL;
	2520	+ int ret;
2738	2521	bool found = false;
2739	2522
2740		- if (copy_from_user(&cmd, buf, sizeof cmd))
2741		- return -EFAULT;
	2523	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	2524	+ if (ret)
	2525	+ return ret;
2742	2526
2743		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
	2527	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2744	2528	if (!qp)
2745	2529	return -EINVAL;
2746	2530
2747		- obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
	2531	+ obj = qp->uobject;
2748	2532	mutex_lock(&obj->mcast_lock);
2749	2533
2750	2534	list_for_each_entry(mcast, &obj->mcast_list, list)
..	..	@@ -2765,20 +2549,12 @@
2765	2549
2766	2550	out_put:
2767	2551	mutex_unlock(&obj->mcast_lock);
2768		- uobj_put_obj_read(qp);
2769		- return ret ? ret : in_len;
	2552	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	2553	+ UVERBS_LOOKUP_READ);
	2554	+ return ret;
2770	2555	}
2771	2556
2772		-struct ib_uflow_resources {
2773		- size_t max;
2774		- size_t num;
2775		- size_t collection_num;
2776		- size_t counters_num;
2777		- struct ib_counters **counters;
2778		- struct ib_flow_action **collection;
2779		-};
2780		-
2781		-static struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
	2557	+struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
2782	2558	{
2783	2559	struct ib_uflow_resources *resources;
2784	2560
..	..	@@ -2808,6 +2584,7 @@
2808	2584
2809	2585	return NULL;
2810	2586	}
	2587	+EXPORT_SYMBOL(flow_resources_alloc);
2811	2588
2812	2589	void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
2813	2590	{
..	..	@@ -2826,10 +2603,11 @@
2826	2603	kfree(uflow_res->counters);
2827	2604	kfree(uflow_res);
2828	2605	}
	2606	+EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
2829	2607
2830		-static void flow_resources_add(struct ib_uflow_resources *uflow_res,
2831		- enum ib_flow_spec_type type,
2832		- void *ibobj)
	2608	+void flow_resources_add(struct ib_uflow_resources *uflow_res,
	2609	+ enum ib_flow_spec_type type,
	2610	+ void *ibobj)
2833	2611	{
2834	2612	WARN_ON(uflow_res->num >= uflow_res->max);
2835	2613
..	..	@@ -2850,8 +2628,9 @@
2850	2628
2851	2629	uflow_res->num++;
2852	2630	}
	2631	+EXPORT_SYMBOL(flow_resources_add);
2853	2632
2854		-static int kern_spec_to_ib_spec_action(struct ib_uverbs_file *ufile,
	2633	+static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,
2855	2634	struct ib_uverbs_flow_spec *kern_spec,
2856	2635	union ib_flow_spec *ib_spec,
2857	2636	struct ib_uflow_resources *uflow_res)
..	..	@@ -2880,7 +2659,7 @@
2880	2659	ib_spec->action.act = uobj_get_obj_read(flow_action,
2881	2660	UVERBS_OBJECT_FLOW_ACTION,
2882	2661	kern_spec->action.handle,
2883		- ufile);
	2662	+ attrs);
2884	2663	if (!ib_spec->action.act)
2885	2664	return -EINVAL;
2886	2665	ib_spec->action.size =
..	..	@@ -2898,7 +2677,7 @@
2898	2677	uobj_get_obj_read(counters,
2899	2678	UVERBS_OBJECT_COUNTERS,
2900	2679	kern_spec->flow_count.handle,
2901		- ufile);
	2680	+ attrs);
2902	2681	if (!ib_spec->flow_count.counters)
2903	2682	return -EINVAL;
2904	2683	ib_spec->flow_count.size =
..	..	@@ -3078,7 +2857,7 @@
3078	2857	kern_filter_sz, ib_spec);
3079	2858	}
3080	2859
3081		-static int kern_spec_to_ib_spec(struct ib_uverbs_file *ufile,
	2860	+static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,
3082	2861	struct ib_uverbs_flow_spec *kern_spec,
3083	2862	union ib_flow_spec *ib_spec,
3084	2863	struct ib_uflow_resources *uflow_res)
..	..	@@ -3087,17 +2866,15 @@
3087	2866	return -EINVAL;
3088	2867
3089	2868	if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
3090		- return kern_spec_to_ib_spec_action(ufile, kern_spec, ib_spec,
	2869	+ return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,
3091	2870	uflow_res);
3092	2871	else
3093	2872	return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
3094	2873	}
3095	2874
3096		-int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file,
3097		- struct ib_udata *ucore,
3098		- struct ib_udata *uhw)
	2875	+static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
3099	2876	{
3100		- struct ib_uverbs_ex_create_wq cmd = {};
	2877	+ struct ib_uverbs_ex_create_wq cmd;
3101	2878	struct ib_uverbs_ex_create_wq_resp resp = {};
3102	2879	struct ib_uwq_object *obj;
3103	2880	int err = 0;
..	..	@@ -3105,43 +2882,27 @@
3105	2882	struct ib_pd *pd;
3106	2883	struct ib_wq *wq;
3107	2884	struct ib_wq_init_attr wq_init_attr = {};
3108		- size_t required_cmd_sz;
3109		- size_t required_resp_len;
3110	2885	struct ib_device *ib_dev;
3111	2886
3112		- required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge);
3113		- required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn);
3114		-
3115		- if (ucore->inlen < required_cmd_sz)
3116		- return -EINVAL;
3117		-
3118		- if (ucore->outlen < required_resp_len)
3119		- return -ENOSPC;
3120		-
3121		- if (ucore->inlen > sizeof(cmd) &&
3122		- !ib_is_udata_cleared(ucore, sizeof(cmd),
3123		- ucore->inlen - sizeof(cmd)))
3124		- return -EOPNOTSUPP;
3125		-
3126		- err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	2887	+ err = uverbs_request(attrs, &cmd, sizeof(cmd));
3127	2888	if (err)
3128	2889	return err;
3129	2890
3130	2891	if (cmd.comp_mask)
3131	2892	return -EOPNOTSUPP;
3132	2893
3133		- obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, file,
	2894	+ obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,
3134	2895	&ib_dev);
3135	2896	if (IS_ERR(obj))
3136	2897	return PTR_ERR(obj);
3137	2898
3138		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file);
	2899	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
3139	2900	if (!pd) {
3140	2901	err = -EINVAL;
3141	2902	goto err_uobj;
3142	2903	}
3143	2904
3144		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	2905	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
3145	2906	if (!cq) {
3146	2907	err = -EINVAL;
3147	2908	goto err_put_pd;
..	..	@@ -3150,100 +2911,71 @@
3150	2911	wq_init_attr.cq = cq;
3151	2912	wq_init_attr.max_sge = cmd.max_sge;
3152	2913	wq_init_attr.max_wr = cmd.max_wr;
3153		- wq_init_attr.wq_context = file;
3154	2914	wq_init_attr.wq_type = cmd.wq_type;
3155	2915	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
3156		- if (ucore->inlen >= (offsetof(typeof(cmd), create_flags) +
3157		- sizeof(cmd.create_flags)))
3158		- wq_init_attr.create_flags = cmd.create_flags;
3159		- obj->uevent.events_reported = 0;
	2916	+ wq_init_attr.create_flags = cmd.create_flags;
3160	2917	INIT_LIST_HEAD(&obj->uevent.event_list);
	2918	+ obj->uevent.uobject.user_handle = cmd.user_handle;
3161	2919
3162		- if (!pd->device->create_wq) {
3163		- err = -EOPNOTSUPP;
3164		- goto err_put_cq;
3165		- }
3166		- wq = pd->device->create_wq(pd, &wq_init_attr, uhw);
	2920	+ wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
3167	2921	if (IS_ERR(wq)) {
3168	2922	err = PTR_ERR(wq);
3169	2923	goto err_put_cq;
3170	2924	}
3171	2925
3172		- wq->uobject = &obj->uevent.uobject;
	2926	+ wq->uobject = obj;
3173	2927	obj->uevent.uobject.object = wq;
3174	2928	wq->wq_type = wq_init_attr.wq_type;
3175	2929	wq->cq = cq;
3176	2930	wq->pd = pd;
3177	2931	wq->device = pd->device;
3178		- wq->wq_context = wq_init_attr.wq_context;
3179	2932	atomic_set(&wq->usecnt, 0);
3180	2933	atomic_inc(&pd->usecnt);
3181	2934	atomic_inc(&cq->usecnt);
3182		- wq->uobject = &obj->uevent.uobject;
3183		- obj->uevent.uobject.object = wq;
	2935	+ obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
	2936	+ if (obj->uevent.event_file)
	2937	+ uverbs_uobject_get(&obj->uevent.event_file->uobj);
3184	2938
3185		- memset(&resp, 0, sizeof(resp));
	2939	+ uobj_put_obj_read(pd);
	2940	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	2941	+ UVERBS_LOOKUP_READ);
	2942	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
	2943	+
3186	2944	resp.wq_handle = obj->uevent.uobject.id;
3187	2945	resp.max_sge = wq_init_attr.max_sge;
3188	2946	resp.max_wr = wq_init_attr.max_wr;
3189	2947	resp.wqn = wq->wq_num;
3190		- resp.response_length = required_resp_len;
3191		- err = ib_copy_to_udata(ucore,
3192		- &resp, resp.response_length);
3193		- if (err)
3194		- goto err_copy;
	2948	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	2949	+ return uverbs_response(attrs, &resp, sizeof(resp));
3195	2950
3196		- uobj_put_obj_read(pd);
3197		- uobj_put_obj_read(cq);
3198		- return uobj_alloc_commit(&obj->uevent.uobject, 0);
3199		-
3200		-err_copy:
3201		- ib_destroy_wq(wq);
3202	2951	err_put_cq:
3203		- uobj_put_obj_read(cq);
	2952	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	2953	+ UVERBS_LOOKUP_READ);
3204	2954	err_put_pd:
3205	2955	uobj_put_obj_read(pd);
3206	2956	err_uobj:
3207		- uobj_alloc_abort(&obj->uevent.uobject);
	2957	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
3208	2958
3209	2959	return err;
3210	2960	}
3211	2961
3212		-int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file,
3213		- struct ib_udata *ucore,
3214		- struct ib_udata *uhw)
	2962	+static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)
3215	2963	{
3216		- struct ib_uverbs_ex_destroy_wq cmd = {};
	2964	+ struct ib_uverbs_ex_destroy_wq cmd;
3217	2965	struct ib_uverbs_ex_destroy_wq_resp resp = {};
3218	2966	struct ib_uobject *uobj;
3219	2967	struct ib_uwq_object *obj;
3220		- size_t required_cmd_sz;
3221		- size_t required_resp_len;
3222	2968	int ret;
3223	2969
3224		- required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle);
3225		- required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
3226		-
3227		- if (ucore->inlen < required_cmd_sz)
3228		- return -EINVAL;
3229		-
3230		- if (ucore->outlen < required_resp_len)
3231		- return -ENOSPC;
3232		-
3233		- if (ucore->inlen > sizeof(cmd) &&
3234		- !ib_is_udata_cleared(ucore, sizeof(cmd),
3235		- ucore->inlen - sizeof(cmd)))
3236		- return -EOPNOTSUPP;
3237		-
3238		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	2970	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3239	2971	if (ret)
3240	2972	return ret;
3241	2973
3242	2974	if (cmd.comp_mask)
3243	2975	return -EOPNOTSUPP;
3244	2976
3245		- resp.response_length = required_resp_len;
3246		- uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, file);
	2977	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	2978	+ uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3247	2979	if (IS_ERR(uobj))
3248	2980	return PTR_ERR(uobj);
3249	2981
..	..	@@ -3252,29 +2984,17 @@
3252	2984
3253	2985	uobj_put_destroy(uobj);
3254	2986
3255		- return ib_copy_to_udata(ucore, &resp, resp.response_length);
	2987	+ return uverbs_response(attrs, &resp, sizeof(resp));
3256	2988	}
3257	2989
3258		-int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file,
3259		- struct ib_udata *ucore,
3260		- struct ib_udata *uhw)
	2990	+static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)
3261	2991	{
3262		- struct ib_uverbs_ex_modify_wq cmd = {};
	2992	+ struct ib_uverbs_ex_modify_wq cmd;
3263	2993	struct ib_wq *wq;
3264	2994	struct ib_wq_attr wq_attr = {};
3265		- size_t required_cmd_sz;
3266	2995	int ret;
3267	2996
3268		- required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state);
3269		- if (ucore->inlen < required_cmd_sz)
3270		- return -EINVAL;
3271		-
3272		- if (ucore->inlen > sizeof(cmd) &&
3273		- !ib_is_udata_cleared(ucore, sizeof(cmd),
3274		- ucore->inlen - sizeof(cmd)))
3275		- return -EOPNOTSUPP;
3276		-
3277		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	2997	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3278	2998	if (ret)
3279	2999	return ret;
3280	3000
..	..	@@ -3284,61 +3004,59 @@
3284	3004	if (cmd.attr_mask > (IB_WQ_STATE \| IB_WQ_CUR_STATE \| IB_WQ_FLAGS))
3285	3005	return -EINVAL;
3286	3006
3287		- wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, file);
	3007	+ wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3288	3008	if (!wq)
3289	3009	return -EINVAL;
3290	3010
3291		- wq_attr.curr_wq_state = cmd.curr_wq_state;
3292		- wq_attr.wq_state = cmd.wq_state;
3293	3011	if (cmd.attr_mask & IB_WQ_FLAGS) {
3294	3012	wq_attr.flags = cmd.flags;
3295	3013	wq_attr.flags_mask = cmd.flags_mask;
3296	3014	}
3297		- if (!wq->device->modify_wq) {
3298		- ret = -EOPNOTSUPP;
3299		- goto out;
	3015	+
	3016	+ if (cmd.attr_mask & IB_WQ_CUR_STATE) {
	3017	+ if (cmd.curr_wq_state > IB_WQS_ERR)
	3018	+ return -EINVAL;
	3019	+
	3020	+ wq_attr.curr_wq_state = cmd.curr_wq_state;
	3021	+ } else {
	3022	+ wq_attr.curr_wq_state = wq->state;
3300	3023	}
3301		- ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw);
3302		-out:
3303		- uobj_put_obj_read(wq);
	3024	+
	3025	+ if (cmd.attr_mask & IB_WQ_STATE) {
	3026	+ if (cmd.wq_state > IB_WQS_ERR)
	3027	+ return -EINVAL;
	3028	+
	3029	+ wq_attr.wq_state = cmd.wq_state;
	3030	+ } else {
	3031	+ wq_attr.wq_state = wq_attr.curr_wq_state;
	3032	+ }
	3033	+
	3034	+ ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,
	3035	+ &attrs->driver_udata);
	3036	+ rdma_lookup_put_uobject(&wq->uobject->uevent.uobject,
	3037	+ UVERBS_LOOKUP_READ);
3304	3038	return ret;
3305	3039	}
3306	3040
3307		-int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file,
3308		- struct ib_udata *ucore,
3309		- struct ib_udata *uhw)
	3041	+static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3310	3042	{
3311		- struct ib_uverbs_ex_create_rwq_ind_table cmd = {};
	3043	+ struct ib_uverbs_ex_create_rwq_ind_table cmd;
3312	3044	struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {};
3313		- struct ib_uobject *uobj;
3314		- int err = 0;
	3045	+ struct ib_uobject *uobj;
	3046	+ int err;
3315	3047	struct ib_rwq_ind_table_init_attr init_attr = {};
3316	3048	struct ib_rwq_ind_table *rwq_ind_tbl;
3317		- struct ib_wq **wqs = NULL;
	3049	+ struct ib_wq **wqs = NULL;
3318	3050	u32 *wqs_handles = NULL;
3319	3051	struct ib_wq *wq = NULL;
3320		- int i, j, num_read_wqs;
	3052	+ int i, num_read_wqs;
3321	3053	u32 num_wq_handles;
3322		- u32 expected_in_size;
3323		- size_t required_cmd_sz_header;
3324		- size_t required_resp_len;
	3054	+ struct uverbs_req_iter iter;
3325	3055	struct ib_device *ib_dev;
3326	3056
3327		- required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size);
3328		- required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num);
3329		-
3330		- if (ucore->inlen < required_cmd_sz_header)
3331		- return -EINVAL;
3332		-
3333		- if (ucore->outlen < required_resp_len)
3334		- return -ENOSPC;
3335		-
3336		- err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header);
	3057	+ err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3337	3058	if (err)
3338	3059	return err;
3339		-
3340		- ucore->inbuf += required_cmd_sz_header;
3341		- ucore->inlen -= required_cmd_sz_header;
3342	3060
3343	3061	if (cmd.comp_mask)
3344	3062	return -EOPNOTSUPP;
..	..	@@ -3347,26 +3065,17 @@
3347	3065	return -EINVAL;
3348	3066
3349	3067	num_wq_handles = 1 << cmd.log_ind_tbl_size;
3350		- expected_in_size = num_wq_handles * sizeof(__u32);
3351		- if (num_wq_handles == 1)
3352		- /* input size for wq handles is u64 aligned */
3353		- expected_in_size += sizeof(__u32);
3354		-
3355		- if (ucore->inlen < expected_in_size)
3356		- return -EINVAL;
3357		-
3358		- if (ucore->inlen > expected_in_size &&
3359		- !ib_is_udata_cleared(ucore, expected_in_size,
3360		- ucore->inlen - expected_in_size))
3361		- return -EOPNOTSUPP;
3362		-
3363	3068	wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3364	3069	GFP_KERNEL);
3365	3070	if (!wqs_handles)
3366	3071	return -ENOMEM;
3367	3072
3368		- err = ib_copy_from_udata(wqs_handles, ucore,
3369		- num_wq_handles * sizeof(__u32));
	3073	+ err = uverbs_request_next(&iter, wqs_handles,
	3074	+ num_wq_handles * sizeof(__u32));
	3075	+ if (err)
	3076	+ goto err_free;
	3077	+
	3078	+ err = uverbs_request_finish(&iter);
3370	3079	if (err)
3371	3080	goto err_free;
3372	3081
..	..	@@ -3379,34 +3088,30 @@
3379	3088	for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3380	3089	num_read_wqs++) {
3381	3090	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,
3382		- wqs_handles[num_read_wqs], file);
	3091	+ wqs_handles[num_read_wqs], attrs);
3383	3092	if (!wq) {
3384	3093	err = -EINVAL;
3385	3094	goto put_wqs;
3386	3095	}
3387	3096
3388	3097	wqs[num_read_wqs] = wq;
	3098	+ atomic_inc(&wqs[num_read_wqs]->usecnt);
3389	3099	}
3390	3100
3391		- uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, file, &ib_dev);
	3101	+ uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
3392	3102	if (IS_ERR(uobj)) {
3393	3103	err = PTR_ERR(uobj);
3394	3104	goto put_wqs;
3395	3105	}
3396	3106
	3107	+ rwq_ind_tbl = rdma_zalloc_drv_obj(ib_dev, ib_rwq_ind_table);
	3108	+ if (!rwq_ind_tbl) {
	3109	+ err = -ENOMEM;
	3110	+ goto err_uobj;
	3111	+ }
	3112	+
3397	3113	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3398	3114	init_attr.ind_tbl = wqs;
3399		-
3400		- if (!ib_dev->create_rwq_ind_table) {
3401		- err = -EOPNOTSUPP;
3402		- goto err_uobj;
3403		- }
3404		- rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw);
3405		-
3406		- if (IS_ERR(rwq_ind_tbl)) {
3407		- err = PTR_ERR(rwq_ind_tbl);
3408		- goto err_uobj;
3409		- }
3410	3115
3411	3116	rwq_ind_tbl->ind_tbl = wqs;
3412	3117	rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
..	..	@@ -3415,57 +3120,44 @@
3415	3120	rwq_ind_tbl->device = ib_dev;
3416	3121	atomic_set(&rwq_ind_tbl->usecnt, 0);
3417	3122
	3123	+ err = ib_dev->ops.create_rwq_ind_table(rwq_ind_tbl, &init_attr,
	3124	+ &attrs->driver_udata);
	3125	+ if (err)
	3126	+ goto err_create;
	3127	+
3418	3128	for (i = 0; i < num_wq_handles; i++)
3419		- atomic_inc(&wqs[i]->usecnt);
	3129	+ rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
	3130	+ UVERBS_LOOKUP_READ);
	3131	+ kfree(wqs_handles);
	3132	+ uobj_finalize_uobj_create(uobj, attrs);
3420	3133
3421	3134	resp.ind_tbl_handle = uobj->id;
3422	3135	resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3423		- resp.response_length = required_resp_len;
	3136	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	3137	+ return uverbs_response(attrs, &resp, sizeof(resp));
3424	3138
3425		- err = ib_copy_to_udata(ucore,
3426		- &resp, resp.response_length);
3427		- if (err)
3428		- goto err_copy;
3429		-
3430		- kfree(wqs_handles);
3431		-
3432		- for (j = 0; j < num_read_wqs; j++)
3433		- uobj_put_obj_read(wqs[j]);
3434		-
3435		- return uobj_alloc_commit(uobj, 0);
3436		-
3437		-err_copy:
3438		- ib_destroy_rwq_ind_table(rwq_ind_tbl);
	3139	+err_create:
	3140	+ kfree(rwq_ind_tbl);
3439	3141	err_uobj:
3440		- uobj_alloc_abort(uobj);
	3142	+ uobj_alloc_abort(uobj, attrs);
3441	3143	put_wqs:
3442		- for (j = 0; j < num_read_wqs; j++)
3443		- uobj_put_obj_read(wqs[j]);
	3144	+ for (i = 0; i < num_read_wqs; i++) {
	3145	+ rdma_lookup_put_uobject(&wqs[i]->uobject->uevent.uobject,
	3146	+ UVERBS_LOOKUP_READ);
	3147	+ atomic_dec(&wqs[i]->usecnt);
	3148	+ }
3444	3149	err_free:
3445	3150	kfree(wqs_handles);
3446	3151	kfree(wqs);
3447	3152	return err;
3448	3153	}
3449	3154
3450		-int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file,
3451		- struct ib_udata *ucore,
3452		- struct ib_udata *uhw)
	3155	+static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3453	3156	{
3454		- struct ib_uverbs_ex_destroy_rwq_ind_table cmd = {};
3455		- int ret;
3456		- size_t required_cmd_sz;
	3157	+ struct ib_uverbs_ex_destroy_rwq_ind_table cmd;
	3158	+ int ret;
3457	3159
3458		- required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle);
3459		-
3460		- if (ucore->inlen < required_cmd_sz)
3461		- return -EINVAL;
3462		-
3463		- if (ucore->inlen > sizeof(cmd) &&
3464		- !ib_is_udata_cleared(ucore, sizeof(cmd),
3465		- ucore->inlen - sizeof(cmd)))
3466		- return -EOPNOTSUPP;
3467		-
3468		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	3160	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3469	3161	if (ret)
3470	3162	return ret;
3471	3163
..	..	@@ -3473,40 +3165,29 @@
3473	3165	return -EOPNOTSUPP;
3474	3166
3475	3167	return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,
3476		- cmd.ind_tbl_handle, file, 0);
	3168	+ cmd.ind_tbl_handle, attrs);
3477	3169	}
3478	3170
3479		-int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
3480		- struct ib_udata *ucore,
3481		- struct ib_udata *uhw)
	3171	+static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
3482	3172	{
3483	3173	struct ib_uverbs_create_flow cmd;
3484		- struct ib_uverbs_create_flow_resp resp;
	3174	+ struct ib_uverbs_create_flow_resp resp = {};
3485	3175	struct ib_uobject *uobj;
3486		- struct ib_uflow_object *uflow;
3487	3176	struct ib_flow *flow_id;
3488	3177	struct ib_uverbs_flow_attr *kern_flow_attr;
3489	3178	struct ib_flow_attr *flow_attr;
3490	3179	struct ib_qp *qp;
3491	3180	struct ib_uflow_resources *uflow_res;
3492	3181	struct ib_uverbs_flow_spec_hdr *kern_spec;
3493		- int err = 0;
	3182	+ struct uverbs_req_iter iter;
	3183	+ int err;
3494	3184	void *ib_spec;
3495	3185	int i;
3496	3186	struct ib_device *ib_dev;
3497	3187
3498		- if (ucore->inlen < sizeof(cmd))
3499		- return -EINVAL;
3500		-
3501		- if (ucore->outlen < sizeof(resp))
3502		- return -ENOSPC;
3503		-
3504		- err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
	3188	+ err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd));
3505	3189	if (err)
3506	3190	return err;
3507		-
3508		- ucore->inbuf += sizeof(cmd);
3509		- ucore->inlen -= sizeof(cmd);
3510	3191
3511	3192	if (cmd.comp_mask)
3512	3193	return -EINVAL;
..	..	@@ -3525,8 +3206,7 @@
3525	3206	if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3526	3207	return -EINVAL;
3527	3208
3528		- if (cmd.flow_attr.size > ucore->inlen \|\|
3529		- cmd.flow_attr.size >
	3209	+ if (cmd.flow_attr.size >
3530	3210	(cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3531	3211	return -EINVAL;
3532	3212
..	..	@@ -3541,21 +3221,25 @@
3541	3221	return -ENOMEM;
3542	3222
3543	3223	*kern_flow_attr = cmd.flow_attr;
3544		- err = ib_copy_from_udata(&kern_flow_attr->flow_specs, ucore,
3545		- cmd.flow_attr.size);
	3224	+ err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs,
	3225	+ cmd.flow_attr.size);
3546	3226	if (err)
3547	3227	goto err_free_attr;
3548	3228	} else {
3549	3229	kern_flow_attr = &cmd.flow_attr;
3550	3230	}
3551	3231
3552		- uobj = uobj_alloc(UVERBS_OBJECT_FLOW, file, &ib_dev);
	3232	+ err = uverbs_request_finish(&iter);
	3233	+ if (err)
	3234	+ goto err_free_attr;
	3235	+
	3236	+ uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);
3553	3237	if (IS_ERR(uobj)) {
3554	3238	err = PTR_ERR(uobj);
3555	3239	goto err_free_attr;
3556	3240	}
3557	3241
3558		- qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file);
	3242	+ qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
3559	3243	if (!qp) {
3560	3244	err = -EINVAL;
3561	3245	goto err_uobj;
..	..	@@ -3563,11 +3247,6 @@
3563	3247
3564	3248	if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3565	3249	err = -EINVAL;
3566		- goto err_put;
3567		- }
3568		-
3569		- if (!qp->device->create_flow) {
3570		- err = -EOPNOTSUPP;
3571	3250	goto err_put;
3572	3251	}
3573	3252
..	..	@@ -3597,7 +3276,7 @@
3597	3276	cmd.flow_attr.size >= kern_spec->size;
3598	3277	i++) {
3599	3278	err = kern_spec_to_ib_spec(
3600		- file, (struct ib_uverbs_flow_spec *)kern_spec,
	3279	+ attrs, (struct ib_uverbs_flow_spec *)kern_spec,
3601	3280	ib_spec, uflow_res);
3602	3281	if (err)
3603	3282	goto err_free;
..	..	@@ -3615,86 +3294,71 @@
3615	3294	goto err_free;
3616	3295	}
3617	3296
3618		- flow_id = qp->device->create_flow(qp, flow_attr,
3619		- IB_FLOW_DOMAIN_USER, uhw);
	3297	+ flow_id = qp->device->ops.create_flow(qp, flow_attr,
	3298	+ &attrs->driver_udata);
3620	3299
3621	3300	if (IS_ERR(flow_id)) {
3622	3301	err = PTR_ERR(flow_id);
3623	3302	goto err_free;
3624	3303	}
3625		- atomic_inc(&qp->usecnt);
3626		- flow_id->qp = qp;
3627		- flow_id->device = qp->device;
3628		- flow_id->uobject = uobj;
3629		- uobj->object = flow_id;
3630		- uflow = container_of(uobj, typeof(*uflow), uobject);
3631		- uflow->resources = uflow_res;
3632	3304
3633		- memset(&resp, 0, sizeof(resp));
3634		- resp.flow_handle = uobj->id;
	3305	+ ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res);
3635	3306
3636		- err = ib_copy_to_udata(ucore,
3637		- &resp, sizeof(resp));
3638		- if (err)
3639		- goto err_copy;
3640		-
3641		- uobj_put_obj_read(qp);
	3307	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	3308	+ UVERBS_LOOKUP_READ);
3642	3309	kfree(flow_attr);
	3310	+
3643	3311	if (cmd.flow_attr.num_of_specs)
3644	3312	kfree(kern_flow_attr);
3645		- return uobj_alloc_commit(uobj, 0);
3646		-err_copy:
3647		- if (!qp->device->destroy_flow(flow_id))
3648		- atomic_dec(&qp->usecnt);
	3313	+ uobj_finalize_uobj_create(uobj, attrs);
	3314	+
	3315	+ resp.flow_handle = uobj->id;
	3316	+ return uverbs_response(attrs, &resp, sizeof(resp));
	3317	+
3649	3318	err_free:
3650	3319	ib_uverbs_flow_resources_free(uflow_res);
3651	3320	err_free_flow_attr:
3652	3321	kfree(flow_attr);
3653	3322	err_put:
3654		- uobj_put_obj_read(qp);
	3323	+ rdma_lookup_put_uobject(&qp->uobject->uevent.uobject,
	3324	+ UVERBS_LOOKUP_READ);
3655	3325	err_uobj:
3656		- uobj_alloc_abort(uobj);
	3326	+ uobj_alloc_abort(uobj, attrs);
3657	3327	err_free_attr:
3658	3328	if (cmd.flow_attr.num_of_specs)
3659	3329	kfree(kern_flow_attr);
3660	3330	return err;
3661	3331	}
3662	3332
3663		-int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file,
3664		- struct ib_udata *ucore,
3665		- struct ib_udata *uhw)
	3333	+static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)
3666	3334	{
3667	3335	struct ib_uverbs_destroy_flow cmd;
3668	3336	int ret;
3669	3337
3670		- if (ucore->inlen < sizeof(cmd))
3671		- return -EINVAL;
3672		-
3673		- ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
	3338	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3674	3339	if (ret)
3675	3340	return ret;
3676	3341
3677	3342	if (cmd.comp_mask)
3678	3343	return -EINVAL;
3679	3344
3680		- return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, file,
3681		- 0);
	3345	+ return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);
3682	3346	}
3683	3347
3684		-static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
	3348	+static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
3685	3349	struct ib_uverbs_create_xsrq *cmd,
3686	3350	struct ib_udata *udata)
3687	3351	{
3688		- struct ib_uverbs_create_srq_resp resp;
	3352	+ struct ib_uverbs_create_srq_resp resp = {};
3689	3353	struct ib_usrq_object *obj;
3690	3354	struct ib_pd *pd;
3691	3355	struct ib_srq *srq;
3692		- struct ib_uobject *uninitialized_var(xrcd_uobj);
3693	3356	struct ib_srq_init_attr attr;
3694	3357	int ret;
	3358	+ struct ib_uobject *xrcd_uobj;
3695	3359	struct ib_device *ib_dev;
3696	3360
3697		- obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, file,
	3361	+ obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
3698	3362	&ib_dev);
3699	3363	if (IS_ERR(obj))
3700	3364	return PTR_ERR(obj);
..	..	@@ -3704,7 +3368,7 @@
3704	3368
3705	3369	if (cmd->srq_type == IB_SRQT_XRC) {
3706	3370	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,
3707		- file);
	3371	+ attrs);
3708	3372	if (IS_ERR(xrcd_uobj)) {
3709	3373	ret = -EINVAL;
3710	3374	goto err;
..	..	@@ -3722,89 +3386,64 @@
3722	3386
3723	3387	if (ib_srq_has_cq(cmd->srq_type)) {
3724	3388	attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
3725		- cmd->cq_handle, file);
	3389	+ cmd->cq_handle, attrs);
3726	3390	if (!attr.ext.cq) {
3727	3391	ret = -EINVAL;
3728	3392	goto err_put_xrcd;
3729	3393	}
3730	3394	}
3731	3395
3732		- pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, file);
	3396	+ pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);
3733	3397	if (!pd) {
3734	3398	ret = -EINVAL;
3735	3399	goto err_put_cq;
3736	3400	}
3737	3401
3738	3402	attr.event_handler = ib_uverbs_srq_event_handler;
3739		- attr.srq_context = file;
3740	3403	attr.srq_type = cmd->srq_type;
3741	3404	attr.attr.max_wr = cmd->max_wr;
3742	3405	attr.attr.max_sge = cmd->max_sge;
3743	3406	attr.attr.srq_limit = cmd->srq_limit;
3744	3407
3745		- obj->uevent.events_reported = 0;
3746	3408	INIT_LIST_HEAD(&obj->uevent.event_list);
	3409	+ obj->uevent.uobject.user_handle = cmd->user_handle;
3747	3410
3748		- srq = pd->device->create_srq(pd, &attr, udata);
	3411	+ srq = ib_create_srq_user(pd, &attr, obj, udata);
3749	3412	if (IS_ERR(srq)) {
3750	3413	ret = PTR_ERR(srq);
3751		- goto err_put;
	3414	+ goto err_put_pd;
3752	3415	}
3753		-
3754		- srq->device = pd->device;
3755		- srq->pd = pd;
3756		- srq->srq_type = cmd->srq_type;
3757		- srq->uobject = &obj->uevent.uobject;
3758		- srq->event_handler = attr.event_handler;
3759		- srq->srq_context = attr.srq_context;
3760		-
3761		- if (ib_srq_has_cq(cmd->srq_type)) {
3762		- srq->ext.cq = attr.ext.cq;
3763		- atomic_inc(&attr.ext.cq->usecnt);
3764		- }
3765		-
3766		- if (cmd->srq_type == IB_SRQT_XRC) {
3767		- srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
3768		- atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3769		- }
3770		-
3771		- atomic_inc(&pd->usecnt);
3772		- atomic_set(&srq->usecnt, 0);
3773	3416
3774	3417	obj->uevent.uobject.object = srq;
3775	3418	obj->uevent.uobject.user_handle = cmd->user_handle;
	3419	+ obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
	3420	+ if (obj->uevent.event_file)
	3421	+ uverbs_uobject_get(&obj->uevent.event_file->uobj);
3776	3422
3777		- memset(&resp, 0, sizeof resp);
3778		- resp.srq_handle = obj->uevent.uobject.id;
3779		- resp.max_wr = attr.attr.max_wr;
3780		- resp.max_sge = attr.attr.max_sge;
3781	3423	if (cmd->srq_type == IB_SRQT_XRC)
3782	3424	resp.srqn = srq->ext.xrc.srq_num;
3783		-
3784		- if (copy_to_user(u64_to_user_ptr(cmd->response),
3785		- &resp, sizeof resp)) {
3786		- ret = -EFAULT;
3787		- goto err_copy;
3788		- }
3789	3425
3790	3426	if (cmd->srq_type == IB_SRQT_XRC)
3791	3427	uobj_put_read(xrcd_uobj);
3792	3428
3793	3429	if (ib_srq_has_cq(cmd->srq_type))
3794		- uobj_put_obj_read(attr.ext.cq);
	3430	+ rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,
	3431	+ UVERBS_LOOKUP_READ);
3795	3432
3796	3433	uobj_put_obj_read(pd);
3797		- return uobj_alloc_commit(&obj->uevent.uobject, 0);
	3434	+ uobj_finalize_uobj_create(&obj->uevent.uobject, attrs);
3798	3435
3799		-err_copy:
3800		- ib_destroy_srq(srq);
	3436	+ resp.srq_handle = obj->uevent.uobject.id;
	3437	+ resp.max_wr = attr.attr.max_wr;
	3438	+ resp.max_sge = attr.attr.max_sge;
	3439	+ return uverbs_response(attrs, &resp, sizeof(resp));
3801	3440
3802		-err_put:
	3441	+err_put_pd:
3803	3442	uobj_put_obj_read(pd);
3804		-
3805	3443	err_put_cq:
3806	3444	if (ib_srq_has_cq(cmd->srq_type))
3807		- uobj_put_obj_read(attr.ext.cq);
	3445	+ rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,
	3446	+ UVERBS_LOOKUP_READ);
3808	3447
3809	3448	err_put_xrcd:
3810	3449	if (cmd->srq_type == IB_SRQT_XRC) {
..	..	@@ -3813,25 +3452,19 @@
3813	3452	}
3814	3453
3815	3454	err:
3816		- uobj_alloc_abort(&obj->uevent.uobject);
	3455	+ uobj_alloc_abort(&obj->uevent.uobject, attrs);
3817	3456	return ret;
3818	3457	}
3819	3458
3820		-ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file,
3821		- const char __user *buf, int in_len,
3822		- int out_len)
	3459	+static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)
3823	3460	{
3824	3461	struct ib_uverbs_create_srq cmd;
3825	3462	struct ib_uverbs_create_xsrq xcmd;
3826		- struct ib_uverbs_create_srq_resp resp;
3827		- struct ib_udata udata;
3828	3463	int ret;
3829	3464
3830		- if (out_len < sizeof resp)
3831		- return -ENOSPC;
3832		-
3833		- if (copy_from_user(&cmd, buf, sizeof cmd))
3834		- return -EFAULT;
	3465	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	3466	+ if (ret)
	3467	+ return ret;
3835	3468
3836	3469	memset(&xcmd, 0, sizeof(xcmd));
3837	3470	xcmd.response = cmd.response;
..	..	@@ -3842,77 +3475,49 @@
3842	3475	xcmd.max_sge = cmd.max_sge;
3843	3476	xcmd.srq_limit = cmd.srq_limit;
3844	3477
3845		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
3846		- u64_to_user_ptr(cmd.response) + sizeof(resp),
3847		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
3848		- out_len - sizeof(resp));
3849		-
3850		- ret = __uverbs_create_xsrq(file, &xcmd, &udata);
3851		- if (ret)
3852		- return ret;
3853		-
3854		- return in_len;
	3478	+ return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata);
3855	3479	}
3856	3480
3857		-ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file,
3858		- const char __user *buf, int in_len, int out_len)
	3481	+static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)
3859	3482	{
3860	3483	struct ib_uverbs_create_xsrq cmd;
3861		- struct ib_uverbs_create_srq_resp resp;
3862		- struct ib_udata udata;
3863	3484	int ret;
3864	3485
3865		- if (out_len < sizeof resp)
3866		- return -ENOSPC;
3867		-
3868		- if (copy_from_user(&cmd, buf, sizeof cmd))
3869		- return -EFAULT;
3870		-
3871		- ib_uverbs_init_udata(&udata, buf + sizeof(cmd),
3872		- u64_to_user_ptr(cmd.response) + sizeof(resp),
3873		- in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
3874		- out_len - sizeof(resp));
3875		-
3876		- ret = __uverbs_create_xsrq(file, &cmd, &udata);
	3486	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
3877	3487	if (ret)
3878	3488	return ret;
3879	3489
3880		- return in_len;
	3490	+ return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata);
3881	3491	}
3882	3492
3883		-ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
3884		- const char __user *buf, int in_len,
3885		- int out_len)
	3493	+static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)
3886	3494	{
3887	3495	struct ib_uverbs_modify_srq cmd;
3888		- struct ib_udata udata;
3889	3496	struct ib_srq *srq;
3890	3497	struct ib_srq_attr attr;
3891	3498	int ret;
3892	3499
3893		- if (copy_from_user(&cmd, buf, sizeof cmd))
3894		- return -EFAULT;
	3500	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	3501	+ if (ret)
	3502	+ return ret;
3895	3503
3896		- ib_uverbs_init_udata(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
3897		- out_len);
3898		-
3899		- srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file);
	3504	+ srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3900	3505	if (!srq)
3901	3506	return -EINVAL;
3902	3507
3903	3508	attr.max_wr = cmd.max_wr;
3904	3509	attr.srq_limit = cmd.srq_limit;
3905	3510
3906		- ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata);
	3511	+ ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,
	3512	+ &attrs->driver_udata);
3907	3513
3908		- uobj_put_obj_read(srq);
	3514	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	3515	+ UVERBS_LOOKUP_READ);
3909	3516
3910		- return ret ? ret : in_len;
	3517	+ return ret;
3911	3518	}
3912	3519
3913		-ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file,
3914		- const char __user *buf,
3915		- int in_len, int out_len)
	3520	+static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)
3916	3521	{
3917	3522	struct ib_uverbs_query_srq cmd;
3918	3523	struct ib_uverbs_query_srq_resp resp;
..	..	@@ -3920,19 +3525,18 @@
3920	3525	struct ib_srq *srq;
3921	3526	int ret;
3922	3527
3923		- if (out_len < sizeof resp)
3924		- return -ENOSPC;
	3528	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	3529	+ if (ret)
	3530	+ return ret;
3925	3531
3926		- if (copy_from_user(&cmd, buf, sizeof cmd))
3927		- return -EFAULT;
3928		-
3929		- srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file);
	3532	+ srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3930	3533	if (!srq)
3931	3534	return -EINVAL;
3932	3535
3933	3536	ret = ib_query_srq(srq, &attr);
3934	3537
3935		- uobj_put_obj_read(srq);
	3538	+ rdma_lookup_put_uobject(&srq->uobject->uevent.uobject,
	3539	+ UVERBS_LOOKUP_READ);
3936	3540
3937	3541	if (ret)
3938	3542	return ret;
..	..	@@ -3943,25 +3547,22 @@
3943	3547	resp.max_sge = attr.max_sge;
3944	3548	resp.srq_limit = attr.srq_limit;
3945	3549
3946		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp))
3947		- return -EFAULT;
3948		-
3949		- return in_len;
	3550	+ return uverbs_response(attrs, &resp, sizeof(resp));
3950	3551	}
3951	3552
3952		-ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3953		- const char __user *buf, int in_len,
3954		- int out_len)
	3553	+static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)
3955	3554	{
3956	3555	struct ib_uverbs_destroy_srq cmd;
3957	3556	struct ib_uverbs_destroy_srq_resp resp;
3958	3557	struct ib_uobject *uobj;
3959	3558	struct ib_uevent_object *obj;
	3559	+ int ret;
3960	3560
3961		- if (copy_from_user(&cmd, buf, sizeof cmd))
3962		- return -EFAULT;
	3561	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
	3562	+ if (ret)
	3563	+ return ret;
3963	3564
3964		- uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, file);
	3565	+ uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3965	3566	if (IS_ERR(uobj))
3966	3567	return PTR_ERR(uobj);
3967	3568
..	..	@@ -3971,35 +3572,24 @@
3971	3572
3972	3573	uobj_put_destroy(uobj);
3973	3574
3974		- if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp)))
3975		- return -EFAULT;
3976		-
3977		- return in_len;
	3575	+ return uverbs_response(attrs, &resp, sizeof(resp));
3978	3576	}
3979	3577
3980		-int ib_uverbs_ex_query_device(struct ib_uverbs_file *file,
3981		- struct ib_udata *ucore,
3982		- struct ib_udata *uhw)
	3578	+static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
3983	3579	{
3984		- struct ib_uverbs_ex_query_device_resp resp = { {0} };
	3580	+ struct ib_uverbs_ex_query_device_resp resp = {};
3985	3581	struct ib_uverbs_ex_query_device cmd;
3986	3582	struct ib_device_attr attr = {0};
3987	3583	struct ib_ucontext *ucontext;
3988	3584	struct ib_device *ib_dev;
3989	3585	int err;
3990	3586
3991		- ucontext = ib_uverbs_get_ucontext(file);
	3587	+ ucontext = ib_uverbs_get_ucontext(attrs);
3992	3588	if (IS_ERR(ucontext))
3993	3589	return PTR_ERR(ucontext);
3994	3590	ib_dev = ucontext->device;
3995	3591
3996		- if (!ib_dev->query_device)
3997		- return -EOPNOTSUPP;
3998		-
3999		- if (ucore->inlen < sizeof(cmd))
4000		- return -EINVAL;
4001		-
4002		- err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
	3592	+ err = uverbs_request(attrs, &cmd, sizeof(cmd));
4003	3593	if (err)
4004	3594	return err;
4005	3595
..	..	@@ -4009,21 +3599,12 @@
4009	3599	if (cmd.reserved)
4010	3600	return -EINVAL;
4011	3601
4012		- resp.response_length = offsetof(typeof(resp), odp_caps);
4013		-
4014		- if (ucore->outlen < resp.response_length)
4015		- return -ENOSPC;
4016		-
4017		- err = ib_dev->query_device(ib_dev, &attr, uhw);
	3602	+ err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);
4018	3603	if (err)
4019	3604	return err;
4020	3605
4021	3606	copy_query_dev_fields(ucontext, &resp.base, &attr);
4022	3607
4023		- if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps))
4024		- goto end;
4025		-
4026		-#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
4027	3608	resp.odp_caps.general_caps = attr.odp_caps.general_caps;
4028	3609	resp.odp_caps.per_transport_caps.rc_odp_caps =
4029	3610	attr.odp_caps.per_transport_caps.rc_odp_caps;
..	..	@@ -4031,100 +3612,40 @@
4031	3612	attr.odp_caps.per_transport_caps.uc_odp_caps;
4032	3613	resp.odp_caps.per_transport_caps.ud_odp_caps =
4033	3614	attr.odp_caps.per_transport_caps.ud_odp_caps;
4034		-#endif
4035		- resp.response_length += sizeof(resp.odp_caps);
4036		-
4037		- if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask))
4038		- goto end;
	3615	+ resp.xrc_odp_caps = attr.odp_caps.per_transport_caps.xrc_odp_caps;
4039	3616
4040	3617	resp.timestamp_mask = attr.timestamp_mask;
4041		- resp.response_length += sizeof(resp.timestamp_mask);
4042		-
4043		- if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock))
4044		- goto end;
4045		-
4046	3618	resp.hca_core_clock = attr.hca_core_clock;
4047		- resp.response_length += sizeof(resp.hca_core_clock);
4048		-
4049		- if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex))
4050		- goto end;
4051		-
4052	3619	resp.device_cap_flags_ex = attr.device_cap_flags;
4053		- resp.response_length += sizeof(resp.device_cap_flags_ex);
4054		-
4055		- if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps))
4056		- goto end;
4057		-
4058	3620	resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
4059	3621	resp.rss_caps.max_rwq_indirection_tables =
4060	3622	attr.rss_caps.max_rwq_indirection_tables;
4061	3623	resp.rss_caps.max_rwq_indirection_table_size =
4062	3624	attr.rss_caps.max_rwq_indirection_table_size;
4063		-
4064		- resp.response_length += sizeof(resp.rss_caps);
4065		-
4066		- if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq))
4067		- goto end;
4068		-
4069	3625	resp.max_wq_type_rq = attr.max_wq_type_rq;
4070		- resp.response_length += sizeof(resp.max_wq_type_rq);
4071		-
4072		- if (ucore->outlen < resp.response_length + sizeof(resp.raw_packet_caps))
4073		- goto end;
4074		-
4075	3626	resp.raw_packet_caps = attr.raw_packet_caps;
4076		- resp.response_length += sizeof(resp.raw_packet_caps);
4077		-
4078		- if (ucore->outlen < resp.response_length + sizeof(resp.tm_caps))
4079		- goto end;
4080		-
4081	3627	resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size;
4082	3628	resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags;
4083	3629	resp.tm_caps.max_ops = attr.tm_caps.max_ops;
4084	3630	resp.tm_caps.max_sge = attr.tm_caps.max_sge;
4085	3631	resp.tm_caps.flags = attr.tm_caps.flags;
4086		- resp.response_length += sizeof(resp.tm_caps);
4087		-
4088		- if (ucore->outlen < resp.response_length + sizeof(resp.cq_moderation_caps))
4089		- goto end;
4090		-
4091	3632	resp.cq_moderation_caps.max_cq_moderation_count =
4092	3633	attr.cq_caps.max_cq_moderation_count;
4093	3634	resp.cq_moderation_caps.max_cq_moderation_period =
4094	3635	attr.cq_caps.max_cq_moderation_period;
4095		- resp.response_length += sizeof(resp.cq_moderation_caps);
4096		-
4097		- if (ucore->outlen < resp.response_length + sizeof(resp.max_dm_size))
4098		- goto end;
4099		-
4100	3636	resp.max_dm_size = attr.max_dm_size;
4101		- resp.response_length += sizeof(resp.max_dm_size);
4102		-end:
4103		- err = ib_copy_to_udata(ucore, &resp, resp.response_length);
4104		- return err;
	3637	+ resp.response_length = uverbs_response_length(attrs, sizeof(resp));
	3638	+
	3639	+ return uverbs_response(attrs, &resp, sizeof(resp));
4105	3640	}
4106	3641
4107		-int ib_uverbs_ex_modify_cq(struct ib_uverbs_file *file,
4108		- struct ib_udata *ucore,
4109		- struct ib_udata *uhw)
	3642	+static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
4110	3643	{
4111		- struct ib_uverbs_ex_modify_cq cmd = {};
	3644	+ struct ib_uverbs_ex_modify_cq cmd;
4112	3645	struct ib_cq *cq;
4113		- size_t required_cmd_sz;
4114	3646	int ret;
4115	3647
4116		- required_cmd_sz = offsetof(typeof(cmd), reserved) +
4117		- sizeof(cmd.reserved);
4118		- if (ucore->inlen < required_cmd_sz)
4119		- return -EINVAL;
4120		-
4121		- /* sanity checks */
4122		- if (ucore->inlen > sizeof(cmd) &&
4123		- !ib_is_udata_cleared(ucore, sizeof(cmd),
4124		- ucore->inlen - sizeof(cmd)))
4125		- return -EOPNOTSUPP;
4126		-
4127		- ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
	3648	+ ret = uverbs_request(attrs, &cmd, sizeof(cmd));
4128	3649	if (ret)
4129	3650	return ret;
4130	3651
..	..	@@ -4134,13 +3655,388 @@
4134	3655	if (cmd.attr_mask > IB_CQ_MODERATE)
4135	3656	return -EOPNOTSUPP;
4136	3657
4137		- cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file);
	3658	+ cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
4138	3659	if (!cq)
4139	3660	return -EINVAL;
4140	3661
4141	3662	ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period);
4142	3663
4143		- uobj_put_obj_read(cq);
4144		-
	3664	+ rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
	3665	+ UVERBS_LOOKUP_READ);
4145	3666	return ret;
4146	3667	}
	3668	+
	3669	+/*
	3670	+ * Describe the input structs for write(). Some write methods have an input
	3671	+ * only struct, most have an input and output. If the struct has an output then
	3672	+ * the 'response' u64 must be the first field in the request structure.
	3673	+ *
	3674	+ * If udata is present then both the request and response structs have a
	3675	+ * trailing driver_data flex array. In this case the size of the base struct
	3676	+ * cannot be changed.
	3677	+ */
	3678	+#define UAPI_DEF_WRITE_IO(req, resp) \
	3679	+ .write.has_resp = 1 + \
	3680	+ BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \
	3681	+ BUILD_BUG_ON_ZERO(sizeof_field(req, response) != \
	3682	+ sizeof(u64)), \
	3683	+ .write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
	3684	+
	3685	+#define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
	3686	+
	3687	+#define UAPI_DEF_WRITE_UDATA_IO(req, resp) \
	3688	+ UAPI_DEF_WRITE_IO(req, resp), \
	3689	+ .write.has_udata = \
	3690	+ 1 + \
	3691	+ BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
	3692	+ sizeof(req)) + \
	3693	+ BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) != \
	3694	+ sizeof(resp))
	3695	+
	3696	+#define UAPI_DEF_WRITE_UDATA_I(req) \
	3697	+ UAPI_DEF_WRITE_I(req), \
	3698	+ .write.has_udata = \
	3699	+ 1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
	3700	+ sizeof(req))
	3701	+
	3702	+/*
	3703	+ * The _EX versions are for use with WRITE_EX and allow the last struct member
	3704	+ * to be specified. Buffers that do not include that member will be rejected.
	3705	+ */
	3706	+#define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \
	3707	+ .write.has_resp = 1, \
	3708	+ .write.req_size = offsetofend(req, req_last_member), \
	3709	+ .write.resp_size = offsetofend(resp, resp_last_member)
	3710	+
	3711	+#define UAPI_DEF_WRITE_I_EX(req, req_last_member) \
	3712	+ .write.req_size = offsetofend(req, req_last_member)
	3713	+
	3714	+const struct uapi_definition uverbs_def_write_intf[] = {
	3715	+ DECLARE_UVERBS_OBJECT(
	3716	+ UVERBS_OBJECT_AH,
	3717	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,
	3718	+ ib_uverbs_create_ah,
	3719	+ UAPI_DEF_WRITE_UDATA_IO(
	3720	+ struct ib_uverbs_create_ah,
	3721	+ struct ib_uverbs_create_ah_resp),
	3722	+ UAPI_DEF_METHOD_NEEDS_FN(create_ah)),
	3723	+ DECLARE_UVERBS_WRITE(
	3724	+ IB_USER_VERBS_CMD_DESTROY_AH,
	3725	+ ib_uverbs_destroy_ah,
	3726	+ UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah),
	3727	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))),
	3728	+
	3729	+ DECLARE_UVERBS_OBJECT(
	3730	+ UVERBS_OBJECT_COMP_CHANNEL,
	3731	+ DECLARE_UVERBS_WRITE(
	3732	+ IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
	3733	+ ib_uverbs_create_comp_channel,
	3734	+ UAPI_DEF_WRITE_IO(
	3735	+ struct ib_uverbs_create_comp_channel,
	3736	+ struct ib_uverbs_create_comp_channel_resp))),
	3737	+
	3738	+ DECLARE_UVERBS_OBJECT(
	3739	+ UVERBS_OBJECT_CQ,
	3740	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,
	3741	+ ib_uverbs_create_cq,
	3742	+ UAPI_DEF_WRITE_UDATA_IO(
	3743	+ struct ib_uverbs_create_cq,
	3744	+ struct ib_uverbs_create_cq_resp),
	3745	+ UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
	3746	+ DECLARE_UVERBS_WRITE(
	3747	+ IB_USER_VERBS_CMD_DESTROY_CQ,
	3748	+ ib_uverbs_destroy_cq,
	3749	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,
	3750	+ struct ib_uverbs_destroy_cq_resp),
	3751	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),
	3752	+ DECLARE_UVERBS_WRITE(
	3753	+ IB_USER_VERBS_CMD_POLL_CQ,
	3754	+ ib_uverbs_poll_cq,
	3755	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,
	3756	+ struct ib_uverbs_poll_cq_resp),
	3757	+ UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),
	3758	+ DECLARE_UVERBS_WRITE(
	3759	+ IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,
	3760	+ ib_uverbs_req_notify_cq,
	3761	+ UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),
	3762	+ UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),
	3763	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,
	3764	+ ib_uverbs_resize_cq,
	3765	+ UAPI_DEF_WRITE_UDATA_IO(
	3766	+ struct ib_uverbs_resize_cq,
	3767	+ struct ib_uverbs_resize_cq_resp),
	3768	+ UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),
	3769	+ DECLARE_UVERBS_WRITE_EX(
	3770	+ IB_USER_VERBS_EX_CMD_CREATE_CQ,
	3771	+ ib_uverbs_ex_create_cq,
	3772	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,
	3773	+ reserved,
	3774	+ struct ib_uverbs_ex_create_cq_resp,
	3775	+ response_length),
	3776	+ UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
	3777	+ DECLARE_UVERBS_WRITE_EX(
	3778	+ IB_USER_VERBS_EX_CMD_MODIFY_CQ,
	3779	+ ib_uverbs_ex_modify_cq,
	3780	+ UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
	3781	+ UAPI_DEF_METHOD_NEEDS_FN(modify_cq))),
	3782	+
	3783	+ DECLARE_UVERBS_OBJECT(
	3784	+ UVERBS_OBJECT_DEVICE,
	3785	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,
	3786	+ ib_uverbs_get_context,
	3787	+ UAPI_DEF_WRITE_UDATA_IO(
	3788	+ struct ib_uverbs_get_context,
	3789	+ struct ib_uverbs_get_context_resp)),
	3790	+ DECLARE_UVERBS_WRITE(
	3791	+ IB_USER_VERBS_CMD_QUERY_DEVICE,
	3792	+ ib_uverbs_query_device,
	3793	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,
	3794	+ struct ib_uverbs_query_device_resp)),
	3795	+ DECLARE_UVERBS_WRITE(
	3796	+ IB_USER_VERBS_CMD_QUERY_PORT,
	3797	+ ib_uverbs_query_port,
	3798	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,
	3799	+ struct ib_uverbs_query_port_resp),
	3800	+ UAPI_DEF_METHOD_NEEDS_FN(query_port)),
	3801	+ DECLARE_UVERBS_WRITE_EX(
	3802	+ IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
	3803	+ ib_uverbs_ex_query_device,
	3804	+ UAPI_DEF_WRITE_IO_EX(
	3805	+ struct ib_uverbs_ex_query_device,
	3806	+ reserved,
	3807	+ struct ib_uverbs_ex_query_device_resp,
	3808	+ response_length),
	3809	+ UAPI_DEF_METHOD_NEEDS_FN(query_device)),
	3810	+ UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),
	3811	+ UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),
	3812	+
	3813	+ DECLARE_UVERBS_OBJECT(
	3814	+ UVERBS_OBJECT_FLOW,
	3815	+ DECLARE_UVERBS_WRITE_EX(
	3816	+ IB_USER_VERBS_EX_CMD_CREATE_FLOW,
	3817	+ ib_uverbs_ex_create_flow,
	3818	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,
	3819	+ flow_attr,
	3820	+ struct ib_uverbs_create_flow_resp,
	3821	+ flow_handle),
	3822	+ UAPI_DEF_METHOD_NEEDS_FN(create_flow)),
	3823	+ DECLARE_UVERBS_WRITE_EX(
	3824	+ IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
	3825	+ ib_uverbs_ex_destroy_flow,
	3826	+ UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),
	3827	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),
	3828	+
	3829	+ DECLARE_UVERBS_OBJECT(
	3830	+ UVERBS_OBJECT_MR,
	3831	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,
	3832	+ ib_uverbs_dereg_mr,
	3833	+ UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),
	3834	+ UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),
	3835	+ DECLARE_UVERBS_WRITE(
	3836	+ IB_USER_VERBS_CMD_REG_MR,
	3837	+ ib_uverbs_reg_mr,
	3838	+ UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,
	3839	+ struct ib_uverbs_reg_mr_resp),
	3840	+ UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),
	3841	+ DECLARE_UVERBS_WRITE(
	3842	+ IB_USER_VERBS_CMD_REREG_MR,
	3843	+ ib_uverbs_rereg_mr,
	3844	+ UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,
	3845	+ struct ib_uverbs_rereg_mr_resp),
	3846	+ UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),
	3847	+
	3848	+ DECLARE_UVERBS_OBJECT(
	3849	+ UVERBS_OBJECT_MW,
	3850	+ DECLARE_UVERBS_WRITE(
	3851	+ IB_USER_VERBS_CMD_ALLOC_MW,
	3852	+ ib_uverbs_alloc_mw,
	3853	+ UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,
	3854	+ struct ib_uverbs_alloc_mw_resp),
	3855	+ UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),
	3856	+ DECLARE_UVERBS_WRITE(
	3857	+ IB_USER_VERBS_CMD_DEALLOC_MW,
	3858	+ ib_uverbs_dealloc_mw,
	3859	+ UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),
	3860	+ UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),
	3861	+
	3862	+ DECLARE_UVERBS_OBJECT(
	3863	+ UVERBS_OBJECT_PD,
	3864	+ DECLARE_UVERBS_WRITE(
	3865	+ IB_USER_VERBS_CMD_ALLOC_PD,
	3866	+ ib_uverbs_alloc_pd,
	3867	+ UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,
	3868	+ struct ib_uverbs_alloc_pd_resp),
	3869	+ UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),
	3870	+ DECLARE_UVERBS_WRITE(
	3871	+ IB_USER_VERBS_CMD_DEALLOC_PD,
	3872	+ ib_uverbs_dealloc_pd,
	3873	+ UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),
	3874	+ UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),
	3875	+
	3876	+ DECLARE_UVERBS_OBJECT(
	3877	+ UVERBS_OBJECT_QP,
	3878	+ DECLARE_UVERBS_WRITE(
	3879	+ IB_USER_VERBS_CMD_ATTACH_MCAST,
	3880	+ ib_uverbs_attach_mcast,
	3881	+ UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),
	3882	+ UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),
	3883	+ UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
	3884	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,
	3885	+ ib_uverbs_create_qp,
	3886	+ UAPI_DEF_WRITE_UDATA_IO(
	3887	+ struct ib_uverbs_create_qp,
	3888	+ struct ib_uverbs_create_qp_resp),
	3889	+ UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
	3890	+ DECLARE_UVERBS_WRITE(
	3891	+ IB_USER_VERBS_CMD_DESTROY_QP,
	3892	+ ib_uverbs_destroy_qp,
	3893	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,
	3894	+ struct ib_uverbs_destroy_qp_resp),
	3895	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),
	3896	+ DECLARE_UVERBS_WRITE(
	3897	+ IB_USER_VERBS_CMD_DETACH_MCAST,
	3898	+ ib_uverbs_detach_mcast,
	3899	+ UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),
	3900	+ UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
	3901	+ DECLARE_UVERBS_WRITE(
	3902	+ IB_USER_VERBS_CMD_MODIFY_QP,
	3903	+ ib_uverbs_modify_qp,
	3904	+ UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),
	3905	+ UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),
	3906	+ DECLARE_UVERBS_WRITE(
	3907	+ IB_USER_VERBS_CMD_POST_RECV,
	3908	+ ib_uverbs_post_recv,
	3909	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,
	3910	+ struct ib_uverbs_post_recv_resp),
	3911	+ UAPI_DEF_METHOD_NEEDS_FN(post_recv)),
	3912	+ DECLARE_UVERBS_WRITE(
	3913	+ IB_USER_VERBS_CMD_POST_SEND,
	3914	+ ib_uverbs_post_send,
	3915	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,
	3916	+ struct ib_uverbs_post_send_resp),
	3917	+ UAPI_DEF_METHOD_NEEDS_FN(post_send)),
	3918	+ DECLARE_UVERBS_WRITE(
	3919	+ IB_USER_VERBS_CMD_QUERY_QP,
	3920	+ ib_uverbs_query_qp,
	3921	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,
	3922	+ struct ib_uverbs_query_qp_resp),
	3923	+ UAPI_DEF_METHOD_NEEDS_FN(query_qp)),
	3924	+ DECLARE_UVERBS_WRITE_EX(
	3925	+ IB_USER_VERBS_EX_CMD_CREATE_QP,
	3926	+ ib_uverbs_ex_create_qp,
	3927	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,
	3928	+ comp_mask,
	3929	+ struct ib_uverbs_ex_create_qp_resp,
	3930	+ response_length),
	3931	+ UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
	3932	+ DECLARE_UVERBS_WRITE_EX(
	3933	+ IB_USER_VERBS_EX_CMD_MODIFY_QP,
	3934	+ ib_uverbs_ex_modify_qp,
	3935	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,
	3936	+ base,
	3937	+ struct ib_uverbs_ex_modify_qp_resp,
	3938	+ response_length),
	3939	+ UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),
	3940	+
	3941	+ DECLARE_UVERBS_OBJECT(
	3942	+ UVERBS_OBJECT_RWQ_IND_TBL,
	3943	+ DECLARE_UVERBS_WRITE_EX(
	3944	+ IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
	3945	+ ib_uverbs_ex_create_rwq_ind_table,
	3946	+ UAPI_DEF_WRITE_IO_EX(
	3947	+ struct ib_uverbs_ex_create_rwq_ind_table,
	3948	+ log_ind_tbl_size,
	3949	+ struct ib_uverbs_ex_create_rwq_ind_table_resp,
	3950	+ ind_tbl_num),
	3951	+ UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),
	3952	+ DECLARE_UVERBS_WRITE_EX(
	3953	+ IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
	3954	+ ib_uverbs_ex_destroy_rwq_ind_table,
	3955	+ UAPI_DEF_WRITE_I(
	3956	+ struct ib_uverbs_ex_destroy_rwq_ind_table),
	3957	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),
	3958	+
	3959	+ DECLARE_UVERBS_OBJECT(
	3960	+ UVERBS_OBJECT_WQ,
	3961	+ DECLARE_UVERBS_WRITE_EX(
	3962	+ IB_USER_VERBS_EX_CMD_CREATE_WQ,
	3963	+ ib_uverbs_ex_create_wq,
	3964	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,
	3965	+ max_sge,
	3966	+ struct ib_uverbs_ex_create_wq_resp,
	3967	+ wqn),
	3968	+ UAPI_DEF_METHOD_NEEDS_FN(create_wq)),
	3969	+ DECLARE_UVERBS_WRITE_EX(
	3970	+ IB_USER_VERBS_EX_CMD_DESTROY_WQ,
	3971	+ ib_uverbs_ex_destroy_wq,
	3972	+ UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,
	3973	+ wq_handle,
	3974	+ struct ib_uverbs_ex_destroy_wq_resp,
	3975	+ reserved),
	3976	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),
	3977	+ DECLARE_UVERBS_WRITE_EX(
	3978	+ IB_USER_VERBS_EX_CMD_MODIFY_WQ,
	3979	+ ib_uverbs_ex_modify_wq,
	3980	+ UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,
	3981	+ curr_wq_state),
	3982	+ UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),
	3983	+
	3984	+ DECLARE_UVERBS_OBJECT(
	3985	+ UVERBS_OBJECT_SRQ,
	3986	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,
	3987	+ ib_uverbs_create_srq,
	3988	+ UAPI_DEF_WRITE_UDATA_IO(
	3989	+ struct ib_uverbs_create_srq,
	3990	+ struct ib_uverbs_create_srq_resp),
	3991	+ UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
	3992	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,
	3993	+ ib_uverbs_create_xsrq,
	3994	+ UAPI_DEF_WRITE_UDATA_IO(
	3995	+ struct ib_uverbs_create_xsrq,
	3996	+ struct ib_uverbs_create_srq_resp),
	3997	+ UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
	3998	+ DECLARE_UVERBS_WRITE(
	3999	+ IB_USER_VERBS_CMD_DESTROY_SRQ,
	4000	+ ib_uverbs_destroy_srq,
	4001	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,
	4002	+ struct ib_uverbs_destroy_srq_resp),
	4003	+ UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),
	4004	+ DECLARE_UVERBS_WRITE(
	4005	+ IB_USER_VERBS_CMD_MODIFY_SRQ,
	4006	+ ib_uverbs_modify_srq,
	4007	+ UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),
	4008	+ UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),
	4009	+ DECLARE_UVERBS_WRITE(
	4010	+ IB_USER_VERBS_CMD_POST_SRQ_RECV,
	4011	+ ib_uverbs_post_srq_recv,
	4012	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,
	4013	+ struct ib_uverbs_post_srq_recv_resp),
	4014	+ UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),
	4015	+ DECLARE_UVERBS_WRITE(
	4016	+ IB_USER_VERBS_CMD_QUERY_SRQ,
	4017	+ ib_uverbs_query_srq,
	4018	+ UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,
	4019	+ struct ib_uverbs_query_srq_resp),
	4020	+ UAPI_DEF_METHOD_NEEDS_FN(query_srq))),
	4021	+
	4022	+ DECLARE_UVERBS_OBJECT(
	4023	+ UVERBS_OBJECT_XRCD,
	4024	+ DECLARE_UVERBS_WRITE(
	4025	+ IB_USER_VERBS_CMD_CLOSE_XRCD,
	4026	+ ib_uverbs_close_xrcd,
	4027	+ UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd),
	4028	+ UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)),
	4029	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
	4030	+ ib_uverbs_open_qp,
	4031	+ UAPI_DEF_WRITE_UDATA_IO(
	4032	+ struct ib_uverbs_open_qp,
	4033	+ struct ib_uverbs_create_qp_resp)),
	4034	+ DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,
	4035	+ ib_uverbs_open_xrcd,
	4036	+ UAPI_DEF_WRITE_UDATA_IO(
	4037	+ struct ib_uverbs_open_xrcd,
	4038	+ struct ib_uverbs_open_xrcd_resp),
	4039	+ UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))),
	4040	+
	4041	+ {},
	4042	+};