~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,15 +1,7 @@
	1	+// SPDX-License-Identifier: GPL-2.0
1	2	/*
2	3	* NVMe over Fabrics RDMA host code.
3	4	* Copyright (c) 2015-2016 HGST, a Western Digital Company.
4		- *
5		- * This program is free software; you can redistribute it and/or modify it
6		- * under the terms and conditions of the GNU General Public License,
7		- * version 2, as published by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope it will be useful, but WITHOUT
10		- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11		- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12		- * more details.
13	5	*/
14	6	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15	7	#include <linux/module.h>
..	..	@@ -42,6 +34,11 @@
42	34
43	35	#define NVME_RDMA_MAX_INLINE_SEGMENTS 4
44	36
	37	+#define NVME_RDMA_DATA_SGL_SIZE \
	38	+ (sizeof(struct scatterlist) * NVME_INLINE_SG_CNT)
	39	+#define NVME_RDMA_METADATA_SGL_SIZE \
	40	+ (sizeof(struct scatterlist) * NVME_INLINE_METADATA_SG_CNT)
	41	+
45	42	struct nvme_rdma_device {
46	43	struct ib_device *dev;
47	44	struct ib_pd *pd;
..	..	@@ -56,6 +53,11 @@
56	53	u64 dma;
57	54	};
58	55
	56	+struct nvme_rdma_sgl {
	57	+ int nents;
	58	+ struct sg_table sg_table;
	59	+};
	60	+
59	61	struct nvme_rdma_queue;
60	62	struct nvme_rdma_request {
61	63	struct nvme_request req;
..	..	@@ -66,12 +68,12 @@
66	68	refcount_t ref;
67	69	struct ib_sge sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
68	70	u32 num_sge;
69		- int nents;
70	71	struct ib_reg_wr reg_wr;
71	72	struct ib_cqe reg_cqe;
72	73	struct nvme_rdma_queue *queue;
73		- struct sg_table sg_table;
74		- struct scatterlist first_sgl[];
	74	+ struct nvme_rdma_sgl data_sgl;
	75	+ struct nvme_rdma_sgl *metadata_sgl;
	76	+ bool use_sig_mr;
75	77	};
76	78
77	79	enum nvme_rdma_queue_flags {
..	..	@@ -93,6 +95,9 @@
93	95	struct rdma_cm_id *cm_id;
94	96	int cm_error;
95	97	struct completion cm_done;
	98	+ bool pi_support;
	99	+ int cq_size;
	100	+ struct mutex queue_lock;
96	101	};
97	102
98	103	struct nvme_rdma_ctrl {
..	..	@@ -118,8 +123,8 @@
118	123	struct sockaddr_storage src_addr;
119	124
120	125	struct nvme_ctrl ctrl;
121		- struct mutex teardown_lock;
122	126	bool use_inline_data;
	127	+ u32 io_queues[HCTX_MAX_TYPES];
123	128	};
124	129
125	130	static inline struct nvme_rdma_ctrl to_rdma_ctrl(struct nvme_ctrl ctrl)
..	..	@@ -146,21 +151,21 @@
146	151	static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
147	152	struct rdma_cm_event *event);
148	153	static void nvme_rdma_recv_done(struct ib_cq cq, struct ib_wc wc);
	154	+static void nvme_rdma_complete_rq(struct request *rq);
149	155
150	156	static const struct blk_mq_ops nvme_rdma_mq_ops;
151	157	static const struct blk_mq_ops nvme_rdma_admin_mq_ops;
152	158
153		-/* XXX: really should move to a generic header sooner or later.. */
154		-static inline void put_unaligned_le24(u32 val, u8 *p)
155		-{
156		- *p++ = val;
157		- *p++ = val >> 8;
158		- *p++ = val >> 16;
159		-}
160		-
161	159	static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)
162	160	{
163	161	return queue - queue->ctrl->queues;
	162	+}
	163	+
	164	+static bool nvme_rdma_poll_queue(struct nvme_rdma_queue *queue)
	165	+{
	166	+ return nvme_rdma_queue_idx(queue) >
	167	+ queue->ctrl->io_queues[HCTX_TYPE_DEFAULT] +
	168	+ queue->ctrl->io_queues[HCTX_TYPE_READ];
164	169	}
165	170
166	171	static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)
..	..	@@ -214,6 +219,11 @@
214	219	if (!ring)
215	220	return NULL;
216	221
	222	+ /*
	223	+ * Bind the CQEs (post recv buffers) DMA mapping to the RDMA queue
	224	+ * lifetime. It's safe, since any chage in the underlying RDMA device
	225	+ * will issue error recovery and queue re-creation.
	226	+ */
217	227	for (i = 0; i < ib_queue_size; i++) {
218	228	if (nvme_rdma_alloc_qe(ibdev, &ring[i], capsule_size, dir))
219	229	goto out_free_ring;
..	..	@@ -235,8 +245,15 @@
235	245
236	246	static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
237	247	{
238		- wait_for_completion_interruptible_timeout(&queue->cm_done,
	248	+ int ret;
	249	+
	250	+ ret = wait_for_completion_interruptible_timeout(&queue->cm_done,
239	251	msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1);
	252	+ if (ret < 0)
	253	+ return ret;
	254	+ if (ret == 0)
	255	+ return -ETIMEDOUT;
	256	+ WARN_ON_ONCE(queue->cm_error > 0);
240	257	return queue->cm_error;
241	258	}
242	259
..	..	@@ -258,6 +275,9 @@
258	275	init_attr.qp_type = IB_QPT_RC;
259	276	init_attr.send_cq = queue->ib_cq;
260	277	init_attr.recv_cq = queue->ib_cq;
	278	+ if (queue->pi_support)
	279	+ init_attr.create_flags \|= IB_QP_CREATE_INTEGRITY_EN;
	280	+ init_attr.qp_context = queue;
261	281
262	282	ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);
263	283
..	..	@@ -268,14 +288,9 @@
268	288	static void nvme_rdma_exit_request(struct blk_mq_tag_set *set,
269	289	struct request *rq, unsigned int hctx_idx)
270	290	{
271		- struct nvme_rdma_ctrl *ctrl = set->driver_data;
272	291	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
273		- int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
274		- struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
275		- struct nvme_rdma_device *dev = queue->device;
276	292
277		- nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
278		- DMA_TO_DEVICE);
	293	+ kfree(req->sqe.data);
279	294	}
280	295
281	296	static int nvme_rdma_init_request(struct blk_mq_tag_set *set,
..	..	@@ -286,15 +301,17 @@
286	301	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
287	302	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
288	303	struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
289		- struct nvme_rdma_device *dev = queue->device;
290		- struct ib_device *ibdev = dev->dev;
291		- int ret;
292	304
293	305	nvme_req(rq)->ctrl = &ctrl->ctrl;
294		- ret = nvme_rdma_alloc_qe(ibdev, &req->sqe, sizeof(struct nvme_command),
295		- DMA_TO_DEVICE);
296		- if (ret)
297		- return ret;
	306	+ req->sqe.data = kzalloc(sizeof(struct nvme_command), GFP_KERNEL);
	307	+ if (!req->sqe.data)
	308	+ return -ENOMEM;
	309	+
	310	+ /* metadata nvme_rdma_sgl struct is located after command's data SGL */
	311	+ if (queue->pi_support)
	312	+ req->metadata_sgl = (void *)nvme_req(rq) +
	313	+ sizeof(struct nvme_rdma_request) +
	314	+ NVME_RDMA_DATA_SGL_SIZE;
298	315
299	316	req->queue = queue;
300	317
..	..	@@ -395,6 +412,14 @@
395	412	return NULL;
396	413	}
397	414
	415	+static void nvme_rdma_free_cq(struct nvme_rdma_queue *queue)
	416	+{
	417	+ if (nvme_rdma_poll_queue(queue))
	418	+ ib_free_cq(queue->ib_cq);
	419	+ else
	420	+ ib_cq_pool_put(queue->ib_cq, queue->cq_size);
	421	+}
	422	+
398	423	static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
399	424	{
400	425	struct nvme_rdma_device *dev;
..	..	@@ -406,6 +431,8 @@
406	431	dev = queue->device;
407	432	ibdev = dev->dev;
408	433
	434	+ if (queue->pi_support)
	435	+ ib_mr_pool_destroy(queue->qp, &queue->qp->sig_mrs);
409	436	ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs);
410	437
411	438	/*
..	..	@@ -414,7 +441,7 @@
414	441	* the destruction of the QP shouldn't use rdma_cm API.
415	442	*/
416	443	ib_destroy_qp(queue->qp);
417		- ib_free_cq(queue->ib_cq);
	444	+ nvme_rdma_free_cq(queue);
418	445
419	446	nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
420	447	sizeof(struct nvme_completion), DMA_FROM_DEVICE);
..	..	@@ -422,10 +449,47 @@
422	449	nvme_rdma_dev_put(dev);
423	450	}
424	451
425		-static int nvme_rdma_get_max_fr_pages(struct ib_device *ibdev)
	452	+static int nvme_rdma_get_max_fr_pages(struct ib_device *ibdev, bool pi_support)
426	453	{
427		- return min_t(u32, NVME_RDMA_MAX_SEGMENTS,
428		- ibdev->attrs.max_fast_reg_page_list_len);
	454	+ u32 max_page_list_len;
	455	+
	456	+ if (pi_support)
	457	+ max_page_list_len = ibdev->attrs.max_pi_fast_reg_page_list_len;
	458	+ else
	459	+ max_page_list_len = ibdev->attrs.max_fast_reg_page_list_len;
	460	+
	461	+ return min_t(u32, NVME_RDMA_MAX_SEGMENTS, max_page_list_len - 1);
	462	+}
	463	+
	464	+static int nvme_rdma_create_cq(struct ib_device *ibdev,
	465	+ struct nvme_rdma_queue *queue)
	466	+{
	467	+ int ret, comp_vector, idx = nvme_rdma_queue_idx(queue);
	468	+ enum ib_poll_context poll_ctx;
	469	+
	470	+ /*
	471	+ * Spread I/O queues completion vectors according their queue index.
	472	+ * Admin queues can always go on completion vector 0.
	473	+ */
	474	+ comp_vector = (idx == 0 ? idx : idx - 1) % ibdev->num_comp_vectors;
	475	+
	476	+ /* Polling queues need direct cq polling context */
	477	+ if (nvme_rdma_poll_queue(queue)) {
	478	+ poll_ctx = IB_POLL_DIRECT;
	479	+ queue->ib_cq = ib_alloc_cq(ibdev, queue, queue->cq_size,
	480	+ comp_vector, poll_ctx);
	481	+ } else {
	482	+ poll_ctx = IB_POLL_SOFTIRQ;
	483	+ queue->ib_cq = ib_cq_pool_get(ibdev, queue->cq_size,
	484	+ comp_vector, poll_ctx);
	485	+ }
	486	+
	487	+ if (IS_ERR(queue->ib_cq)) {
	488	+ ret = PTR_ERR(queue->ib_cq);
	489	+ return ret;
	490	+ }
	491	+
	492	+ return 0;
429	493	}
430	494
431	495	static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
..	..	@@ -433,8 +497,7 @@
433	497	struct ib_device *ibdev;
434	498	const int send_wr_factor = 3; /* MR, SEND, INV */
435	499	const int cq_factor = send_wr_factor + 1; /* + RECV */
436		- int comp_vector, idx = nvme_rdma_queue_idx(queue);
437		- int ret;
	500	+ int ret, pages_per_mr;
438	501
439	502	queue->device = nvme_rdma_find_get_device(queue->cm_id);
440	503	if (!queue->device) {
..	..	@@ -444,20 +507,12 @@
444	507	}
445	508	ibdev = queue->device->dev;
446	509
447		- /*
448		- * Spread I/O queues completion vectors according their queue index.
449		- * Admin queues can always go on completion vector 0.
450		- */
451		- comp_vector = (idx == 0 ? idx : idx - 1) % ibdev->num_comp_vectors;
452		-
453	510	/* +1 for ib_stop_cq */
454		- queue->ib_cq = ib_alloc_cq(ibdev, queue,
455		- cq_factor * queue->queue_size + 1,
456		- comp_vector, IB_POLL_SOFTIRQ);
457		- if (IS_ERR(queue->ib_cq)) {
458		- ret = PTR_ERR(queue->ib_cq);
	511	+ queue->cq_size = cq_factor * queue->queue_size + 1;
	512	+
	513	+ ret = nvme_rdma_create_cq(ibdev, queue);
	514	+ if (ret)
459	515	goto out_put_dev;
460		- }
461	516
462	517	ret = nvme_rdma_create_qp(queue, send_wr_factor);
463	518	if (ret)
..	..	@@ -470,28 +525,48 @@
470	525	goto out_destroy_qp;
471	526	}
472	527
	528	+ /*
	529	+ * Currently we don't use SG_GAPS MR's so if the first entry is
	530	+ * misaligned we'll end up using two entries for a single data page,
	531	+ * so one additional entry is required.
	532	+ */
	533	+ pages_per_mr = nvme_rdma_get_max_fr_pages(ibdev, queue->pi_support) + 1;
473	534	ret = ib_mr_pool_init(queue->qp, &queue->qp->rdma_mrs,
474	535	queue->queue_size,
475	536	IB_MR_TYPE_MEM_REG,
476		- nvme_rdma_get_max_fr_pages(ibdev));
	537	+ pages_per_mr, 0);
477	538	if (ret) {
478	539	dev_err(queue->ctrl->ctrl.device,
479	540	"failed to initialize MR pool sized %d for QID %d\n",
480		- queue->queue_size, idx);
	541	+ queue->queue_size, nvme_rdma_queue_idx(queue));
481	542	goto out_destroy_ring;
	543	+ }
	544	+
	545	+ if (queue->pi_support) {
	546	+ ret = ib_mr_pool_init(queue->qp, &queue->qp->sig_mrs,
	547	+ queue->queue_size, IB_MR_TYPE_INTEGRITY,
	548	+ pages_per_mr, pages_per_mr);
	549	+ if (ret) {
	550	+ dev_err(queue->ctrl->ctrl.device,
	551	+ "failed to initialize PI MR pool sized %d for QID %d\n",
	552	+ queue->queue_size, nvme_rdma_queue_idx(queue));
	553	+ goto out_destroy_mr_pool;
	554	+ }
482	555	}
483	556
484	557	set_bit(NVME_RDMA_Q_TR_READY, &queue->flags);
485	558
486	559	return 0;
487	560
	561	+out_destroy_mr_pool:
	562	+ ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs);
488	563	out_destroy_ring:
489	564	nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
490	565	sizeof(struct nvme_completion), DMA_FROM_DEVICE);
491	566	out_destroy_qp:
492	567	rdma_destroy_qp(queue->cm_id);
493	568	out_destroy_ib_cq:
494		- ib_free_cq(queue->ib_cq);
	569	+ nvme_rdma_free_cq(queue);
495	570	out_put_dev:
496	571	nvme_rdma_dev_put(queue->device);
497	572	return ret;
..	..	@@ -505,7 +580,12 @@
505	580	int ret;
506	581
507	582	queue = &ctrl->queues[idx];
	583	+ mutex_init(&queue->queue_lock);
508	584	queue->ctrl = ctrl;
	585	+ if (idx && ctrl->ctrl.max_integrity_segments)
	586	+ queue->pi_support = true;
	587	+ else
	588	+ queue->pi_support = false;
509	589	init_completion(&queue->cm_done);
510	590
511	591	if (idx > 0)
..	..	@@ -520,7 +600,8 @@
520	600	if (IS_ERR(queue->cm_id)) {
521	601	dev_info(ctrl->ctrl.device,
522	602	"failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
523		- return PTR_ERR(queue->cm_id);
	603	+ ret = PTR_ERR(queue->cm_id);
	604	+ goto out_destroy_mutex;
524	605	}
525	606
526	607	if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
..	..	@@ -550,16 +631,23 @@
550	631	out_destroy_cm_id:
551	632	rdma_destroy_id(queue->cm_id);
552	633	nvme_rdma_destroy_queue_ib(queue);
	634	+out_destroy_mutex:
	635	+ mutex_destroy(&queue->queue_lock);
553	636	return ret;
	637	+}
	638	+
	639	+static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
	640	+{
	641	+ rdma_disconnect(queue->cm_id);
	642	+ ib_drain_qp(queue->qp);
554	643	}
555	644
556	645	static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
557	646	{
558		- if (!test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
559		- return;
560		-
561		- rdma_disconnect(queue->cm_id);
562		- ib_drain_qp(queue->qp);
	647	+ mutex_lock(&queue->queue_lock);
	648	+ if (test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
	649	+ __nvme_rdma_stop_queue(queue);
	650	+ mutex_unlock(&queue->queue_lock);
563	651	}
564	652
565	653	static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
..	..	@@ -567,8 +655,9 @@
567	655	if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
568	656	return;
569	657
570		- nvme_rdma_destroy_queue_ib(queue);
571	658	rdma_destroy_id(queue->cm_id);
	659	+ nvme_rdma_destroy_queue_ib(queue);
	660	+ mutex_destroy(&queue->queue_lock);
572	661	}
573	662
574	663	static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
..	..	@@ -589,18 +678,23 @@
589	678
590	679	static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
591	680	{
	681	+ struct nvme_rdma_queue *queue = &ctrl->queues[idx];
	682	+ bool poll = nvme_rdma_poll_queue(queue);
592	683	int ret;
593	684
594	685	if (idx)
595		- ret = nvmf_connect_io_queue(&ctrl->ctrl, idx);
	686	+ ret = nvmf_connect_io_queue(&ctrl->ctrl, idx, poll);
596	687	else
597	688	ret = nvmf_connect_admin_queue(&ctrl->ctrl);
598	689
599		- if (!ret)
600		- set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[idx].flags);
601		- else
	690	+ if (!ret) {
	691	+ set_bit(NVME_RDMA_Q_LIVE, &queue->flags);
	692	+ } else {
	693	+ if (test_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
	694	+ __nvme_rdma_stop_queue(queue);
602	695	dev_info(ctrl->ctrl.device,
603	696	"failed to connect queue: %d ret=%d\n", idx, ret);
	697	+ }
604	698	return ret;
605	699	}
606	700
..	..	@@ -626,18 +720,16 @@
626	720	{
627	721	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
628	722	struct ib_device *ibdev = ctrl->device->dev;
629		- unsigned int nr_io_queues;
	723	+ unsigned int nr_io_queues, nr_default_queues;
	724	+ unsigned int nr_read_queues, nr_poll_queues;
630	725	int i, ret;
631	726
632		- nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
633		-
634		- /*
635		- * we map queues according to the device irq vectors for
636		- * optimal locality so we don't need more queues than
637		- * completion vectors.
638		- */
639		- nr_io_queues = min_t(unsigned int, nr_io_queues,
640		- ibdev->num_comp_vectors);
	727	+ nr_read_queues = min_t(unsigned int, ibdev->num_comp_vectors,
	728	+ min(opts->nr_io_queues, num_online_cpus()));
	729	+ nr_default_queues = min_t(unsigned int, ibdev->num_comp_vectors,
	730	+ min(opts->nr_write_queues, num_online_cpus()));
	731	+ nr_poll_queues = min(opts->nr_poll_queues, num_online_cpus());
	732	+ nr_io_queues = nr_read_queues + nr_default_queues + nr_poll_queues;
641	733
642	734	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
643	735	if (ret)
..	..	@@ -652,6 +744,34 @@
652	744	ctrl->ctrl.queue_count = nr_io_queues + 1;
653	745	dev_info(ctrl->ctrl.device,
654	746	"creating %d I/O queues.\n", nr_io_queues);
	747	+
	748	+ if (opts->nr_write_queues && nr_read_queues < nr_io_queues) {
	749	+ /*
	750	+ * separate read/write queues
	751	+ * hand out dedicated default queues only after we have
	752	+ * sufficient read queues.
	753	+ */
	754	+ ctrl->io_queues[HCTX_TYPE_READ] = nr_read_queues;
	755	+ nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
	756	+ ctrl->io_queues[HCTX_TYPE_DEFAULT] =
	757	+ min(nr_default_queues, nr_io_queues);
	758	+ nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
	759	+ } else {
	760	+ /*
	761	+ * shared read/write queues
	762	+ * either no write queues were requested, or we don't have
	763	+ * sufficient queue count to have dedicated default queues.
	764	+ */
	765	+ ctrl->io_queues[HCTX_TYPE_DEFAULT] =
	766	+ min(nr_read_queues, nr_io_queues);
	767	+ nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
	768	+ }
	769	+
	770	+ if (opts->nr_poll_queues && nr_io_queues) {
	771	+ /* map dedicated poll queues only if we have queues left */
	772	+ ctrl->io_queues[HCTX_TYPE_POLL] =
	773	+ min(nr_poll_queues, nr_io_queues);
	774	+ }
655	775
656	776	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
657	777	ret = nvme_rdma_alloc_queue(ctrl, i,
..	..	@@ -669,15 +789,6 @@
669	789	return ret;
670	790	}
671	791
672		-static void nvme_rdma_free_tagset(struct nvme_ctrl *nctrl,
673		- struct blk_mq_tag_set *set)
674		-{
675		- struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
676		-
677		- blk_mq_free_tag_set(set);
678		- nvme_rdma_dev_put(ctrl->device);
679		-}
680		-
681	792	static struct blk_mq_tag_set nvme_rdma_alloc_tagset(struct nvme_ctrl nctrl,
682	793	bool admin)
683	794	{
..	..	@@ -691,9 +802,9 @@
691	802	set->ops = &nvme_rdma_admin_mq_ops;
692	803	set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
693	804	set->reserved_tags = 2; /* connect + keep-alive */
694		- set->numa_node = NUMA_NO_NODE;
	805	+ set->numa_node = nctrl->numa_node;
695	806	set->cmd_size = sizeof(struct nvme_rdma_request) +
696		- SG_CHUNK_SIZE * sizeof(struct scatterlist);
	807	+ NVME_RDMA_DATA_SGL_SIZE;
697	808	set->driver_data = ctrl;
698	809	set->nr_hw_queues = 1;
699	810	set->timeout = ADMIN_TIMEOUT;
..	..	@@ -704,35 +815,24 @@
704	815	set->ops = &nvme_rdma_mq_ops;
705	816	set->queue_depth = nctrl->sqsize + 1;
706	817	set->reserved_tags = 1; /* fabric connect */
707		- set->numa_node = NUMA_NO_NODE;
	818	+ set->numa_node = nctrl->numa_node;
708	819	set->flags = BLK_MQ_F_SHOULD_MERGE;
709	820	set->cmd_size = sizeof(struct nvme_rdma_request) +
710		- SG_CHUNK_SIZE * sizeof(struct scatterlist);
	821	+ NVME_RDMA_DATA_SGL_SIZE;
	822	+ if (nctrl->max_integrity_segments)
	823	+ set->cmd_size += sizeof(struct nvme_rdma_sgl) +
	824	+ NVME_RDMA_METADATA_SGL_SIZE;
711	825	set->driver_data = ctrl;
712	826	set->nr_hw_queues = nctrl->queue_count - 1;
713	827	set->timeout = NVME_IO_TIMEOUT;
	828	+ set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
714	829	}
715	830
716	831	ret = blk_mq_alloc_tag_set(set);
717	832	if (ret)
718		- goto out;
719		-
720		- /*
721		- * We need a reference on the device as long as the tag_set is alive,
722		- * as the MRs in the request structures need a valid ib_device.
723		- */
724		- ret = nvme_rdma_dev_get(ctrl->device);
725		- if (!ret) {
726		- ret = -EINVAL;
727		- goto out_free_tagset;
728		- }
	833	+ return ERR_PTR(ret);
729	834
730	835	return set;
731		-
732		-out_free_tagset:
733		- blk_mq_free_tag_set(set);
734		-out:
735		- return ERR_PTR(ret);
736	836	}
737	837
738	838	static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl,
..	..	@@ -740,7 +840,8 @@
740	840	{
741	841	if (remove) {
742	842	blk_cleanup_queue(ctrl->ctrl.admin_q);
743		- nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
	843	+ blk_cleanup_queue(ctrl->ctrl.fabrics_q);
	844	+ blk_mq_free_tag_set(ctrl->ctrl.admin_tagset);
744	845	}
745	846	if (ctrl->async_event_sqe.data) {
746	847	cancel_work_sync(&ctrl->ctrl.async_event_work);
..	..	@@ -754,6 +855,7 @@
754	855	static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
755	856	bool new)
756	857	{
	858	+ bool pi_capable = false;
757	859	int error;
758	860
759	861	error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
..	..	@@ -761,9 +863,21 @@
761	863	return error;
762	864
763	865	ctrl->device = ctrl->queues[0].device;
	866	+ ctrl->ctrl.numa_node = ibdev_to_node(ctrl->device->dev);
764	867
765		- ctrl->max_fr_pages = nvme_rdma_get_max_fr_pages(ctrl->device->dev);
	868	+ /* T10-PI support */
	869	+ if (ctrl->device->dev->attrs.device_cap_flags &
	870	+ IB_DEVICE_INTEGRITY_HANDOVER)
	871	+ pi_capable = true;
766	872
	873	+ ctrl->max_fr_pages = nvme_rdma_get_max_fr_pages(ctrl->device->dev,
	874	+ pi_capable);
	875	+
	876	+ /*
	877	+ * Bind the async event SQE DMA mapping to the admin queue lifetime.
	878	+ * It's safe, since any chage in the underlying RDMA device will issue
	879	+ * error recovery and queue re-creation.
	880	+ */
767	881	error = nvme_rdma_alloc_qe(ctrl->device->dev, &ctrl->async_event_sqe,
768	882	sizeof(struct nvme_command), DMA_TO_DEVICE);
769	883	if (error)
..	..	@@ -776,10 +890,16 @@
776	890	goto out_free_async_qe;
777	891	}
778	892
	893	+ ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set);
	894	+ if (IS_ERR(ctrl->ctrl.fabrics_q)) {
	895	+ error = PTR_ERR(ctrl->ctrl.fabrics_q);
	896	+ goto out_free_tagset;
	897	+ }
	898	+
779	899	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
780	900	if (IS_ERR(ctrl->ctrl.admin_q)) {
781	901	error = PTR_ERR(ctrl->ctrl.admin_q);
782		- goto out_free_tagset;
	902	+ goto out_cleanup_fabrics_q;
783	903	}
784	904	}
785	905
..	..	@@ -787,38 +907,40 @@
787	907	if (error)
788	908	goto out_cleanup_queue;
789	909
790		- error = ctrl->ctrl.ops->reg_read64(&ctrl->ctrl, NVME_REG_CAP,
791		- &ctrl->ctrl.cap);
792		- if (error) {
793		- dev_err(ctrl->ctrl.device,
794		- "prop_get NVME_REG_CAP failed\n");
795		- goto out_stop_queue;
796		- }
797		-
798		- ctrl->ctrl.sqsize =
799		- min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
800		-
801		- error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
	910	+ error = nvme_enable_ctrl(&ctrl->ctrl);
802	911	if (error)
803	912	goto out_stop_queue;
804	913
805		- ctrl->ctrl.max_hw_sectors =
806		- (ctrl->max_fr_pages - 1) << (ilog2(SZ_4K) - 9);
	914	+ ctrl->ctrl.max_segments = ctrl->max_fr_pages;
	915	+ ctrl->ctrl.max_hw_sectors = ctrl->max_fr_pages << (ilog2(SZ_4K) - 9);
	916	+ if (pi_capable)
	917	+ ctrl->ctrl.max_integrity_segments = ctrl->max_fr_pages;
	918	+ else
	919	+ ctrl->ctrl.max_integrity_segments = 0;
	920	+
	921	+ blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
807	922
808	923	error = nvme_init_identify(&ctrl->ctrl);
809	924	if (error)
810		- goto out_stop_queue;
	925	+ goto out_quiesce_queue;
811	926
812	927	return 0;
813	928
	929	+out_quiesce_queue:
	930	+ blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
	931	+ blk_sync_queue(ctrl->ctrl.admin_q);
814	932	out_stop_queue:
815	933	nvme_rdma_stop_queue(&ctrl->queues[0]);
	934	+ nvme_cancel_admin_tagset(&ctrl->ctrl);
816	935	out_cleanup_queue:
817	936	if (new)
818	937	blk_cleanup_queue(ctrl->ctrl.admin_q);
	938	+out_cleanup_fabrics_q:
	939	+ if (new)
	940	+ blk_cleanup_queue(ctrl->ctrl.fabrics_q);
819	941	out_free_tagset:
820	942	if (new)
821		- nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
	943	+ blk_mq_free_tag_set(ctrl->ctrl.admin_tagset);
822	944	out_free_async_qe:
823	945	if (ctrl->async_event_sqe.data) {
824	946	nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe,
..	..	@@ -835,7 +957,7 @@
835	957	{
836	958	if (remove) {
837	959	blk_cleanup_queue(ctrl->ctrl.connect_q);
838		- nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
	960	+ blk_mq_free_tag_set(ctrl->ctrl.tagset);
839	961	}
840	962	nvme_rdma_free_io_queues(ctrl);
841	963	}
..	..	@@ -860,23 +982,41 @@
860	982	ret = PTR_ERR(ctrl->ctrl.connect_q);
861	983	goto out_free_tag_set;
862	984	}
863		- } else {
864		- blk_mq_update_nr_hw_queues(&ctrl->tag_set,
865		- ctrl->ctrl.queue_count - 1);
866	985	}
867	986
868	987	ret = nvme_rdma_start_io_queues(ctrl);
869	988	if (ret)
870	989	goto out_cleanup_connect_q;
871	990
	991	+ if (!new) {
	992	+ nvme_start_queues(&ctrl->ctrl);
	993	+ if (!nvme_wait_freeze_timeout(&ctrl->ctrl, NVME_IO_TIMEOUT)) {
	994	+ /*
	995	+ * If we timed out waiting for freeze we are likely to
	996	+ * be stuck. Fail the controller initialization just
	997	+ * to be safe.
	998	+ */
	999	+ ret = -ENODEV;
	1000	+ goto out_wait_freeze_timed_out;
	1001	+ }
	1002	+ blk_mq_update_nr_hw_queues(ctrl->ctrl.tagset,
	1003	+ ctrl->ctrl.queue_count - 1);
	1004	+ nvme_unfreeze(&ctrl->ctrl);
	1005	+ }
	1006	+
872	1007	return 0;
873	1008
	1009	+out_wait_freeze_timed_out:
	1010	+ nvme_stop_queues(&ctrl->ctrl);
	1011	+ nvme_sync_io_queues(&ctrl->ctrl);
	1012	+ nvme_rdma_stop_io_queues(ctrl);
874	1013	out_cleanup_connect_q:
	1014	+ nvme_cancel_tagset(&ctrl->ctrl);
875	1015	if (new)
876	1016	blk_cleanup_queue(ctrl->ctrl.connect_q);
877	1017	out_free_tag_set:
878	1018	if (new)
879		- nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
	1019	+ blk_mq_free_tag_set(ctrl->ctrl.tagset);
880	1020	out_free_io_queues:
881	1021	nvme_rdma_free_io_queues(ctrl);
882	1022	return ret;
..	..	@@ -885,32 +1025,36 @@
885	1025	static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl,
886	1026	bool remove)
887	1027	{
888		- mutex_lock(&ctrl->teardown_lock);
889	1028	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
	1029	+ blk_sync_queue(ctrl->ctrl.admin_q);
890	1030	nvme_rdma_stop_queue(&ctrl->queues[0]);
891		- if (ctrl->ctrl.admin_tagset)
	1031	+ if (ctrl->ctrl.admin_tagset) {
892	1032	blk_mq_tagset_busy_iter(ctrl->ctrl.admin_tagset,
893	1033	nvme_cancel_request, &ctrl->ctrl);
894		- blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
	1034	+ blk_mq_tagset_wait_completed_request(ctrl->ctrl.admin_tagset);
	1035	+ }
	1036	+ if (remove)
	1037	+ blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
895	1038	nvme_rdma_destroy_admin_queue(ctrl, remove);
896		- mutex_unlock(&ctrl->teardown_lock);
897	1039	}
898	1040
899	1041	static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl,
900	1042	bool remove)
901	1043	{
902		- mutex_lock(&ctrl->teardown_lock);
903	1044	if (ctrl->ctrl.queue_count > 1) {
	1045	+ nvme_start_freeze(&ctrl->ctrl);
904	1046	nvme_stop_queues(&ctrl->ctrl);
	1047	+ nvme_sync_io_queues(&ctrl->ctrl);
905	1048	nvme_rdma_stop_io_queues(ctrl);
906		- if (ctrl->ctrl.tagset)
	1049	+ if (ctrl->ctrl.tagset) {
907	1050	blk_mq_tagset_busy_iter(ctrl->ctrl.tagset,
908	1051	nvme_cancel_request, &ctrl->ctrl);
	1052	+ blk_mq_tagset_wait_completed_request(ctrl->ctrl.tagset);
	1053	+ }
909	1054	if (remove)
910	1055	nvme_start_queues(&ctrl->ctrl);
911	1056	nvme_rdma_destroy_io_queues(ctrl, remove);
912	1057	}
913		- mutex_unlock(&ctrl->teardown_lock);
914	1058	}
915	1059
916	1060	static void nvme_rdma_stop_ctrl(struct nvme_ctrl *nctrl)
..	..	@@ -1003,8 +1147,14 @@
1003	1147
1004	1148	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1005	1149	if (!changed) {
1006		- /* state change failure is ok if we're in DELETING state */
1007		- WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
	1150	+ /*
	1151	+ * state change failure is ok if we started ctrl delete,
	1152	+ * unless we're during creation of a new controller to
	1153	+ * avoid races with teardown flow.
	1154	+ */
	1155	+ WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING &&
	1156	+ ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO);
	1157	+ WARN_ON_ONCE(new);
1008	1158	ret = -EINVAL;
1009	1159	goto destroy_io;
1010	1160	}
..	..	@@ -1013,10 +1163,18 @@
1013	1163	return 0;
1014	1164
1015	1165	destroy_io:
1016		- if (ctrl->ctrl.queue_count > 1)
	1166	+ if (ctrl->ctrl.queue_count > 1) {
	1167	+ nvme_stop_queues(&ctrl->ctrl);
	1168	+ nvme_sync_io_queues(&ctrl->ctrl);
	1169	+ nvme_rdma_stop_io_queues(ctrl);
	1170	+ nvme_cancel_tagset(&ctrl->ctrl);
1017	1171	nvme_rdma_destroy_io_queues(ctrl, new);
	1172	+ }
1018	1173	destroy_admin:
	1174	+ blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
	1175	+ blk_sync_queue(ctrl->ctrl.admin_q);
1019	1176	nvme_rdma_stop_queue(&ctrl->queues[0]);
	1177	+ nvme_cancel_admin_tagset(&ctrl->ctrl);
1020	1178	nvme_rdma_destroy_admin_queue(ctrl, new);
1021	1179	return ret;
1022	1180	}
..	..	@@ -1054,10 +1212,12 @@
1054	1212	nvme_rdma_teardown_io_queues(ctrl, false);
1055	1213	nvme_start_queues(&ctrl->ctrl);
1056	1214	nvme_rdma_teardown_admin_queue(ctrl, false);
	1215	+ blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
1057	1216
1058	1217	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
1059		- /* state change failure is ok if we're in DELETING state */
1060		- WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
	1218	+ /* state change failure is ok if we started ctrl delete */
	1219	+ WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING &&
	1220	+ ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO);
1061	1221	return;
1062	1222	}
1063	1223
..	..	@@ -1069,13 +1229,24 @@
1069	1229	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
1070	1230	return;
1071	1231
1072		- queue_work(nvme_wq, &ctrl->err_work);
	1232	+ dev_warn(ctrl->ctrl.device, "starting error recovery\n");
	1233	+ queue_work(nvme_reset_wq, &ctrl->err_work);
	1234	+}
	1235	+
	1236	+static void nvme_rdma_end_request(struct nvme_rdma_request *req)
	1237	+{
	1238	+ struct request *rq = blk_mq_rq_from_pdu(req);
	1239	+
	1240	+ if (!refcount_dec_and_test(&req->ref))
	1241	+ return;
	1242	+ if (!nvme_try_complete_req(rq, req->status, req->result))
	1243	+ nvme_rdma_complete_rq(rq);
1073	1244	}
1074	1245
1075	1246	static void nvme_rdma_wr_error(struct ib_cq cq, struct ib_wc wc,
1076	1247	const char *op)
1077	1248	{
1078		- struct nvme_rdma_queue *queue = cq->cq_context;
	1249	+ struct nvme_rdma_queue *queue = wc->qp->qp_context;
1079	1250	struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1080	1251
1081	1252	if (ctrl->ctrl.state == NVME_CTRL_LIVE)
..	..	@@ -1096,16 +1267,11 @@
1096	1267	{
1097	1268	struct nvme_rdma_request *req =
1098	1269	container_of(wc->wr_cqe, struct nvme_rdma_request, reg_cqe);
1099		- struct request *rq = blk_mq_rq_from_pdu(req);
1100	1270
1101		- if (unlikely(wc->status != IB_WC_SUCCESS)) {
	1271	+ if (unlikely(wc->status != IB_WC_SUCCESS))
1102	1272	nvme_rdma_wr_error(cq, wc, "LOCAL_INV");
1103		- return;
1104		- }
1105		-
1106		- if (refcount_dec_and_test(&req->ref))
1107		- nvme_end_request(rq, req->status, req->result);
1108		-
	1273	+ else
	1274	+ nvme_rdma_end_request(req);
1109	1275	}
1110	1276
1111	1277	static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,
..	..	@@ -1131,21 +1297,29 @@
1131	1297	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1132	1298	struct nvme_rdma_device *dev = queue->device;
1133	1299	struct ib_device *ibdev = dev->dev;
	1300	+ struct list_head *pool = &queue->qp->rdma_mrs;
1134	1301
1135		- if (!blk_rq_payload_bytes(rq))
	1302	+ if (!blk_rq_nr_phys_segments(rq))
1136	1303	return;
1137	1304
	1305	+ if (blk_integrity_rq(rq)) {
	1306	+ ib_dma_unmap_sg(ibdev, req->metadata_sgl->sg_table.sgl,
	1307	+ req->metadata_sgl->nents, rq_dma_dir(rq));
	1308	+ sg_free_table_chained(&req->metadata_sgl->sg_table,
	1309	+ NVME_INLINE_METADATA_SG_CNT);
	1310	+ }
	1311	+
	1312	+ if (req->use_sig_mr)
	1313	+ pool = &queue->qp->sig_mrs;
	1314	+
1138	1315	if (req->mr) {
1139		- ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr);
	1316	+ ib_mr_pool_put(queue->qp, pool, req->mr);
1140	1317	req->mr = NULL;
1141	1318	}
1142	1319
1143		- ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
1144		- req->nents, rq_data_dir(rq) ==
1145		- WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1146		-
1147		- nvme_cleanup_cmd(rq);
1148		- sg_free_table_chained(&req->sg_table, true);
	1320	+ ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents,
	1321	+ rq_dma_dir(rq));
	1322	+ sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT);
1149	1323	}
1150	1324
1151	1325	static int nvme_rdma_set_sg_null(struct nvme_command *c)
..	..	@@ -1164,16 +1338,17 @@
1164	1338	int count)
1165	1339	{
1166	1340	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
1167		- struct scatterlist *sgl = req->sg_table.sgl;
1168	1341	struct ib_sge *sge = &req->sge[1];
	1342	+ struct scatterlist *sgl;
1169	1343	u32 len = 0;
1170	1344	int i;
1171	1345
1172		- for (i = 0; i < count; i++, sgl++, sge++) {
	1346	+ for_each_sg(req->data_sgl.sg_table.sgl, sgl, count, i) {
1173	1347	sge->addr = sg_dma_address(sgl);
1174	1348	sge->length = sg_dma_len(sgl);
1175	1349	sge->lkey = queue->device->pd->local_dma_lkey;
1176	1350	len += sge->length;
	1351	+ sge++;
1177	1352	}
1178	1353
1179	1354	sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
..	..	@@ -1189,8 +1364,8 @@
1189	1364	{
1190	1365	struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1191	1366
1192		- sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
1193		- put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
	1367	+ sg->addr = cpu_to_le64(sg_dma_address(req->data_sgl.sg_table.sgl));
	1368	+ put_unaligned_le24(sg_dma_len(req->data_sgl.sg_table.sgl), sg->length);
1194	1369	put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);
1195	1370	sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
1196	1371	return 0;
..	..	@@ -1211,7 +1386,8 @@
1211	1386	* Align the MR to a 4K page size to match the ctrl page size and
1212	1387	* the block virtual boundary.
1213	1388	*/
1214		- nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, SZ_4K);
	1389	+ nr = ib_map_mr_sg(req->mr, req->data_sgl.sg_table.sgl, count, NULL,
	1390	+ SZ_4K);
1215	1391	if (unlikely(nr < count)) {
1216	1392	ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr);
1217	1393	req->mr = NULL;
..	..	@@ -1242,12 +1418,125 @@
1242	1418	return 0;
1243	1419	}
1244	1420
	1421	+static void nvme_rdma_set_sig_domain(struct blk_integrity *bi,
	1422	+ struct nvme_command cmd, struct ib_sig_domain domain,
	1423	+ u16 control, u8 pi_type)
	1424	+{
	1425	+ domain->sig_type = IB_SIG_TYPE_T10_DIF;
	1426	+ domain->sig.dif.bg_type = IB_T10DIF_CRC;
	1427	+ domain->sig.dif.pi_interval = 1 << bi->interval_exp;
	1428	+ domain->sig.dif.ref_tag = le32_to_cpu(cmd->rw.reftag);
	1429	+ if (control & NVME_RW_PRINFO_PRCHK_REF)
	1430	+ domain->sig.dif.ref_remap = true;
	1431	+
	1432	+ domain->sig.dif.app_tag = le16_to_cpu(cmd->rw.apptag);
	1433	+ domain->sig.dif.apptag_check_mask = le16_to_cpu(cmd->rw.appmask);
	1434	+ domain->sig.dif.app_escape = true;
	1435	+ if (pi_type == NVME_NS_DPS_PI_TYPE3)
	1436	+ domain->sig.dif.ref_escape = true;
	1437	+}
	1438	+
	1439	+static void nvme_rdma_set_sig_attrs(struct blk_integrity *bi,
	1440	+ struct nvme_command cmd, struct ib_sig_attrs sig_attrs,
	1441	+ u8 pi_type)
	1442	+{
	1443	+ u16 control = le16_to_cpu(cmd->rw.control);
	1444	+
	1445	+ memset(sig_attrs, 0, sizeof(*sig_attrs));
	1446	+ if (control & NVME_RW_PRINFO_PRACT) {
	1447	+ /* for WRITE_INSERT/READ_STRIP no memory domain */
	1448	+ sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
	1449	+ nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control,
	1450	+ pi_type);
	1451	+ /* Clear the PRACT bit since HCA will generate/verify the PI */
	1452	+ control &= ~NVME_RW_PRINFO_PRACT;
	1453	+ cmd->rw.control = cpu_to_le16(control);
	1454	+ } else {
	1455	+ /* for WRITE_PASS/READ_PASS both wire/memory domains exist */
	1456	+ nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control,
	1457	+ pi_type);
	1458	+ nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control,
	1459	+ pi_type);
	1460	+ }
	1461	+}
	1462	+
	1463	+static void nvme_rdma_set_prot_checks(struct nvme_command cmd, u8 mask)
	1464	+{
	1465	+ *mask = 0;
	1466	+ if (le16_to_cpu(cmd->rw.control) & NVME_RW_PRINFO_PRCHK_REF)
	1467	+ *mask \|= IB_SIG_CHECK_REFTAG;
	1468	+ if (le16_to_cpu(cmd->rw.control) & NVME_RW_PRINFO_PRCHK_GUARD)
	1469	+ *mask \|= IB_SIG_CHECK_GUARD;
	1470	+}
	1471	+
	1472	+static void nvme_rdma_sig_done(struct ib_cq cq, struct ib_wc wc)
	1473	+{
	1474	+ if (unlikely(wc->status != IB_WC_SUCCESS))
	1475	+ nvme_rdma_wr_error(cq, wc, "SIG");
	1476	+}
	1477	+
	1478	+static int nvme_rdma_map_sg_pi(struct nvme_rdma_queue *queue,
	1479	+ struct nvme_rdma_request req, struct nvme_command c,
	1480	+ int count, int pi_count)
	1481	+{
	1482	+ struct nvme_rdma_sgl *sgl = &req->data_sgl;
	1483	+ struct ib_reg_wr *wr = &req->reg_wr;
	1484	+ struct request *rq = blk_mq_rq_from_pdu(req);
	1485	+ struct nvme_ns *ns = rq->q->queuedata;
	1486	+ struct bio *bio = rq->bio;
	1487	+ struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
	1488	+ int nr;
	1489	+
	1490	+ req->mr = ib_mr_pool_get(queue->qp, &queue->qp->sig_mrs);
	1491	+ if (WARN_ON_ONCE(!req->mr))
	1492	+ return -EAGAIN;
	1493	+
	1494	+ nr = ib_map_mr_sg_pi(req->mr, sgl->sg_table.sgl, count, NULL,
	1495	+ req->metadata_sgl->sg_table.sgl, pi_count, NULL,
	1496	+ SZ_4K);
	1497	+ if (unlikely(nr))
	1498	+ goto mr_put;
	1499	+
	1500	+ nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_disk), c,
	1501	+ req->mr->sig_attrs, ns->pi_type);
	1502	+ nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask);
	1503	+
	1504	+ ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
	1505	+
	1506	+ req->reg_cqe.done = nvme_rdma_sig_done;
	1507	+ memset(wr, 0, sizeof(*wr));
	1508	+ wr->wr.opcode = IB_WR_REG_MR_INTEGRITY;
	1509	+ wr->wr.wr_cqe = &req->reg_cqe;
	1510	+ wr->wr.num_sge = 0;
	1511	+ wr->wr.send_flags = 0;
	1512	+ wr->mr = req->mr;
	1513	+ wr->key = req->mr->rkey;
	1514	+ wr->access = IB_ACCESS_LOCAL_WRITE \|
	1515	+ IB_ACCESS_REMOTE_READ \|
	1516	+ IB_ACCESS_REMOTE_WRITE;
	1517	+
	1518	+ sg->addr = cpu_to_le64(req->mr->iova);
	1519	+ put_unaligned_le24(req->mr->length, sg->length);
	1520	+ put_unaligned_le32(req->mr->rkey, sg->key);
	1521	+ sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
	1522	+
	1523	+ return 0;
	1524	+
	1525	+mr_put:
	1526	+ ib_mr_pool_put(queue->qp, &queue->qp->sig_mrs, req->mr);
	1527	+ req->mr = NULL;
	1528	+ if (nr < 0)
	1529	+ return nr;
	1530	+ return -EINVAL;
	1531	+}
	1532	+
1245	1533	static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
1246	1534	struct request rq, struct nvme_command c)
1247	1535	{
1248	1536	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1249	1537	struct nvme_rdma_device *dev = queue->device;
1250	1538	struct ib_device *ibdev = dev->dev;
	1539	+ int pi_count = 0;
1251	1540	int count, ret;
1252	1541
1253	1542	req->num_sge = 1;
..	..	@@ -1255,22 +1544,53 @@
1255	1544
1256	1545	c->common.flags \|= NVME_CMD_SGL_METABUF;
1257	1546
1258		- if (!blk_rq_payload_bytes(rq))
	1547	+ if (!blk_rq_nr_phys_segments(rq))
1259	1548	return nvme_rdma_set_sg_null(c);
1260	1549
1261		- req->sg_table.sgl = req->first_sgl;
1262		- ret = sg_alloc_table_chained(&req->sg_table,
1263		- blk_rq_nr_phys_segments(rq), req->sg_table.sgl);
	1550	+ req->data_sgl.sg_table.sgl = (struct scatterlist *)(req + 1);
	1551	+ ret = sg_alloc_table_chained(&req->data_sgl.sg_table,
	1552	+ blk_rq_nr_phys_segments(rq), req->data_sgl.sg_table.sgl,
	1553	+ NVME_INLINE_SG_CNT);
1264	1554	if (ret)
1265	1555	return -ENOMEM;
1266	1556
1267		- req->nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
	1557	+ req->data_sgl.nents = blk_rq_map_sg(rq->q, rq,
	1558	+ req->data_sgl.sg_table.sgl);
1268	1559
1269		- count = ib_dma_map_sg(ibdev, req->sg_table.sgl, req->nents,
1270		- rq_data_dir(rq) == WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
	1560	+ count = ib_dma_map_sg(ibdev, req->data_sgl.sg_table.sgl,
	1561	+ req->data_sgl.nents, rq_dma_dir(rq));
1271	1562	if (unlikely(count <= 0)) {
1272	1563	ret = -EIO;
1273	1564	goto out_free_table;
	1565	+ }
	1566	+
	1567	+ if (blk_integrity_rq(rq)) {
	1568	+ req->metadata_sgl->sg_table.sgl =
	1569	+ (struct scatterlist *)(req->metadata_sgl + 1);
	1570	+ ret = sg_alloc_table_chained(&req->metadata_sgl->sg_table,
	1571	+ blk_rq_count_integrity_sg(rq->q, rq->bio),
	1572	+ req->metadata_sgl->sg_table.sgl,
	1573	+ NVME_INLINE_METADATA_SG_CNT);
	1574	+ if (unlikely(ret)) {
	1575	+ ret = -ENOMEM;
	1576	+ goto out_unmap_sg;
	1577	+ }
	1578	+
	1579	+ req->metadata_sgl->nents = blk_rq_map_integrity_sg(rq->q,
	1580	+ rq->bio, req->metadata_sgl->sg_table.sgl);
	1581	+ pi_count = ib_dma_map_sg(ibdev,
	1582	+ req->metadata_sgl->sg_table.sgl,
	1583	+ req->metadata_sgl->nents,
	1584	+ rq_dma_dir(rq));
	1585	+ if (unlikely(pi_count <= 0)) {
	1586	+ ret = -EIO;
	1587	+ goto out_free_pi_table;
	1588	+ }
	1589	+ }
	1590	+
	1591	+ if (req->use_sig_mr) {
	1592	+ ret = nvme_rdma_map_sg_pi(queue, req, c, count, pi_count);
	1593	+ goto out;
1274	1594	}
1275	1595
1276	1596	if (count <= dev->num_inline_segments) {
..	..	@@ -1291,16 +1611,23 @@
1291	1611	ret = nvme_rdma_map_sg_fr(queue, req, c, count);
1292	1612	out:
1293	1613	if (unlikely(ret))
1294		- goto out_unmap_sg;
	1614	+ goto out_unmap_pi_sg;
1295	1615
1296	1616	return 0;
1297	1617
	1618	+out_unmap_pi_sg:
	1619	+ if (blk_integrity_rq(rq))
	1620	+ ib_dma_unmap_sg(ibdev, req->metadata_sgl->sg_table.sgl,
	1621	+ req->metadata_sgl->nents, rq_dma_dir(rq));
	1622	+out_free_pi_table:
	1623	+ if (blk_integrity_rq(rq))
	1624	+ sg_free_table_chained(&req->metadata_sgl->sg_table,
	1625	+ NVME_INLINE_METADATA_SG_CNT);
1298	1626	out_unmap_sg:
1299		- ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
1300		- req->nents, rq_data_dir(rq) ==
1301		- WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
	1627	+ ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents,
	1628	+ rq_dma_dir(rq));
1302	1629	out_free_table:
1303		- sg_free_table_chained(&req->sg_table, true);
	1630	+ sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT);
1304	1631	return ret;
1305	1632	}
1306	1633
..	..	@@ -1310,15 +1637,11 @@
1310	1637	container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1311	1638	struct nvme_rdma_request *req =
1312	1639	container_of(qe, struct nvme_rdma_request, sqe);
1313		- struct request *rq = blk_mq_rq_from_pdu(req);
1314	1640
1315		- if (unlikely(wc->status != IB_WC_SUCCESS)) {
	1641	+ if (unlikely(wc->status != IB_WC_SUCCESS))
1316	1642	nvme_rdma_wr_error(cq, wc, "SEND");
1317		- return;
1318		- }
1319		-
1320		- if (refcount_dec_and_test(&req->ref))
1321		- nvme_end_request(rq, req->status, req->result);
	1643	+ else
	1644	+ nvme_rdma_end_request(req);
1322	1645	}
1323	1646
1324	1647	static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
..	..	@@ -1329,7 +1652,7 @@
1329	1652	int ret;
1330	1653
1331	1654	sge->addr = qe->dma;
1332		- sge->length = sizeof(struct nvme_command),
	1655	+ sge->length = sizeof(struct nvme_command);
1333	1656	sge->lkey = queue->device->pd->local_dma_lkey;
1334	1657
1335	1658	wr.next = NULL;
..	..	@@ -1420,20 +1743,19 @@
1420	1743	WARN_ON_ONCE(ret);
1421	1744	}
1422	1745
1423		-static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
1424		- struct nvme_completion cqe, struct ib_wc wc, int tag)
	1746	+static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
	1747	+ struct nvme_completion cqe, struct ib_wc wc)
1425	1748	{
1426	1749	struct request *rq;
1427	1750	struct nvme_rdma_request *req;
1428		- int ret = 0;
1429	1751
1430		- rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
	1752	+ rq = nvme_find_rq(nvme_rdma_tagset(queue), cqe->command_id);
1431	1753	if (!rq) {
1432	1754	dev_err(queue->ctrl->ctrl.device,
1433		- "tag 0x%x on QP %#x not found\n",
	1755	+ "got bad command_id %#x on QP %#x\n",
1434	1756	cqe->command_id, queue->qp->qp_num);
1435	1757	nvme_rdma_error_recovery(queue->ctrl);
1436		- return ret;
	1758	+ return;
1437	1759	}
1438	1760	req = blk_mq_rq_to_pdu(rq);
1439	1761
..	..	@@ -1441,13 +1763,16 @@
1441	1763	req->result = cqe->result;
1442	1764
1443	1765	if (wc->wc_flags & IB_WC_WITH_INVALIDATE) {
1444		- if (unlikely(wc->ex.invalidate_rkey != req->mr->rkey)) {
	1766	+ if (unlikely(!req->mr \|\|
	1767	+ wc->ex.invalidate_rkey != req->mr->rkey)) {
1445	1768	dev_err(queue->ctrl->ctrl.device,
1446	1769	"Bogus remote invalidation for rkey %#x\n",
1447		- req->mr->rkey);
	1770	+ req->mr ? req->mr->rkey : 0);
1448	1771	nvme_rdma_error_recovery(queue->ctrl);
1449	1772	}
1450	1773	} else if (req->mr) {
	1774	+ int ret;
	1775	+
1451	1776	ret = nvme_rdma_inv_rkey(queue, req);
1452	1777	if (unlikely(ret < 0)) {
1453	1778	dev_err(queue->ctrl->ctrl.device,
..	..	@@ -1456,31 +1781,32 @@
1456	1781	nvme_rdma_error_recovery(queue->ctrl);
1457	1782	}
1458	1783	/* the local invalidation completion will end the request */
1459		- return 0;
	1784	+ return;
1460	1785	}
1461	1786
1462		- if (refcount_dec_and_test(&req->ref)) {
1463		- if (rq->tag == tag)
1464		- ret = 1;
1465		- nvme_end_request(rq, req->status, req->result);
1466		- }
1467		-
1468		- return ret;
	1787	+ nvme_rdma_end_request(req);
1469	1788	}
1470	1789
1471		-static int __nvme_rdma_recv_done(struct ib_cq cq, struct ib_wc wc, int tag)
	1790	+static void nvme_rdma_recv_done(struct ib_cq cq, struct ib_wc wc)
1472	1791	{
1473	1792	struct nvme_rdma_qe *qe =
1474	1793	container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1475		- struct nvme_rdma_queue *queue = cq->cq_context;
	1794	+ struct nvme_rdma_queue *queue = wc->qp->qp_context;
1476	1795	struct ib_device *ibdev = queue->device->dev;
1477	1796	struct nvme_completion *cqe = qe->data;
1478	1797	const size_t len = sizeof(struct nvme_completion);
1479		- int ret = 0;
1480	1798
1481	1799	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1482	1800	nvme_rdma_wr_error(cq, wc, "RECV");
1483		- return 0;
	1801	+ return;
	1802	+ }
	1803	+
	1804	+ /* sanity checking for received data length */
	1805	+ if (unlikely(wc->byte_len < len)) {
	1806	+ dev_err(queue->ctrl->ctrl.device,
	1807	+ "Unexpected nvme completion length(%d)\n", wc->byte_len);
	1808	+ nvme_rdma_error_recovery(queue->ctrl);
	1809	+ return;
1484	1810	}
1485	1811
1486	1812	ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
..	..	@@ -1490,21 +1816,15 @@
1490	1816	* aborts. We don't even bother to allocate a struct request
1491	1817	* for them but rather special case them here.
1492	1818	*/
1493		- if (unlikely(nvme_rdma_queue_idx(queue) == 0 &&
1494		- cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH))
	1819	+ if (unlikely(nvme_is_aen_req(nvme_rdma_queue_idx(queue),
	1820	+ cqe->command_id)))
1495	1821	nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
1496	1822	&cqe->result);
1497	1823	else
1498		- ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag);
	1824	+ nvme_rdma_process_nvme_rsp(queue, cqe, wc);
1499	1825	ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1500	1826
1501	1827	nvme_rdma_post_recv(queue, qe);
1502		- return ret;
1503		-}
1504		-
1505		-static void nvme_rdma_recv_done(struct ib_cq cq, struct ib_wc wc)
1506		-{
1507		- __nvme_rdma_recv_done(cq, wc, -1);
1508	1828	}
1509	1829
1510	1830	static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
..	..	@@ -1514,14 +1834,10 @@
1514	1834	for (i = 0; i < queue->queue_size; i++) {
1515	1835	ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);
1516	1836	if (ret)
1517		- goto out_destroy_queue_ib;
	1837	+ return ret;
1518	1838	}
1519	1839
1520	1840	return 0;
1521		-
1522		-out_destroy_queue_ib:
1523		- nvme_rdma_destroy_queue_ib(queue);
1524		- return ret;
1525	1841	}
1526	1842
1527	1843	static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
..	..	@@ -1552,16 +1868,18 @@
1552	1868
1553	1869	static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)
1554	1870	{
	1871	+ struct nvme_ctrl *ctrl = &queue->ctrl->ctrl;
1555	1872	int ret;
1556	1873
1557	1874	ret = nvme_rdma_create_queue_ib(queue);
1558	1875	if (ret)
1559	1876	return ret;
1560	1877
	1878	+ if (ctrl->opts->tos >= 0)
	1879	+ rdma_set_service_type(queue->cm_id, ctrl->opts->tos);
1561	1880	ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CONNECT_TIMEOUT_MS);
1562	1881	if (ret) {
1563		- dev_err(queue->ctrl->ctrl.device,
1564		- "rdma_resolve_route failed (%d).\n",
	1882	+ dev_err(ctrl->device, "rdma_resolve_route failed (%d).\n",
1565	1883	queue->cm_error);
1566	1884	goto out_destroy_queue;
1567	1885	}
..	..	@@ -1609,18 +1927,14 @@
1609	1927	priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);
1610	1928	}
1611	1929
1612		- ret = rdma_connect(queue->cm_id, &param);
	1930	+ ret = rdma_connect_locked(queue->cm_id, &param);
1613	1931	if (ret) {
1614	1932	dev_err(ctrl->ctrl.device,
1615		- "rdma_connect failed (%d).\n", ret);
1616		- goto out_destroy_queue_ib;
	1933	+ "rdma_connect_locked failed (%d).\n", ret);
	1934	+ return ret;
1617	1935	}
1618	1936
1619	1937	return 0;
1620		-
1621		-out_destroy_queue_ib:
1622		- nvme_rdma_destroy_queue_ib(queue);
1623		- return ret;
1624	1938	}
1625	1939
1626	1940	static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
..	..	@@ -1651,8 +1965,6 @@
1651	1965	case RDMA_CM_EVENT_ROUTE_ERROR:
1652	1966	case RDMA_CM_EVENT_CONNECT_ERROR:
1653	1967	case RDMA_CM_EVENT_UNREACHABLE:
1654		- nvme_rdma_destroy_queue_ib(queue);
1655		- /* fall through */
1656	1968	case RDMA_CM_EVENT_ADDR_ERROR:
1657	1969	dev_dbg(queue->ctrl->ctrl.device,
1658	1970	"CM error event %d\n", ev->event);
..	..	@@ -1683,6 +1995,18 @@
1683	1995	return 0;
1684	1996	}
1685	1997
	1998	+static void nvme_rdma_complete_timed_out(struct request *rq)
	1999	+{
	2000	+ struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
	2001	+ struct nvme_rdma_queue *queue = req->queue;
	2002	+
	2003	+ nvme_rdma_stop_queue(queue);
	2004	+ if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) {
	2005	+ nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;
	2006	+ blk_mq_complete_request(rq);
	2007	+ }
	2008	+}
	2009	+
1686	2010	static enum blk_eh_timer_return
1687	2011	nvme_rdma_timeout(struct request *rq, bool reserved)
1688	2012	{
..	..	@@ -1695,19 +2019,27 @@
1695	2019
1696	2020	if (ctrl->ctrl.state != NVME_CTRL_LIVE) {
1697	2021	/*
1698		- * Teardown immediately if controller times out while starting
1699		- * or we are already started error recovery. all outstanding
1700		- * requests are completed on shutdown, so we return BLK_EH_DONE.
	2022	+ * If we are resetting, connecting or deleting we should
	2023	+ * complete immediately because we may block controller
	2024	+ * teardown or setup sequence
	2025	+ * - ctrl disable/shutdown fabrics requests
	2026	+ * - connect requests
	2027	+ * - initialization admin requests
	2028	+ * - I/O requests that entered after unquiescing and
	2029	+ * the controller stopped responding
	2030	+ *
	2031	+ * All other requests should be cancelled by the error
	2032	+ * recovery work, so it's fine that we fail it here.
1701	2033	*/
1702		- flush_work(&ctrl->err_work);
1703		- nvme_rdma_teardown_io_queues(ctrl, false);
1704		- nvme_rdma_teardown_admin_queue(ctrl, false);
	2034	+ nvme_rdma_complete_timed_out(rq);
1705	2035	return BLK_EH_DONE;
1706	2036	}
1707	2037
1708		- dev_warn(ctrl->ctrl.device, "starting error recovery\n");
	2038	+ /*
	2039	+ * LIVE state should trigger the normal error recovery which will
	2040	+ * handle completing this request.
	2041	+ */
1709	2042	nvme_rdma_error_recovery(ctrl);
1710		-
1711	2043	return BLK_EH_RESET_TIMER;
1712	2044	}
1713	2045
..	..	@@ -1731,20 +2063,36 @@
1731	2063	return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);
1732	2064
1733	2065	dev = queue->device->dev;
	2066	+
	2067	+ req->sqe.dma = ib_dma_map_single(dev, req->sqe.data,
	2068	+ sizeof(struct nvme_command),
	2069	+ DMA_TO_DEVICE);
	2070	+ err = ib_dma_mapping_error(dev, req->sqe.dma);
	2071	+ if (unlikely(err))
	2072	+ return BLK_STS_RESOURCE;
	2073	+
1734	2074	ib_dma_sync_single_for_cpu(dev, sqe->dma,
1735	2075	sizeof(struct nvme_command), DMA_TO_DEVICE);
1736	2076
1737	2077	ret = nvme_setup_cmd(ns, rq, c);
1738	2078	if (ret)
1739		- return ret;
	2079	+ goto unmap_qe;
1740	2080
1741	2081	blk_mq_start_request(rq);
	2082	+
	2083	+ if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
	2084	+ queue->pi_support &&
	2085	+ (c->common.opcode == nvme_cmd_write \|\|
	2086	+ c->common.opcode == nvme_cmd_read) &&
	2087	+ nvme_ns_has_pi(ns))
	2088	+ req->use_sig_mr = true;
	2089	+ else
	2090	+ req->use_sig_mr = false;
1742	2091
1743	2092	err = nvme_rdma_map_data(queue, rq, c);
1744	2093	if (unlikely(err < 0)) {
1745	2094	dev_err(queue->ctrl->ctrl.device,
1746	2095	"Failed to map data (%d)\n", err);
1747		- nvme_cleanup_cmd(rq);
1748	2096	goto err;
1749	2097	}
1750	2098
..	..	@@ -1755,52 +2103,123 @@
1755	2103
1756	2104	err = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
1757	2105	req->mr ? &req->reg_wr.wr : NULL);
1758		- if (unlikely(err)) {
1759		- nvme_rdma_unmap_data(queue, rq);
1760		- goto err;
1761		- }
	2106	+ if (unlikely(err))
	2107	+ goto err_unmap;
1762	2108
1763	2109	return BLK_STS_OK;
	2110	+
	2111	+err_unmap:
	2112	+ nvme_rdma_unmap_data(queue, rq);
1764	2113	err:
1765	2114	if (err == -ENOMEM \|\| err == -EAGAIN)
1766		- return BLK_STS_RESOURCE;
1767		- return BLK_STS_IOERR;
	2115	+ ret = BLK_STS_RESOURCE;
	2116	+ else
	2117	+ ret = BLK_STS_IOERR;
	2118	+ nvme_cleanup_cmd(rq);
	2119	+unmap_qe:
	2120	+ ib_dma_unmap_single(dev, req->sqe.dma, sizeof(struct nvme_command),
	2121	+ DMA_TO_DEVICE);
	2122	+ return ret;
1768	2123	}
1769	2124
1770		-static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
	2125	+static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx)
1771	2126	{
1772	2127	struct nvme_rdma_queue *queue = hctx->driver_data;
1773		- struct ib_cq *cq = queue->ib_cq;
1774		- struct ib_wc wc;
1775		- int found = 0;
1776	2128
1777		- while (ib_poll_cq(cq, 1, &wc) > 0) {
1778		- struct ib_cqe *cqe = wc.wr_cqe;
	2129	+ return ib_process_cq_direct(queue->ib_cq, -1);
	2130	+}
1779	2131
1780		- if (cqe) {
1781		- if (cqe->done == nvme_rdma_recv_done)
1782		- found \|= __nvme_rdma_recv_done(cq, &wc, tag);
1783		- else
1784		- cqe->done(cq, &wc);
1785		- }
	2132	+static void nvme_rdma_check_pi_status(struct nvme_rdma_request *req)
	2133	+{
	2134	+ struct request *rq = blk_mq_rq_from_pdu(req);
	2135	+ struct ib_mr_status mr_status;
	2136	+ int ret;
	2137	+
	2138	+ ret = ib_check_mr_status(req->mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
	2139	+ if (ret) {
	2140	+ pr_err("ib_check_mr_status failed, ret %d\n", ret);
	2141	+ nvme_req(rq)->status = NVME_SC_INVALID_PI;
	2142	+ return;
1786	2143	}
1787	2144
1788		- return found;
	2145	+ if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
	2146	+ switch (mr_status.sig_err.err_type) {
	2147	+ case IB_SIG_BAD_GUARD:
	2148	+ nvme_req(rq)->status = NVME_SC_GUARD_CHECK;
	2149	+ break;
	2150	+ case IB_SIG_BAD_REFTAG:
	2151	+ nvme_req(rq)->status = NVME_SC_REFTAG_CHECK;
	2152	+ break;
	2153	+ case IB_SIG_BAD_APPTAG:
	2154	+ nvme_req(rq)->status = NVME_SC_APPTAG_CHECK;
	2155	+ break;
	2156	+ }
	2157	+ pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n",
	2158	+ mr_status.sig_err.err_type, mr_status.sig_err.expected,
	2159	+ mr_status.sig_err.actual);
	2160	+ }
1789	2161	}
1790	2162
1791	2163	static void nvme_rdma_complete_rq(struct request *rq)
1792	2164	{
1793	2165	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
	2166	+ struct nvme_rdma_queue *queue = req->queue;
	2167	+ struct ib_device *ibdev = queue->device->dev;
1794	2168
1795		- nvme_rdma_unmap_data(req->queue, rq);
	2169	+ if (req->use_sig_mr)
	2170	+ nvme_rdma_check_pi_status(req);
	2171	+
	2172	+ nvme_rdma_unmap_data(queue, rq);
	2173	+ ib_dma_unmap_single(ibdev, req->sqe.dma, sizeof(struct nvme_command),
	2174	+ DMA_TO_DEVICE);
1796	2175	nvme_complete_rq(rq);
1797	2176	}
1798	2177
1799	2178	static int nvme_rdma_map_queues(struct blk_mq_tag_set *set)
1800	2179	{
1801	2180	struct nvme_rdma_ctrl *ctrl = set->driver_data;
	2181	+ struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
1802	2182
1803		- return blk_mq_rdma_map_queues(set, ctrl->device->dev, 0);
	2183	+ if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
	2184	+ /* separate read/write queues */
	2185	+ set->map[HCTX_TYPE_DEFAULT].nr_queues =
	2186	+ ctrl->io_queues[HCTX_TYPE_DEFAULT];
	2187	+ set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
	2188	+ set->map[HCTX_TYPE_READ].nr_queues =
	2189	+ ctrl->io_queues[HCTX_TYPE_READ];
	2190	+ set->map[HCTX_TYPE_READ].queue_offset =
	2191	+ ctrl->io_queues[HCTX_TYPE_DEFAULT];
	2192	+ } else {
	2193	+ /* shared read/write queues */
	2194	+ set->map[HCTX_TYPE_DEFAULT].nr_queues =
	2195	+ ctrl->io_queues[HCTX_TYPE_DEFAULT];
	2196	+ set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
	2197	+ set->map[HCTX_TYPE_READ].nr_queues =
	2198	+ ctrl->io_queues[HCTX_TYPE_DEFAULT];
	2199	+ set->map[HCTX_TYPE_READ].queue_offset = 0;
	2200	+ }
	2201	+ blk_mq_rdma_map_queues(&set->map[HCTX_TYPE_DEFAULT],
	2202	+ ctrl->device->dev, 0);
	2203	+ blk_mq_rdma_map_queues(&set->map[HCTX_TYPE_READ],
	2204	+ ctrl->device->dev, 0);
	2205	+
	2206	+ if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
	2207	+ /* map dedicated poll queues only if we have queues left */
	2208	+ set->map[HCTX_TYPE_POLL].nr_queues =
	2209	+ ctrl->io_queues[HCTX_TYPE_POLL];
	2210	+ set->map[HCTX_TYPE_POLL].queue_offset =
	2211	+ ctrl->io_queues[HCTX_TYPE_DEFAULT] +
	2212	+ ctrl->io_queues[HCTX_TYPE_READ];
	2213	+ blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
	2214	+ }
	2215	+
	2216	+ dev_info(ctrl->ctrl.device,
	2217	+ "mapped %d/%d/%d default/read/poll queues.\n",
	2218	+ ctrl->io_queues[HCTX_TYPE_DEFAULT],
	2219	+ ctrl->io_queues[HCTX_TYPE_READ],
	2220	+ ctrl->io_queues[HCTX_TYPE_POLL]);
	2221	+
	2222	+ return 0;
1804	2223	}
1805	2224
1806	2225	static const struct blk_mq_ops nvme_rdma_mq_ops = {
..	..	@@ -1809,9 +2228,9 @@
1809	2228	.init_request = nvme_rdma_init_request,
1810	2229	.exit_request = nvme_rdma_exit_request,
1811	2230	.init_hctx = nvme_rdma_init_hctx,
1812		- .poll = nvme_rdma_poll,
1813	2231	.timeout = nvme_rdma_timeout,
1814	2232	.map_queues = nvme_rdma_map_queues,
	2233	+ .poll = nvme_rdma_poll,
1815	2234	};
1816	2235
1817	2236	static const struct blk_mq_ops nvme_rdma_admin_mq_ops = {
..	..	@@ -1826,10 +2245,11 @@
1826	2245	static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1827	2246	{
1828	2247	nvme_rdma_teardown_io_queues(ctrl, shutdown);
	2248	+ blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
1829	2249	if (shutdown)
1830	2250	nvme_shutdown_ctrl(&ctrl->ctrl);
1831	2251	else
1832		- nvme_disable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
	2252	+ nvme_disable_ctrl(&ctrl->ctrl);
1833	2253	nvme_rdma_teardown_admin_queue(ctrl, shutdown);
1834	2254	}
1835	2255
..	..	@@ -1865,7 +2285,7 @@
1865	2285	static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
1866	2286	.name = "rdma",
1867	2287	.module = THIS_MODULE,
1868		- .flags = NVME_F_FABRICS,
	2288	+ .flags = NVME_F_FABRICS \| NVME_F_METADATA_SUPPORTED,
1869	2289	.reg_read32 = nvmf_reg_read32,
1870	2290	.reg_read64 = nvmf_reg_read64,
1871	2291	.reg_write32 = nvmf_reg_write32,
..	..	@@ -1875,54 +2295,6 @@
1875	2295	.get_address = nvmf_get_address,
1876	2296	.stop_ctrl = nvme_rdma_stop_ctrl,
1877	2297	};
1878		-
1879		-static inline bool
1880		-__nvme_rdma_options_match(struct nvme_rdma_ctrl *ctrl,
1881		- struct nvmf_ctrl_options *opts)
1882		-{
1883		- char *stdport = __stringify(NVME_RDMA_IP_PORT);
1884		-
1885		-
1886		- if (!nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts) \|\|
1887		- strcmp(opts->traddr, ctrl->ctrl.opts->traddr))
1888		- return false;
1889		-
1890		- if (opts->mask & NVMF_OPT_TRSVCID &&
1891		- ctrl->ctrl.opts->mask & NVMF_OPT_TRSVCID) {
1892		- if (strcmp(opts->trsvcid, ctrl->ctrl.opts->trsvcid))
1893		- return false;
1894		- } else if (opts->mask & NVMF_OPT_TRSVCID) {
1895		- if (strcmp(opts->trsvcid, stdport))
1896		- return false;
1897		- } else if (ctrl->ctrl.opts->mask & NVMF_OPT_TRSVCID) {
1898		- if (strcmp(stdport, ctrl->ctrl.opts->trsvcid))
1899		- return false;
1900		- }
1901		- /* else, it's a match as both have stdport. Fall to next checks */
1902		-
1903		- /*
1904		- * checking the local address is rough. In most cases, one
1905		- * is not specified and the host port is selected by the stack.
1906		- *
1907		- * Assume no match if:
1908		- * local address is specified and address is not the same
1909		- * local address is not specified but remote is, or vice versa
1910		- * (admin using specific host_traddr when it matters).
1911		- */
1912		- if (opts->mask & NVMF_OPT_HOST_TRADDR &&
1913		- ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR) {
1914		- if (strcmp(opts->host_traddr, ctrl->ctrl.opts->host_traddr))
1915		- return false;
1916		- } else if (opts->mask & NVMF_OPT_HOST_TRADDR \|\|
1917		- ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
1918		- return false;
1919		- /*
1920		- * if neither controller had an host port specified, assume it's
1921		- * a match as everything else matched.
1922		- */
1923		-
1924		- return true;
1925		-}
1926	2298
1927	2299	/*
1928	2300	* Fails a connection request if it matches an existing controller
..	..	@@ -1944,7 +2316,7 @@
1944	2316
1945	2317	mutex_lock(&nvme_rdma_ctrl_mutex);
1946	2318	list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
1947		- found = __nvme_rdma_options_match(ctrl, opts);
	2319	+ found = nvmf_ip_options_match(&ctrl->ctrl, opts);
1948	2320	if (found)
1949	2321	break;
1950	2322	}
..	..	@@ -1959,24 +2331,28 @@
1959	2331	struct nvme_rdma_ctrl *ctrl;
1960	2332	int ret;
1961	2333	bool changed;
1962		- char *port;
1963	2334
1964	2335	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1965	2336	if (!ctrl)
1966	2337	return ERR_PTR(-ENOMEM);
1967	2338	ctrl->ctrl.opts = opts;
1968	2339	INIT_LIST_HEAD(&ctrl->list);
1969		- mutex_init(&ctrl->teardown_lock);
1970	2340
1971		- if (opts->mask & NVMF_OPT_TRSVCID)
1972		- port = opts->trsvcid;
1973		- else
1974		- port = __stringify(NVME_RDMA_IP_PORT);
	2341	+ if (!(opts->mask & NVMF_OPT_TRSVCID)) {
	2342	+ opts->trsvcid =
	2343	+ kstrdup(__stringify(NVME_RDMA_IP_PORT), GFP_KERNEL);
	2344	+ if (!opts->trsvcid) {
	2345	+ ret = -ENOMEM;
	2346	+ goto out_free_ctrl;
	2347	+ }
	2348	+ opts->mask \|= NVMF_OPT_TRSVCID;
	2349	+ }
1975	2350
1976	2351	ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
1977		- opts->traddr, port, &ctrl->addr);
	2352	+ opts->traddr, opts->trsvcid, &ctrl->addr);
1978	2353	if (ret) {
1979		- pr_err("malformed address passed: %s:%s\n", opts->traddr, port);
	2354	+ pr_err("malformed address passed: %s:%s\n",
	2355	+ opts->traddr, opts->trsvcid);
1980	2356	goto out_free_ctrl;
1981	2357	}
1982	2358
..	..	@@ -2000,7 +2376,8 @@
2000	2376	INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
2001	2377	INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work);
2002	2378
2003		- ctrl->ctrl.queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
	2379	+ ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
	2380	+ opts->nr_poll_queues + 1;
2004	2381	ctrl->ctrl.sqsize = opts->queue_size - 1;
2005	2382	ctrl->ctrl.kato = opts->kato;
2006	2383
..	..	@@ -2024,8 +2401,6 @@
2024	2401
2025	2402	dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n",
2026	2403	ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2027		-
2028		- nvme_get_ctrl(&ctrl->ctrl);
2029	2404
2030	2405	mutex_lock(&nvme_rdma_ctrl_mutex);
2031	2406	list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
..	..	@@ -2051,7 +2426,9 @@
2051	2426	.module = THIS_MODULE,
2052	2427	.required_opts = NVMF_OPT_TRADDR,
2053	2428	.allowed_opts = NVMF_OPT_TRSVCID \| NVMF_OPT_RECONNECT_DELAY \|
2054		- NVMF_OPT_HOST_TRADDR \| NVMF_OPT_CTRL_LOSS_TMO,
	2429	+ NVMF_OPT_HOST_TRADDR \| NVMF_OPT_CTRL_LOSS_TMO \|
	2430	+ NVMF_OPT_NR_WRITE_QUEUES \| NVMF_OPT_NR_POLL_QUEUES \|
	2431	+ NVMF_OPT_TOS,
2055	2432	.create_ctrl = nvme_rdma_create_ctrl,
2056	2433	};
2057	2434
..	..	@@ -2111,8 +2488,16 @@
2111	2488
2112	2489	static void __exit nvme_rdma_cleanup_module(void)
2113	2490	{
	2491	+ struct nvme_rdma_ctrl *ctrl;
	2492	+
2114	2493	nvmf_unregister_transport(&nvme_rdma_transport);
2115	2494	ib_unregister_client(&nvme_rdma_ib_client);
	2495	+
	2496	+ mutex_lock(&nvme_rdma_ctrl_mutex);
	2497	+ list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list)
	2498	+ nvme_delete_ctrl(&ctrl->ctrl);
	2499	+ mutex_unlock(&nvme_rdma_ctrl_mutex);
	2500	+ flush_workqueue(nvme_delete_wq);
2116	2501	}
2117	2502
2118	2503	module_init(nvme_rdma_init_module);