~hc/RK356X_SDK_RELEASE.git

..	..	@@ -51,8 +51,6 @@
51	51
52	52	/* Name of this kernel module. */
53	53	#define DRV_NAME "ib_srpt"
54		-#define DRV_VERSION "2.0.0"
55		-#define DRV_RELDATE "2011-02-14"
56	54
57	55	#define SRPT_ID_STRING "Linux SRP target"
58	56
..	..	@@ -60,8 +58,7 @@
60	58	#define pr_fmt(fmt) DRV_NAME " " fmt
61	59
62	60	MODULE_AUTHOR("Vu Pham and Bart Van Assche");
63		-MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
64		- "v" DRV_VERSION " (" DRV_RELDATE ")");
	61	+MODULE_DESCRIPTION("SCSI RDMA Protocol target driver");
65	62	MODULE_LICENSE("Dual BSD/GPL");
66	63
67	64	/*
..	..	@@ -84,13 +81,12 @@
84	81
85	82	static int srpt_get_u64_x(char buffer, const struct kernel_param kp)
86	83	{
87		- return sprintf(buffer, "0x%016llx", (u64 )kp->arg);
	84	+ return sprintf(buffer, "0x%016llx\n", (u64 )kp->arg);
88	85	}
89	86	module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
90	87	0444);
91	88	MODULE_PARM_DESC(srpt_service_guid,
92		- "Using this value for ioc_guid, id_ext, and cm_listen_id"
93		- " instead of using the node_guid of the first HCA.");
	89	+ "Using this value for ioc_guid, id_ext, and cm_listen_id instead of using the node_guid of the first HCA.");
94	90
95	91	static struct ib_client srpt_client;
96	92	/* Protects both rdma_cm_port and rdma_cm_id. */
..	..	@@ -139,16 +135,13 @@
139	135	static void srpt_event_handler(struct ib_event_handler *handler,
140	136	struct ib_event *event)
141	137	{
142		- struct srpt_device *sdev;
	138	+ struct srpt_device *sdev =
	139	+ container_of(handler, struct srpt_device, event_handler);
143	140	struct srpt_port *sport;
144	141	u8 port_num;
145	142
146		- sdev = ib_get_client_data(event->device, &srpt_client);
147		- if (!sdev \|\| sdev->device != event->device)
148		- return;
149		-
150	143	pr_debug("ASYNC event= %d on device= %s\n", event->event,
151		- sdev->device->name);
	144	+ dev_name(&sdev->device->dev));
152	145
153	146	switch (event->event) {
154	147	case IB_EVENT_PORT_ERR:
..	..	@@ -221,8 +214,9 @@
221	214	*/
222	215	static void srpt_qp_event(struct ib_event event, struct srpt_rdma_ch ch)
223	216	{
224		- pr_debug("QP event %d on ch=%p sess_name=%s state=%d\n",
225		- event->event, ch, ch->sess_name, ch->state);
	217	+ pr_debug("QP event %d on ch=%p sess_name=%s-%d state=%s\n",
	218	+ event->event, ch, ch->sess_name, ch->qp->qp_num,
	219	+ get_ch_state_name(ch->state));
226	220
227	221	switch (event->event) {
228	222	case IB_EVENT_COMM_EST:
..	..	@@ -462,7 +456,7 @@
462	456	static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
463	457	struct ib_mad_send_wc *mad_wc)
464	458	{
465		- rdma_destroy_ah(mad_wc->send_buf->ah);
	459	+ rdma_destroy_ah(mad_wc->send_buf->ah, RDMA_DESTROY_AH_SLEEPABLE);
466	460	ib_free_send_mad(mad_wc->send_buf);
467	461	}
468	462
..	..	@@ -529,7 +523,7 @@
529	523	ib_free_send_mad(rsp);
530	524
531	525	err_rsp:
532		- rdma_destroy_ah(ah);
	526	+ rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
533	527	err:
534	528	ib_free_recv_mad(mad_wc);
535	529	}
..	..	@@ -555,38 +549,43 @@
555	549	*/
556	550	static int srpt_refresh_port(struct srpt_port *sport)
557	551	{
	552	+ struct ib_mad_agent *mad_agent;
558	553	struct ib_mad_reg_req reg_req;
559	554	struct ib_port_modify port_modify;
560	555	struct ib_port_attr port_attr;
561	556	int ret;
562	557
563		- memset(&port_modify, 0, sizeof(port_modify));
564		- port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
565		- port_modify.clr_port_cap_mask = 0;
566		-
567		- ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
568		- if (ret)
569		- goto err_mod_port;
570		-
571	558	ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
572	559	if (ret)
573		- goto err_query_port;
	560	+ return ret;
574	561
575	562	sport->sm_lid = port_attr.sm_lid;
576	563	sport->lid = port_attr.lid;
577	564
578	565	ret = rdma_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
579	566	if (ret)
580		- goto err_query_port;
	567	+ return ret;
581	568
582		- sport->port_guid_wwn.priv = sport;
583		- srpt_format_guid(sport->port_guid, sizeof(sport->port_guid),
	569	+ srpt_format_guid(sport->guid_name, ARRAY_SIZE(sport->guid_name),
584	570	&sport->gid.global.interface_id);
585		- sport->port_gid_wwn.priv = sport;
586		- snprintf(sport->port_gid, sizeof(sport->port_gid),
	571	+ snprintf(sport->gid_name, ARRAY_SIZE(sport->gid_name),
587	572	"0x%016llx%016llx",
588	573	be64_to_cpu(sport->gid.global.subnet_prefix),
589	574	be64_to_cpu(sport->gid.global.interface_id));
	575	+
	576	+ if (rdma_protocol_iwarp(sport->sdev->device, sport->port))
	577	+ return 0;
	578	+
	579	+ memset(&port_modify, 0, sizeof(port_modify));
	580	+ port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
	581	+ port_modify.clr_port_cap_mask = 0;
	582	+
	583	+ ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
	584	+ if (ret) {
	585	+ pr_warn("%s-%d: enabling device management failed (%d). Note: this is expected if SR-IOV is enabled.\n",
	586	+ dev_name(&sport->sdev->device->dev), sport->port, ret);
	587	+ return 0;
	588	+ }
590	589
591	590	if (!sport->mad_agent) {
592	591	memset(&reg_req, 0, sizeof(reg_req));
..	..	@@ -595,40 +594,39 @@
595	594	set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
596	595	set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
597	596
598		- sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
599		- sport->port,
600		- IB_QPT_GSI,
601		- &reg_req, 0,
602		- srpt_mad_send_handler,
603		- srpt_mad_recv_handler,
604		- sport, 0);
605		- if (IS_ERR(sport->mad_agent)) {
606		- ret = PTR_ERR(sport->mad_agent);
	597	+ mad_agent = ib_register_mad_agent(sport->sdev->device,
	598	+ sport->port,
	599	+ IB_QPT_GSI,
	600	+ &reg_req, 0,
	601	+ srpt_mad_send_handler,
	602	+ srpt_mad_recv_handler,
	603	+ sport, 0);
	604	+ if (IS_ERR(mad_agent)) {
	605	+ pr_err("%s-%d: MAD agent registration failed (%ld). Note: this is expected if SR-IOV is enabled.\n",
	606	+ dev_name(&sport->sdev->device->dev), sport->port,
	607	+ PTR_ERR(mad_agent));
607	608	sport->mad_agent = NULL;
608		- goto err_query_port;
	609	+ memset(&port_modify, 0, sizeof(port_modify));
	610	+ port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
	611	+ ib_modify_port(sport->sdev->device, sport->port, 0,
	612	+ &port_modify);
	613	+ return 0;
609	614	}
	615	+
	616	+ sport->mad_agent = mad_agent;
610	617	}
611	618
612	619	return 0;
613		-
614		-err_query_port:
615		-
616		- port_modify.set_port_cap_mask = 0;
617		- port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
618		- ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
619		-
620		-err_mod_port:
621		-
622		- return ret;
623	620	}
624	621
625	622	/**
626	623	* srpt_unregister_mad_agent - unregister MAD callback functions
627	624	* @sdev: SRPT HCA pointer.
	625	+ * @port_cnt: number of ports with registered MAD
628	626	*
629	627	* Note: It is safe to call this function more than once for the same device.
630	628	*/
631		-static void srpt_unregister_mad_agent(struct srpt_device *sdev)
	629	+static void srpt_unregister_mad_agent(struct srpt_device *sdev, int port_cnt)
632	630	{
633	631	struct ib_port_modify port_modify = {
634	632	.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
..	..	@@ -636,12 +634,11 @@
636	634	struct srpt_port *sport;
637	635	int i;
638	636
639		- for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
	637	+ for (i = 1; i <= port_cnt; i++) {
640	638	sport = &sdev->port[i - 1];
641	639	WARN_ON(sport->port != i);
642		- if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
643		- pr_err("disabling MAD processing failed.\n");
644	640	if (sport->mad_agent) {
	641	+ ib_modify_port(sdev->device, i, 0, &port_modify);
645	642	ib_unregister_mad_agent(sport->mad_agent);
646	643	sport->mad_agent = NULL;
647	644	}
..	..	@@ -652,31 +649,33 @@
652	649	* srpt_alloc_ioctx - allocate a SRPT I/O context structure
653	650	* @sdev: SRPT HCA pointer.
654	651	* @ioctx_size: I/O context size.
655		- * @dma_size: Size of I/O context DMA buffer.
	652	+ * @buf_cache: I/O buffer cache.
656	653	* @dir: DMA data direction.
657	654	*/
658	655	static struct srpt_ioctx srpt_alloc_ioctx(struct srpt_device sdev,
659		- int ioctx_size, int dma_size,
	656	+ int ioctx_size,
	657	+ struct kmem_cache *buf_cache,
660	658	enum dma_data_direction dir)
661	659	{
662	660	struct srpt_ioctx *ioctx;
663	661
664		- ioctx = kmalloc(ioctx_size, GFP_KERNEL);
	662	+ ioctx = kzalloc(ioctx_size, GFP_KERNEL);
665	663	if (!ioctx)
666	664	goto err;
667	665
668		- ioctx->buf = kmalloc(dma_size, GFP_KERNEL);
	666	+ ioctx->buf = kmem_cache_alloc(buf_cache, GFP_KERNEL);
669	667	if (!ioctx->buf)
670	668	goto err_free_ioctx;
671	669
672		- ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);
	670	+ ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
	671	+ kmem_cache_size(buf_cache), dir);
673	672	if (ib_dma_mapping_error(sdev->device, ioctx->dma))
674	673	goto err_free_buf;
675	674
676	675	return ioctx;
677	676
678	677	err_free_buf:
679		- kfree(ioctx->buf);
	678	+ kmem_cache_free(buf_cache, ioctx->buf);
680	679	err_free_ioctx:
681	680	kfree(ioctx);
682	681	err:
..	..	@@ -687,17 +686,19 @@
687	686	* srpt_free_ioctx - free a SRPT I/O context structure
688	687	* @sdev: SRPT HCA pointer.
689	688	* @ioctx: I/O context pointer.
690		- * @dma_size: Size of I/O context DMA buffer.
	689	+ * @buf_cache: I/O buffer cache.
691	690	* @dir: DMA data direction.
692	691	*/
693	692	static void srpt_free_ioctx(struct srpt_device sdev, struct srpt_ioctx ioctx,
694		- int dma_size, enum dma_data_direction dir)
	693	+ struct kmem_cache *buf_cache,
	694	+ enum dma_data_direction dir)
695	695	{
696	696	if (!ioctx)
697	697	return;
698	698
699		- ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);
700		- kfree(ioctx->buf);
	699	+ ib_dma_unmap_single(sdev->device, ioctx->dma,
	700	+ kmem_cache_size(buf_cache), dir);
	701	+ kmem_cache_free(buf_cache, ioctx->buf);
701	702	kfree(ioctx);
702	703	}
703	704
..	..	@@ -706,33 +707,38 @@
706	707	* @sdev: Device to allocate the I/O context ring for.
707	708	* @ring_size: Number of elements in the I/O context ring.
708	709	* @ioctx_size: I/O context size.
709		- * @dma_size: DMA buffer size.
	710	+ * @buf_cache: I/O buffer cache.
	711	+ * @alignment_offset: Offset in each ring buffer at which the SRP information
	712	+ * unit starts.
710	713	* @dir: DMA data direction.
711	714	*/
712	715	static struct srpt_ioctx *srpt_alloc_ioctx_ring(struct srpt_device sdev,
713	716	int ring_size, int ioctx_size,
714		- int dma_size, enum dma_data_direction dir)
	717	+ struct kmem_cache *buf_cache,
	718	+ int alignment_offset,
	719	+ enum dma_data_direction dir)
715	720	{
716	721	struct srpt_ioctx **ring;
717	722	int i;
718	723
719		- WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx)
720		- && ioctx_size != sizeof(struct srpt_send_ioctx));
	724	+ WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx) &&
	725	+ ioctx_size != sizeof(struct srpt_send_ioctx));
721	726
722	727	ring = kvmalloc_array(ring_size, sizeof(ring[0]), GFP_KERNEL);
723	728	if (!ring)
724	729	goto out;
725	730	for (i = 0; i < ring_size; ++i) {
726		- ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir);
	731	+ ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, buf_cache, dir);
727	732	if (!ring[i])
728	733	goto err;
729	734	ring[i]->index = i;
	735	+ ring[i]->offset = alignment_offset;
730	736	}
731	737	goto out;
732	738
733	739	err:
734	740	while (--i >= 0)
735		- srpt_free_ioctx(sdev, ring[i], dma_size, dir);
	741	+ srpt_free_ioctx(sdev, ring[i], buf_cache, dir);
736	742	kvfree(ring);
737	743	ring = NULL;
738	744	out:
..	..	@@ -744,12 +750,13 @@
744	750	* @ioctx_ring: I/O context ring to be freed.
745	751	* @sdev: SRPT HCA pointer.
746	752	* @ring_size: Number of ring elements.
747		- * @dma_size: Size of I/O context DMA buffer.
	753	+ * @buf_cache: I/O buffer cache.
748	754	* @dir: DMA data direction.
749	755	*/
750	756	static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
751	757	struct srpt_device *sdev, int ring_size,
752		- int dma_size, enum dma_data_direction dir)
	758	+ struct kmem_cache *buf_cache,
	759	+ enum dma_data_direction dir)
753	760	{
754	761	int i;
755	762
..	..	@@ -757,7 +764,7 @@
757	764	return;
758	765
759	766	for (i = 0; i < ring_size; ++i)
760		- srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir);
	767	+ srpt_free_ioctx(sdev, ioctx_ring[i], buf_cache, dir);
761	768	kvfree(ioctx_ring);
762	769	}
763	770
..	..	@@ -819,7 +826,7 @@
819	826	struct ib_recv_wr wr;
820	827
821	828	BUG_ON(!sdev);
822		- list.addr = ioctx->ioctx.dma;
	829	+ list.addr = ioctx->ioctx.dma + ioctx->ioctx.offset;
823	830	list.length = srp_max_req_size;
824	831	list.lkey = sdev->lkey;
825	832
..	..	@@ -863,7 +870,7 @@
863	870
864	871	static void srpt_zerolength_write_done(struct ib_cq cq, struct ib_wc wc)
865	872	{
866		- struct srpt_rdma_ch *ch = cq->cq_context;
	873	+ struct srpt_rdma_ch *ch = wc->qp->qp_context;
867	874
868	875	pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num,
869	876	wc->status);
..	..	@@ -985,23 +992,28 @@
985	992
986	993	/**
987	994	* srpt_get_desc_tbl - parse the data descriptors of a SRP_CMD request
988		- * @ioctx: Pointer to the I/O context associated with the request.
	995	+ * @recv_ioctx: I/O context associated with the received command @srp_cmd.
	996	+ * @ioctx: I/O context that will be used for responding to the initiator.
989	997	* @srp_cmd: Pointer to the SRP_CMD request data.
990	998	* @dir: Pointer to the variable to which the transfer direction will be
991	999	* written.
992		- * @sg: [out] scatterlist allocated for the parsed SRP_CMD.
	1000	+ * @sg: [out] scatterlist for the parsed SRP_CMD.
993	1001	* @sg_cnt: [out] length of @sg.
994	1002	* @data_len: Pointer to the variable to which the total data length of all
995	1003	* descriptors in the SRP_CMD request will be written.
	1004	+ * @imm_data_offset: [in] Offset in SRP_CMD requests at which immediate data
	1005	+ * starts.
996	1006	*
997	1007	* This function initializes ioctx->nrbuf and ioctx->r_bufs.
998	1008	*
999	1009	* Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors;
1000	1010	* -ENOMEM when memory allocation fails and zero upon success.
1001	1011	*/
1002		-static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,
	1012	+static int srpt_get_desc_tbl(struct srpt_recv_ioctx *recv_ioctx,
	1013	+ struct srpt_send_ioctx *ioctx,
1003	1014	struct srp_cmd srp_cmd, enum dma_data_direction dir,
1004		- struct scatterlist *sg, unsigned sg_cnt, u64 *data_len)
	1015	+ struct scatterlist *sg, unsigned int sg_cnt, u64 *data_len,
	1016	+ u16 imm_data_offset)
1005	1017	{
1006	1018	BUG_ON(!dir);
1007	1019	BUG_ON(!data_len);
..	..	@@ -1025,7 +1037,7 @@
1025	1037
1026	1038	if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) \|\|
1027	1039	((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
1028		- struct srp_direct_buf *db = srpt_get_desc_buf(srp_cmd);
	1040	+ struct srp_direct_buf *db = srpt_get_desc_buf(srp_cmd);
1029	1041
1030	1042	*data_len = be32_to_cpu(db->len);
1031	1043	return srpt_alloc_rw_ctxs(ioctx, db, 1, sg, sg_cnt);
..	..	@@ -1037,8 +1049,7 @@
1037	1049
1038	1050	if (nbufs >
1039	1051	(srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
1040		- pr_err("received unsupported SRP_CMD request"
1041		- " type (%u out + %u in != %u / %zu)\n",
	1052	+ pr_err("received unsupported SRP_CMD request type (%u out + %u in != %u / %zu)\n",
1042	1053	srp_cmd->data_out_desc_cnt,
1043	1054	srp_cmd->data_in_desc_cnt,
1044	1055	be32_to_cpu(idb->table_desc.len),
..	..	@@ -1049,6 +1060,40 @@
1049	1060	*data_len = be32_to_cpu(idb->len);
1050	1061	return srpt_alloc_rw_ctxs(ioctx, idb->desc_list, nbufs,
1051	1062	sg, sg_cnt);
	1063	+ } else if ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_IMM) {
	1064	+ struct srp_imm_buf *imm_buf = srpt_get_desc_buf(srp_cmd);
	1065	+ void data = (void )srp_cmd + imm_data_offset;
	1066	+ uint32_t len = be32_to_cpu(imm_buf->len);
	1067	+ uint32_t req_size = imm_data_offset + len;
	1068	+
	1069	+ if (req_size > srp_max_req_size) {
	1070	+ pr_err("Immediate data (length %d + %d) exceeds request size %d\n",
	1071	+ imm_data_offset, len, srp_max_req_size);
	1072	+ return -EINVAL;
	1073	+ }
	1074	+ if (recv_ioctx->byte_len < req_size) {
	1075	+ pr_err("Received too few data - %d < %d\n",
	1076	+ recv_ioctx->byte_len, req_size);
	1077	+ return -EIO;
	1078	+ }
	1079	+ /*
	1080	+ * The immediate data buffer descriptor must occur before the
	1081	+ * immediate data itself.
	1082	+ */
	1083	+ if ((void )(imm_buf + 1) > (void )data) {
	1084	+ pr_err("Received invalid write request\n");
	1085	+ return -EINVAL;
	1086	+ }
	1087	+ *data_len = len;
	1088	+ ioctx->recv_ioctx = recv_ioctx;
	1089	+ if ((uintptr_t)data & 511) {
	1090	+ pr_warn_once("Internal error - the receive buffers are not aligned properly.\n");
	1091	+ return -EINVAL;
	1092	+ }
	1093	+ sg_init_one(&ioctx->imm_sg, data, len);
	1094	+ *sg = &ioctx->imm_sg;
	1095	+ *sg_cnt = 1;
	1096	+ return 0;
1052	1097	} else {
1053	1098	*data_len = 0;
1054	1099	return 0;
..	..	@@ -1173,24 +1218,18 @@
1173	1218	static struct srpt_send_ioctx srpt_get_send_ioctx(struct srpt_rdma_ch ch)
1174	1219	{
1175	1220	struct srpt_send_ioctx *ioctx;
1176		- unsigned long flags;
	1221	+ int tag, cpu;
1177	1222
1178	1223	BUG_ON(!ch);
1179	1224
1180		- ioctx = NULL;
1181		- spin_lock_irqsave(&ch->spinlock, flags);
1182		- if (!list_empty(&ch->free_list)) {
1183		- ioctx = list_first_entry(&ch->free_list,
1184		- struct srpt_send_ioctx, free_list);
1185		- list_del(&ioctx->free_list);
1186		- }
1187		- spin_unlock_irqrestore(&ch->spinlock, flags);
	1225	+ tag = sbitmap_queue_get(&ch->sess->sess_tag_pool, &cpu);
	1226	+ if (tag < 0)
	1227	+ return NULL;
1188	1228
1189		- if (!ioctx)
1190		- return ioctx;
1191		-
	1229	+ ioctx = ch->ioctx_ring[tag];
1192	1230	BUG_ON(ioctx->ch != ch);
1193	1231	ioctx->state = SRPT_STATE_NEW;
	1232	+ WARN_ON_ONCE(ioctx->recv_ioctx);
1194	1233	ioctx->n_rdma = 0;
1195	1234	ioctx->n_rw_ctx = 0;
1196	1235	ioctx->queue_status_only = false;
..	..	@@ -1200,6 +1239,8 @@
1200	1239	*/
1201	1240	memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1202	1241	memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
	1242	+ ioctx->cmd.map_tag = tag;
	1243	+ ioctx->cmd.map_cpu = cpu;
1203	1244
1204	1245	return ioctx;
1205	1246	}
..	..	@@ -1282,7 +1323,7 @@
1282	1323	*/
1283	1324	static void srpt_rdma_read_done(struct ib_cq cq, struct ib_wc wc)
1284	1325	{
1285		- struct srpt_rdma_ch *ch = cq->cq_context;
	1326	+ struct srpt_rdma_ch *ch = wc->qp->qp_context;
1286	1327	struct srpt_send_ioctx *ioctx =
1287	1328	container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
1288	1329
..	..	@@ -1376,8 +1417,8 @@
1376	1417	BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1377	1418	max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1378	1419	if (sense_data_len > max_sense_len) {
1379		- pr_warn("truncated sense data from %d to %d"
1380		- " bytes\n", sense_data_len, max_sense_len);
	1420	+ pr_warn("truncated sense data from %d to %d bytes\n",
	1421	+ sense_data_len, max_sense_len);
1381	1422	sense_data_len = max_sense_len;
1382	1423	}
1383	1424
..	..	@@ -1457,7 +1498,7 @@
1457	1498
1458	1499	BUG_ON(!send_ioctx);
1459	1500
1460		- srp_cmd = recv_ioctx->ioctx.buf;
	1501	+ srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1461	1502	cmd = &send_ioctx->cmd;
1462	1503	cmd->tag = srp_cmd->tag;
1463	1504
..	..	@@ -1477,14 +1518,14 @@
1477	1518	break;
1478	1519	}
1479	1520
1480		- rc = srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &sg, &sg_cnt,
1481		- &data_len);
	1521	+ rc = srpt_get_desc_tbl(recv_ioctx, send_ioctx, srp_cmd, &dir,
	1522	+ &sg, &sg_cnt, &data_len, ch->imm_data_offset);
1482	1523	if (rc) {
1483	1524	if (rc != -EAGAIN) {
1484	1525	pr_err("0x%llx: parsing SRP descriptor table failed.\n",
1485	1526	srp_cmd->tag);
1486	1527	}
1487		- goto release_ioctx;
	1528	+ goto busy;
1488	1529	}
1489	1530
1490	1531	rc = target_submit_cmd_map_sgls(cmd, ch->sess, srp_cmd->cdb,
..	..	@@ -1495,13 +1536,12 @@
1495	1536	if (rc != 0) {
1496	1537	pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc,
1497	1538	srp_cmd->tag);
1498		- goto release_ioctx;
	1539	+ goto busy;
1499	1540	}
1500	1541	return;
1501	1542
1502		-release_ioctx:
1503		- send_ioctx->state = SRPT_STATE_DONE;
1504		- srpt_release_cmd(cmd);
	1543	+busy:
	1544	+ target_send_busy(cmd);
1505	1545	}
1506	1546
1507	1547	static int srp_tmr_to_tcm(int fn)
..	..	@@ -1545,7 +1585,7 @@
1545	1585
1546	1586	BUG_ON(!send_ioctx);
1547	1587
1548		- srp_tsk = recv_ioctx->ioctx.buf;
	1588	+ srp_tsk = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1549	1589	cmd = &send_ioctx->cmd;
1550	1590
1551	1591	pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld ch %p sess %p\n",
..	..	@@ -1561,11 +1601,9 @@
1561	1601	TARGET_SCF_ACK_KREF);
1562	1602	if (rc != 0) {
1563	1603	send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
1564		- goto fail;
	1604	+ cmd->se_tfo->queue_tm_rsp(cmd);
1565	1605	}
1566	1606	return;
1567		-fail:
1568		- transport_send_check_condition_and_sense(cmd, 0, 0); // XXX:
1569	1607	}
1570	1608
1571	1609	/**
..	..	@@ -1588,10 +1626,11 @@
1588	1626	goto push;
1589	1627
1590	1628	ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1591		- recv_ioctx->ioctx.dma, srp_max_req_size,
	1629	+ recv_ioctx->ioctx.dma,
	1630	+ recv_ioctx->ioctx.offset + srp_max_req_size,
1592	1631	DMA_FROM_DEVICE);
1593	1632
1594		- srp_cmd = recv_ioctx->ioctx.buf;
	1633	+ srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;
1595	1634	opcode = srp_cmd->opcode;
1596	1635	if (opcode == SRP_CMD \|\| opcode == SRP_TSK_MGMT) {
1597	1636	send_ioctx = srpt_get_send_ioctx(ch);
..	..	@@ -1628,7 +1667,8 @@
1628	1667	break;
1629	1668	}
1630	1669
1631		- srpt_post_recv(ch->sport->sdev, ch, recv_ioctx);
	1670	+ if (!send_ioctx \|\| !send_ioctx->recv_ioctx)
	1671	+ srpt_post_recv(ch->sport->sdev, ch, recv_ioctx);
1632	1672	res = true;
1633	1673
1634	1674	out:
..	..	@@ -1644,7 +1684,7 @@
1644	1684
1645	1685	static void srpt_recv_done(struct ib_cq cq, struct ib_wc wc)
1646	1686	{
1647		- struct srpt_rdma_ch *ch = cq->cq_context;
	1687	+ struct srpt_rdma_ch *ch = wc->qp->qp_context;
1648	1688	struct srpt_recv_ioctx *ioctx =
1649	1689	container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
1650	1690
..	..	@@ -1654,6 +1694,7 @@
1654	1694	req_lim = atomic_dec_return(&ch->req_lim);
1655	1695	if (unlikely(req_lim < 0))
1656	1696	pr_err("req_lim = %d < 0\n", req_lim);
	1697	+ ioctx->byte_len = wc->byte_len;
1657	1698	srpt_handle_new_iu(ch, ioctx);
1658	1699	} else {
1659	1700	pr_info_ratelimited("receiving failed for ioctx %p with status %d\n",
..	..	@@ -1704,7 +1745,7 @@
1704	1745	*/
1705	1746	static void srpt_send_done(struct ib_cq cq, struct ib_wc wc)
1706	1747	{
1707		- struct srpt_rdma_ch *ch = cq->cq_context;
	1748	+ struct srpt_rdma_ch *ch = wc->qp->qp_context;
1708	1749	struct srpt_send_ioctx *ioctx =
1709	1750	container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
1710	1751	enum srpt_command_state state;
..	..	@@ -1717,14 +1758,14 @@
1717	1758	atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
1718	1759
1719	1760	if (wc->status != IB_WC_SUCCESS)
1720		- pr_info("sending response for ioctx 0x%p failed"
1721		- " with status %d\n", ioctx, wc->status);
	1761	+ pr_info("sending response for ioctx 0x%p failed with status %d\n",
	1762	+ ioctx, wc->status);
1722	1763
1723	1764	if (state != SRPT_STATE_DONE) {
1724	1765	transport_generic_free_cmd(&ioctx->cmd, 0);
1725	1766	} else {
1726		- pr_err("IB completion has been received too late for"
1727		- " wr_id = %u.\n", ioctx->ioctx.index);
	1767	+ pr_err("IB completion has been received too late for wr_id = %u.\n",
	1768	+ ioctx->ioctx.index);
1728	1769	}
1729	1770
1730	1771	srpt_process_wait_list(ch);
..	..	@@ -1751,14 +1792,15 @@
1751	1792	goto out;
1752	1793
1753	1794	retry:
1754		- ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + sq_size,
1755		- 0 /* XXX: spread CQs */, IB_POLL_WORKQUEUE);
	1795	+ ch->cq = ib_cq_pool_get(sdev->device, ch->rq_size + sq_size, -1,
	1796	+ IB_POLL_WORKQUEUE);
1756	1797	if (IS_ERR(ch->cq)) {
1757	1798	ret = PTR_ERR(ch->cq);
1758	1799	pr_err("failed to create CQ cqe= %d ret= %d\n",
1759	1800	ch->rq_size + sq_size, ret);
1760	1801	goto out;
1761	1802	}
	1803	+ ch->cq_size = ch->rq_size + sq_size;
1762	1804
1763	1805	qp_init->qp_context = (void *)ch;
1764	1806	qp_init->event_handler
..	..	@@ -1776,16 +1818,13 @@
1776	1818	*/
1777	1819	qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
1778	1820	qp_init->cap.max_rdma_ctxs = sq_size / 2;
1779		- qp_init->cap.max_send_sge = min(attrs->max_send_sge,
1780		- SRPT_MAX_SG_PER_WQE);
	1821	+ qp_init->cap.max_send_sge = attrs->max_send_sge;
	1822	+ qp_init->cap.max_recv_sge = 1;
1781	1823	qp_init->port_num = ch->sport->port;
1782		- if (sdev->use_srq) {
	1824	+ if (sdev->use_srq)
1783	1825	qp_init->srq = sdev->srq;
1784		- } else {
	1826	+ else
1785	1827	qp_init->cap.max_recv_wr = ch->rq_size;
1786		- qp_init->cap.max_recv_sge = min(attrs->max_recv_sge,
1787		- SRPT_MAX_SG_PER_WQE);
1788		- }
1789	1828
1790	1829	if (ch->using_rdma_cm) {
1791	1830	ret = rdma_create_qp(ch->rdma_cm.cm_id, sdev->pd, qp_init);
..	..	@@ -1806,7 +1845,7 @@
1806	1845	if (retry) {
1807	1846	pr_debug("failed to create queue pair with sq_size = %d (%d) - retrying\n",
1808	1847	sq_size, ret);
1809		- ib_free_cq(ch->cq);
	1848	+ ib_cq_pool_put(ch->cq, ch->cq_size);
1810	1849	sq_size = max(sq_size / 2, MIN_SRPT_SQ_SIZE);
1811	1850	goto retry;
1812	1851	} else {
..	..	@@ -1832,14 +1871,14 @@
1832	1871
1833	1872	err_destroy_cq:
1834	1873	ch->qp = NULL;
1835		- ib_free_cq(ch->cq);
	1874	+ ib_cq_pool_put(ch->cq, ch->cq_size);
1836	1875	goto out;
1837	1876	}
1838	1877
1839	1878	static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
1840	1879	{
1841	1880	ib_destroy_qp(ch->qp);
1842		- ib_free_cq(ch->cq);
	1881	+ ib_cq_pool_put(ch->cq, ch->cq_size);
1843	1882	}
1844	1883
1845	1884	/**
..	..	@@ -1913,41 +1952,22 @@
1913	1952	return ret;
1914	1953	}
1915	1954
1916		-static bool srpt_ch_closed(struct srpt_port sport, struct srpt_rdma_ch ch)
1917		-{
1918		- struct srpt_nexus *nexus;
1919		- struct srpt_rdma_ch *ch2;
1920		- bool res = true;
1921		-
1922		- rcu_read_lock();
1923		- list_for_each_entry(nexus, &sport->nexus_list, entry) {
1924		- list_for_each_entry(ch2, &nexus->ch_list, list) {
1925		- if (ch2 == ch) {
1926		- res = false;
1927		- goto done;
1928		- }
1929		- }
1930		- }
1931		-done:
1932		- rcu_read_unlock();
1933		-
1934		- return res;
1935		-}
1936		-
1937	1955	/* Send DREQ and wait for DREP. */
1938	1956	static void srpt_disconnect_ch_sync(struct srpt_rdma_ch *ch)
1939	1957	{
	1958	+ DECLARE_COMPLETION_ONSTACK(closed);
1940	1959	struct srpt_port *sport = ch->sport;
1941	1960
1942	1961	pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
1943	1962	ch->state);
1944	1963
	1964	+ ch->closed = &closed;
	1965	+
1945	1966	mutex_lock(&sport->mutex);
1946	1967	srpt_disconnect_ch(ch);
1947	1968	mutex_unlock(&sport->mutex);
1948	1969
1949		- while (wait_event_timeout(sport->ch_releaseQ, srpt_ch_closed(sport, ch),
1950		- 5 * HZ) == 0)
	1970	+ while (wait_for_completion_timeout(&closed, 5 * HZ) == 0)
1951	1971	pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
1952	1972	ch->sess_name, ch->qp->qp_num, ch->state);
1953	1973
..	..	@@ -1963,9 +1983,10 @@
1963	1983	list_for_each_entry(nexus, &sport->nexus_list, entry) {
1964	1984	list_for_each_entry(ch, &nexus->ch_list, list) {
1965	1985	if (srpt_disconnect_ch(ch) >= 0)
1966		- pr_info("Closing channel %s because target %s_%d has been disabled\n",
1967		- ch->sess_name,
1968		- sport->sdev->device->name, sport->port);
	1986	+ pr_info("Closing channel %s-%d because target %s_%d has been disabled\n",
	1987	+ ch->sess_name, ch->qp->qp_num,
	1988	+ dev_name(&sport->sdev->device->dev),
	1989	+ sport->port);
1969	1990	srpt_close_ch(ch);
1970	1991	}
1971	1992	}
..	..	@@ -2026,10 +2047,17 @@
2026	2047	__srpt_close_all_ch(sport);
2027	2048	}
2028	2049
	2050	+static void srpt_drop_sport_ref(struct srpt_port *sport)
	2051	+{
	2052	+ if (atomic_dec_return(&sport->refcount) == 0 && sport->freed_channels)
	2053	+ complete(sport->freed_channels);
	2054	+}
	2055	+
2029	2056	static void srpt_free_ch(struct kref *kref)
2030	2057	{
2031	2058	struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref);
2032	2059
	2060	+ srpt_drop_sport_ref(ch->sport);
2033	2061	kfree_rcu(ch, rcu);
2034	2062	}
2035	2063
..	..	@@ -2073,17 +2101,22 @@
2073	2101	list_del_rcu(&ch->list);
2074	2102	mutex_unlock(&sport->mutex);
2075	2103
	2104	+ if (ch->closed)
	2105	+ complete(ch->closed);
	2106	+
2076	2107	srpt_destroy_ch_ib(ch);
2077	2108
2078	2109	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2079	2110	ch->sport->sdev, ch->rq_size,
2080		- ch->max_rsp_size, DMA_TO_DEVICE);
	2111	+ ch->rsp_buf_cache, DMA_TO_DEVICE);
	2112	+
	2113	+ kmem_cache_destroy(ch->rsp_buf_cache);
2081	2114
2082	2115	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_recv_ring,
2083	2116	sdev, ch->rq_size,
2084		- srp_max_req_size, DMA_FROM_DEVICE);
	2117	+ ch->req_buf_cache, DMA_FROM_DEVICE);
2085	2118
2086		- wake_up(&sport->ch_releaseQ);
	2119	+ kmem_cache_destroy(ch->req_buf_cache);
2087	2120
2088	2121	kref_put(&ch->kref, srpt_free_ch);
2089	2122	}
..	..	@@ -2120,12 +2153,10 @@
2120	2153	struct srpt_rdma_ch *ch = NULL;
2121	2154	char i_port_id[36];
2122	2155	u32 it_iu_len;
2123		- int i, ret;
	2156	+ int i, tag_num, tag_size, ret;
	2157	+ struct srpt_tpg *stpg;
2124	2158
2125	2159	WARN_ON_ONCE(irqs_disabled());
2126		-
2127		- if (WARN_ON(!sdev \|\| !req))
2128		- return -EINVAL;
2129	2160
2130	2161	it_iu_len = be32_to_cpu(req->req_it_iu_len);
2131	2162
..	..	@@ -2159,7 +2190,7 @@
2159	2190	if (!sport->enabled) {
2160	2191	rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2161	2192	pr_info("rejected SRP_LOGIN_REQ because target port %s_%d has not yet been enabled\n",
2162		- sport->sdev->device->name, port_num);
	2193	+ dev_name(&sport->sdev->device->dev), port_num);
2163	2194	goto reject;
2164	2195	}
2165	2196
..	..	@@ -2205,32 +2236,57 @@
2205	2236	INIT_LIST_HEAD(&ch->cmd_wait_list);
2206	2237	ch->max_rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
2207	2238
	2239	+ ch->rsp_buf_cache = kmem_cache_create("srpt-rsp-buf", ch->max_rsp_size,
	2240	+ 512, 0, NULL);
	2241	+ if (!ch->rsp_buf_cache)
	2242	+ goto free_ch;
	2243	+
2208	2244	ch->ioctx_ring = (struct srpt_send_ioctx **)
2209	2245	srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
2210	2246	sizeof(*ch->ioctx_ring[0]),
2211		- ch->max_rsp_size, DMA_TO_DEVICE);
	2247	+ ch->rsp_buf_cache, 0, DMA_TO_DEVICE);
2212	2248	if (!ch->ioctx_ring) {
2213	2249	pr_err("rejected SRP_LOGIN_REQ because creating a new QP SQ ring failed.\n");
2214	2250	rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2215		- goto free_ch;
	2251	+ goto free_rsp_cache;
2216	2252	}
2217	2253
2218		- INIT_LIST_HEAD(&ch->free_list);
2219		- for (i = 0; i < ch->rq_size; i++) {
	2254	+ for (i = 0; i < ch->rq_size; i++)
2220	2255	ch->ioctx_ring[i]->ch = ch;
2221		- list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list);
2222		- }
2223	2256	if (!sdev->use_srq) {
	2257	+ u16 imm_data_offset = req->req_flags & SRP_IMMED_REQUESTED ?
	2258	+ be16_to_cpu(req->imm_data_offset) : 0;
	2259	+ u16 alignment_offset;
	2260	+ u32 req_sz;
	2261	+
	2262	+ if (req->req_flags & SRP_IMMED_REQUESTED)
	2263	+ pr_debug("imm_data_offset = %d\n",
	2264	+ be16_to_cpu(req->imm_data_offset));
	2265	+ if (imm_data_offset >= sizeof(struct srp_cmd)) {
	2266	+ ch->imm_data_offset = imm_data_offset;
	2267	+ rsp->rsp_flags \|= SRP_LOGIN_RSP_IMMED_SUPP;
	2268	+ } else {
	2269	+ ch->imm_data_offset = 0;
	2270	+ }
	2271	+ alignment_offset = round_up(imm_data_offset, 512) -
	2272	+ imm_data_offset;
	2273	+ req_sz = alignment_offset + imm_data_offset + srp_max_req_size;
	2274	+ ch->req_buf_cache = kmem_cache_create("srpt-req-buf", req_sz,
	2275	+ 512, 0, NULL);
	2276	+ if (!ch->req_buf_cache)
	2277	+ goto free_rsp_ring;
	2278	+
2224	2279	ch->ioctx_recv_ring = (struct srpt_recv_ioctx **)
2225	2280	srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
2226	2281	sizeof(*ch->ioctx_recv_ring[0]),
2227		- srp_max_req_size,
	2282	+ ch->req_buf_cache,
	2283	+ alignment_offset,
2228	2284	DMA_FROM_DEVICE);
2229	2285	if (!ch->ioctx_recv_ring) {
2230	2286	pr_err("rejected SRP_LOGIN_REQ because creating a new QP RQ ring failed.\n");
2231	2287	rej->reason =
2232	2288	cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2233		- goto free_ring;
	2289	+ goto free_recv_cache;
2234	2290	}
2235	2291	for (i = 0; i < ch->rq_size; i++)
2236	2292	INIT_LIST_HEAD(&ch->ioctx_recv_ring[i]->wait_list);
..	..	@@ -2248,21 +2304,42 @@
2248	2304	be64_to_cpu((__be64 )nexus->i_port_id),
2249	2305	be64_to_cpu((__be64 )(nexus->i_port_id + 8)));
2250	2306
2251		- pr_debug("registering session %s\n", ch->sess_name);
	2307	+ pr_debug("registering src addr %s or i_port_id %s\n", ch->sess_name,
	2308	+ i_port_id);
2252	2309
2253		- if (sport->port_guid_tpg.se_tpg_wwn)
2254		- ch->sess = target_setup_session(&sport->port_guid_tpg, 0, 0,
2255		- TARGET_PROT_NORMAL,
	2310	+ tag_num = ch->rq_size;
	2311	+ tag_size = 1; /* ib_srpt does not use se_sess->sess_cmd_map */
	2312	+
	2313	+ if (sport->guid_id) {
	2314	+ mutex_lock(&sport->guid_id->mutex);
	2315	+ list_for_each_entry(stpg, &sport->guid_id->tpg_list, entry) {
	2316	+ if (!IS_ERR_OR_NULL(ch->sess))
	2317	+ break;
	2318	+ ch->sess = target_setup_session(&stpg->tpg, tag_num,
	2319	+ tag_size, TARGET_PROT_NORMAL,
2256	2320	ch->sess_name, ch, NULL);
2257		- if (sport->port_gid_tpg.se_tpg_wwn && IS_ERR_OR_NULL(ch->sess))
2258		- ch->sess = target_setup_session(&sport->port_gid_tpg, 0, 0,
2259		- TARGET_PROT_NORMAL, i_port_id, ch,
2260		- NULL);
2261		- /* Retry without leading "0x" */
2262		- if (sport->port_gid_tpg.se_tpg_wwn && IS_ERR_OR_NULL(ch->sess))
2263		- ch->sess = target_setup_session(&sport->port_gid_tpg, 0, 0,
2264		- TARGET_PROT_NORMAL,
	2321	+ }
	2322	+ mutex_unlock(&sport->guid_id->mutex);
	2323	+ }
	2324	+
	2325	+ if (sport->gid_id) {
	2326	+ mutex_lock(&sport->gid_id->mutex);
	2327	+ list_for_each_entry(stpg, &sport->gid_id->tpg_list, entry) {
	2328	+ if (!IS_ERR_OR_NULL(ch->sess))
	2329	+ break;
	2330	+ ch->sess = target_setup_session(&stpg->tpg, tag_num,
	2331	+ tag_size, TARGET_PROT_NORMAL, i_port_id,
	2332	+ ch, NULL);
	2333	+ if (!IS_ERR_OR_NULL(ch->sess))
	2334	+ break;
	2335	+ /* Retry without leading "0x" */
	2336	+ ch->sess = target_setup_session(&stpg->tpg, tag_num,
	2337	+ tag_size, TARGET_PROT_NORMAL,
2265	2338	i_port_id + 2, ch, NULL);
	2339	+ }
	2340	+ mutex_unlock(&sport->gid_id->mutex);
	2341	+ }
	2342	+
2266	2343	if (IS_ERR_OR_NULL(ch->sess)) {
2267	2344	WARN_ON_ONCE(ch->sess == NULL);
2268	2345	ret = PTR_ERR(ch->sess);
..	..	@@ -2275,22 +2352,26 @@
2275	2352	goto destroy_ib;
2276	2353	}
2277	2354
	2355	+ /*
	2356	+ * Once a session has been created destruction of srpt_rdma_ch objects
	2357	+ * will decrement sport->refcount. Hence increment sport->refcount now.
	2358	+ */
	2359	+ atomic_inc(&sport->refcount);
	2360	+
2278	2361	mutex_lock(&sport->mutex);
2279	2362
2280	2363	if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
2281	2364	struct srpt_rdma_ch *ch2;
2282		-
2283		- rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
2284	2365
2285	2366	list_for_each_entry(ch2, &nexus->ch_list, list) {
2286	2367	if (srpt_disconnect_ch(ch2) < 0)
2287	2368	continue;
2288	2369	pr_info("Relogin - closed existing channel %s\n",
2289	2370	ch2->sess_name);
2290		- rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
	2371	+ rsp->rsp_flags \|= SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
2291	2372	}
2292	2373	} else {
2293		- rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
	2374	+ rsp->rsp_flags \|= SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
2294	2375	}
2295	2376
2296	2377	list_add_tail_rcu(&ch->list, &nexus->ch_list);
..	..	@@ -2299,7 +2380,7 @@
2299	2380	rej->reason = cpu_to_be32(
2300	2381	SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2301	2382	pr_info("rejected SRP_LOGIN_REQ because target %s_%d is not enabled\n",
2302		- sdev->device->name, port_num);
	2383	+ dev_name(&sdev->device->dev), port_num);
2303	2384	mutex_unlock(&sport->mutex);
2304	2385	ret = -EINVAL;
2305	2386	goto reject;
..	..	@@ -2321,7 +2402,7 @@
2321	2402	/* create srp_login_response */
2322	2403	rsp->opcode = SRP_LOGIN_RSP;
2323	2404	rsp->tag = req->tag;
2324		- rsp->max_it_iu_len = req->req_it_iu_len;
	2405	+ rsp->max_it_iu_len = cpu_to_be32(srp_max_req_size);
2325	2406	rsp->max_ti_iu_len = req->req_it_iu_len;
2326	2407	ch->max_ti_iu_len = it_iu_len;
2327	2408	rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT \|
..	..	@@ -2385,12 +2466,18 @@
2385	2466	free_recv_ring:
2386	2467	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_recv_ring,
2387	2468	ch->sport->sdev, ch->rq_size,
2388		- srp_max_req_size, DMA_FROM_DEVICE);
	2469	+ ch->req_buf_cache, DMA_FROM_DEVICE);
2389	2470
2390		-free_ring:
	2471	+free_recv_cache:
	2472	+ kmem_cache_destroy(ch->req_buf_cache);
	2473	+
	2474	+free_rsp_ring:
2391	2475	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2392	2476	ch->sport->sdev, ch->rq_size,
2393		- ch->max_rsp_size, DMA_TO_DEVICE);
	2477	+ ch->rsp_buf_cache, DMA_TO_DEVICE);
	2478	+
	2479	+free_rsp_cache:
	2480	+ kmem_cache_destroy(ch->rsp_buf_cache);
2394	2481
2395	2482	free_ch:
2396	2483	if (rdma_cm_id)
..	..	@@ -2410,7 +2497,8 @@
2410	2497	SRP_BUF_FORMAT_INDIRECT);
2411	2498
2412	2499	if (rdma_cm_id)
2413		- rdma_reject(rdma_cm_id, rej, sizeof(*rej));
	2500	+ rdma_reject(rdma_cm_id, rej, sizeof(*rej),
	2501	+ IB_CM_REJ_CONSUMER_DEFINED);
2414	2502	else
2415	2503	ib_send_cm_rej(ib_cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
2416	2504	rej, sizeof(*rej));
..	..	@@ -2452,6 +2540,7 @@
2452	2540	struct srpt_device *sdev;
2453	2541	struct srp_login_req req;
2454	2542	const struct srp_login_req_rdma *req_rdma;
	2543	+ struct sa_path_rec *path_rec = cm_id->route.path_rec;
2455	2544	char src_addr[40];
2456	2545
2457	2546	sdev = ib_get_client_data(cm_id->device, &srpt_client);
..	..	@@ -2471,12 +2560,13 @@
2471	2560	req.req_flags = req_rdma->req_flags;
2472	2561	memcpy(req.initiator_port_id, req_rdma->initiator_port_id, 16);
2473	2562	memcpy(req.target_port_id, req_rdma->target_port_id, 16);
	2563	+ req.imm_data_offset = req_rdma->imm_data_offset;
2474	2564
2475	2565	snprintf(src_addr, sizeof(src_addr), "%pIS",
2476	2566	&cm_id->route.addr.src_addr);
2477	2567
2478	2568	return srpt_cm_req_recv(sdev, NULL, cm_id, cm_id->port_num,
2479		- cm_id->route.path_rec->pkey, &req, src_addr);
	2569	+ path_rec ? path_rec->pkey : 0, &req, src_addr);
2480	2570	}
2481	2571
2482	2572	static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
..	..	@@ -2640,14 +2730,6 @@
2640	2730	return ret;
2641	2731	}
2642	2732
2643		-static int srpt_write_pending_status(struct se_cmd *se_cmd)
2644		-{
2645		- struct srpt_send_ioctx *ioctx;
2646		-
2647		- ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2648		- return ioctx->state == SRPT_STATE_NEED_DATA;
2649		-}
2650		-
2651	2733	/*
2652	2734	* srpt_write_pending - Start data transfer from initiator to target (write).
2653	2735	*/
..	..	@@ -2660,6 +2742,12 @@
2660	2742	struct ib_cqe *cqe = &ioctx->rdma_cqe;
2661	2743	enum srpt_command_state new_state;
2662	2744	int ret, i;
	2745	+
	2746	+ if (ioctx->recv_ioctx) {
	2747	+ srpt_set_cmd_state(ioctx, SRPT_STATE_DATA_IN);
	2748	+ target_execute_cmd(&ioctx->cmd);
	2749	+ return 0;
	2750	+ }
2663	2751
2664	2752	new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);
2665	2753	WARN_ON(new_state == SRPT_STATE_DONE);
..	..	@@ -2724,8 +2812,6 @@
2724	2812	int resp_len, ret, i;
2725	2813	u8 srp_tm_status;
2726	2814
2727		- BUG_ON(!ch);
2728		-
2729	2815	state = ioctx->state;
2730	2816	switch (state) {
2731	2817	case SRPT_STATE_NEW:
..	..	@@ -2741,7 +2827,7 @@
2741	2827	break;
2742	2828	}
2743	2829
2744		- if (unlikely(WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT)))
	2830	+ if (WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))
2745	2831	return;
2746	2832
2747	2833	/* For read commands, transfer the data to the initiator. */
..	..	@@ -2854,39 +2940,29 @@
2854	2940	srpt_refresh_port(sport);
2855	2941	}
2856	2942
2857		-static bool srpt_ch_list_empty(struct srpt_port *sport)
2858		-{
2859		- struct srpt_nexus *nexus;
2860		- bool res = true;
2861		-
2862		- rcu_read_lock();
2863		- list_for_each_entry(nexus, &sport->nexus_list, entry)
2864		- if (!list_empty(&nexus->ch_list))
2865		- res = false;
2866		- rcu_read_unlock();
2867		-
2868		- return res;
2869		-}
2870		-
2871	2943	/**
2872	2944	* srpt_release_sport - disable login and wait for associated channels
2873	2945	* @sport: SRPT HCA port.
2874	2946	*/
2875	2947	static int srpt_release_sport(struct srpt_port *sport)
2876	2948	{
	2949	+ DECLARE_COMPLETION_ONSTACK(c);
2877	2950	struct srpt_nexus nexus, next_n;
2878	2951	struct srpt_rdma_ch *ch;
2879	2952
2880	2953	WARN_ON_ONCE(irqs_disabled());
2881	2954
	2955	+ sport->freed_channels = &c;
	2956	+
2882	2957	mutex_lock(&sport->mutex);
2883	2958	srpt_set_enabled(sport, false);
2884	2959	mutex_unlock(&sport->mutex);
2885	2960
2886		- while (wait_event_timeout(sport->ch_releaseQ,
2887		- srpt_ch_list_empty(sport), 5 * HZ) <= 0) {
2888		- pr_info("%s_%d: waiting for session unregistration ...\n",
2889		- sport->sdev->device->name, sport->port);
	2961	+ while (atomic_read(&sport->refcount) > 0 &&
	2962	+ wait_for_completion_timeout(&c, 5 * HZ) <= 0) {
	2963	+ pr_info("%s_%d: waiting for unregistration of %d sessions ...\n",
	2964	+ dev_name(&sport->sdev->device->dev), sport->port,
	2965	+ atomic_read(&sport->refcount));
2890	2966	rcu_read_lock();
2891	2967	list_for_each_entry(nexus, &sport->nexus_list, entry) {
2892	2968	list_for_each_entry(ch, &nexus->ch_list, list) {
..	..	@@ -2908,7 +2984,12 @@
2908	2984	return 0;
2909	2985	}
2910	2986
2911		-static struct se_wwn __srpt_lookup_wwn(const char name)
	2987	+struct port_and_port_id {
	2988	+ struct srpt_port *sport;
	2989	+ struct srpt_port_id **port_id;
	2990	+};
	2991	+
	2992	+static struct port_and_port_id __srpt_lookup_port(const char *name)
2912	2993	{
2913	2994	struct ib_device *dev;
2914	2995	struct srpt_device *sdev;
..	..	@@ -2923,25 +3004,38 @@
2923	3004	for (i = 0; i < dev->phys_port_cnt; i++) {
2924	3005	sport = &sdev->port[i];
2925	3006
2926		- if (strcmp(sport->port_guid, name) == 0)
2927		- return &sport->port_guid_wwn;
2928		- if (strcmp(sport->port_gid, name) == 0)
2929		- return &sport->port_gid_wwn;
	3007	+ if (strcmp(sport->guid_name, name) == 0) {
	3008	+ kref_get(&sdev->refcnt);
	3009	+ return (struct port_and_port_id){
	3010	+ sport, &sport->guid_id};
	3011	+ }
	3012	+ if (strcmp(sport->gid_name, name) == 0) {
	3013	+ kref_get(&sdev->refcnt);
	3014	+ return (struct port_and_port_id){
	3015	+ sport, &sport->gid_id};
	3016	+ }
2930	3017	}
2931	3018	}
2932	3019
2933		- return NULL;
	3020	+ return (struct port_and_port_id){};
2934	3021	}
2935	3022
2936		-static struct se_wwn srpt_lookup_wwn(const char name)
	3023	+/**
	3024	+ * srpt_lookup_port() - Look up an RDMA port by name
	3025	+ * @name: ASCII port name
	3026	+ *
	3027	+ * Increments the RDMA port reference count if an RDMA port pointer is returned.
	3028	+ * The caller must drop that reference count by calling srpt_port_put_ref().
	3029	+ */
	3030	+static struct port_and_port_id srpt_lookup_port(const char *name)
2937	3031	{
2938		- struct se_wwn *wwn;
	3032	+ struct port_and_port_id papi;
2939	3033
2940	3034	spin_lock(&srpt_dev_lock);
2941		- wwn = __srpt_lookup_wwn(name);
	3035	+ papi = __srpt_lookup_port(name);
2942	3036	spin_unlock(&srpt_dev_lock);
2943	3037
2944		- return wwn;
	3038	+ return papi;
2945	3039	}
2946	3040
2947	3041	static void srpt_free_srq(struct srpt_device *sdev)
..	..	@@ -2951,7 +3045,9 @@
2951	3045
2952	3046	ib_destroy_srq(sdev->srq);
2953	3047	srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2954		- sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE);
	3048	+ sdev->srq_size, sdev->req_buf_cache,
	3049	+ DMA_FROM_DEVICE);
	3050	+ kmem_cache_destroy(sdev->req_buf_cache);
2955	3051	sdev->srq = NULL;
2956	3052	}
2957	3053
..	..	@@ -2976,16 +3072,19 @@
2976	3072	}
2977	3073
2978	3074	pr_debug("create SRQ #wr= %d max_allow=%d dev= %s\n", sdev->srq_size,
2979		- sdev->device->attrs.max_srq_wr, device->name);
	3075	+ sdev->device->attrs.max_srq_wr, dev_name(&device->dev));
	3076	+
	3077	+ sdev->req_buf_cache = kmem_cache_create("srpt-srq-req-buf",
	3078	+ srp_max_req_size, 0, 0, NULL);
	3079	+ if (!sdev->req_buf_cache)
	3080	+ goto free_srq;
2980	3081
2981	3082	sdev->ioctx_ring = (struct srpt_recv_ioctx **)
2982	3083	srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
2983	3084	sizeof(*sdev->ioctx_ring[0]),
2984		- srp_max_req_size, DMA_FROM_DEVICE);
2985		- if (!sdev->ioctx_ring) {
2986		- ib_destroy_srq(srq);
2987		- return -ENOMEM;
2988		- }
	3085	+ sdev->req_buf_cache, 0, DMA_FROM_DEVICE);
	3086	+ if (!sdev->ioctx_ring)
	3087	+ goto free_cache;
2989	3088
2990	3089	sdev->use_srq = true;
2991	3090	sdev->srq = srq;
..	..	@@ -2996,6 +3095,13 @@
2996	3095	}
2997	3096
2998	3097	return 0;
	3098	+
	3099	+free_cache:
	3100	+ kmem_cache_destroy(sdev->req_buf_cache);
	3101	+
	3102	+free_srq:
	3103	+ ib_destroy_srq(srq);
	3104	+ return -ENOMEM;
2999	3105	}
3000	3106
3001	3107	static int srpt_use_srq(struct srpt_device *sdev, bool use_srq)
..	..	@@ -3009,16 +3115,28 @@
3009	3115	} else if (use_srq && !sdev->srq) {
3010	3116	ret = srpt_alloc_srq(sdev);
3011	3117	}
3012		- pr_debug("%s(%s): use_srq = %d; ret = %d\n", __func__, device->name,
3013		- sdev->use_srq, ret);
	3118	+ pr_debug("%s(%s): use_srq = %d; ret = %d\n", __func__,
	3119	+ dev_name(&device->dev), sdev->use_srq, ret);
3014	3120	return ret;
	3121	+}
	3122	+
	3123	+static void srpt_free_sdev(struct kref *refcnt)
	3124	+{
	3125	+ struct srpt_device sdev = container_of(refcnt, typeof(sdev), refcnt);
	3126	+
	3127	+ kfree(sdev);
	3128	+}
	3129	+
	3130	+static void srpt_sdev_put(struct srpt_device *sdev)
	3131	+{
	3132	+ kref_put(&sdev->refcnt, srpt_free_sdev);
3015	3133	}
3016	3134
3017	3135	/**
3018	3136	* srpt_add_one - InfiniBand device addition callback function
3019	3137	* @device: Describes a HCA.
3020	3138	*/
3021		-static void srpt_add_one(struct ib_device *device)
	3139	+static int srpt_add_one(struct ib_device *device)
3022	3140	{
3023	3141	struct srpt_device *sdev;
3024	3142	struct srpt_port *sport;
..	..	@@ -3029,14 +3147,17 @@
3029	3147	sdev = kzalloc(struct_size(sdev, port, device->phys_port_cnt),
3030	3148	GFP_KERNEL);
3031	3149	if (!sdev)
3032		- goto err;
	3150	+ return -ENOMEM;
3033	3151
	3152	+ kref_init(&sdev->refcnt);
3034	3153	sdev->device = device;
3035	3154	mutex_init(&sdev->sdev_mutex);
3036	3155
3037	3156	sdev->pd = ib_alloc_pd(device, 0);
3038		- if (IS_ERR(sdev->pd))
	3157	+ if (IS_ERR(sdev->pd)) {
	3158	+ ret = PTR_ERR(sdev->pd);
3039	3159	goto free_dev;
	3160	+ }
3040	3161
3041	3162	sdev->lkey = sdev->pd->local_dma_lkey;
3042	3163
..	..	@@ -3052,15 +3173,15 @@
3052	3173	if (IS_ERR(sdev->cm_id)) {
3053	3174	pr_info("ib_create_cm_id() failed: %ld\n",
3054	3175	PTR_ERR(sdev->cm_id));
	3176	+ ret = PTR_ERR(sdev->cm_id);
3055	3177	sdev->cm_id = NULL;
3056	3178	if (!rdma_cm_id)
3057	3179	goto err_ring;
3058	3180	}
3059	3181
3060	3182	/* print out target login information */
3061		- pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,"
3062		- "pkey=ffff,service_id=%016llx\n", srpt_service_guid,
3063		- srpt_service_guid, srpt_service_guid);
	3183	+ pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,pkey=ffff,service_id=%016llx\n",
	3184	+ srpt_service_guid, srpt_service_guid, srpt_service_guid);
3064	3185
3065	3186	/*
3066	3187	* We do not have a consistent service_id (ie. also id_ext of target_id)
..	..	@@ -3084,7 +3205,6 @@
3084	3205	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
3085	3206	sport = &sdev->port[i - 1];
3086	3207	INIT_LIST_HEAD(&sport->nexus_list);
3087		- init_waitqueue_head(&sport->ch_releaseQ);
3088	3208	mutex_init(&sport->mutex);
3089	3209	sport->sdev = sdev;
3090	3210	sport->port = i;
..	..	@@ -3094,10 +3214,12 @@
3094	3214	sport->port_attrib.use_srq = false;
3095	3215	INIT_WORK(&sport->work, srpt_refresh_port_work);
3096	3216
3097		- if (srpt_refresh_port(sport)) {
	3217	+ ret = srpt_refresh_port(sport);
	3218	+ if (ret) {
3098	3219	pr_err("MAD registration failed for %s-%d.\n",
3099		- sdev->device->name, i);
3100		- goto err_event;
	3220	+ dev_name(&sdev->device->dev), i);
	3221	+ i--;
	3222	+ goto err_port;
3101	3223	}
3102	3224	}
3103	3225
..	..	@@ -3105,12 +3227,12 @@
3105	3227	list_add_tail(&sdev->list, &srpt_dev_list);
3106	3228	spin_unlock(&srpt_dev_lock);
3107	3229
3108		-out:
3109	3230	ib_set_client_data(device, &srpt_client, sdev);
3110		- pr_debug("added %s.\n", device->name);
3111		- return;
	3231	+ pr_debug("added %s.\n", dev_name(&device->dev));
	3232	+ return 0;
3112	3233
3113		-err_event:
	3234	+err_port:
	3235	+ srpt_unregister_mad_agent(sdev, i);
3114	3236	ib_unregister_event_handler(&sdev->event_handler);
3115	3237	err_cm:
3116	3238	if (sdev->cm_id)
..	..	@@ -3119,11 +3241,9 @@
3119	3241	srpt_free_srq(sdev);
3120	3242	ib_dealloc_pd(sdev->pd);
3121	3243	free_dev:
3122		- kfree(sdev);
3123		-err:
3124		- sdev = NULL;
3125		- pr_info("%s(%s) failed.\n", __func__, device->name);
3126		- goto out;
	3244	+ srpt_sdev_put(sdev);
	3245	+ pr_info("%s(%s) failed.\n", __func__, dev_name(&device->dev));
	3246	+ return ret;
3127	3247	}
3128	3248
3129	3249	/**
..	..	@@ -3136,12 +3256,7 @@
3136	3256	struct srpt_device *sdev = client_data;
3137	3257	int i;
3138	3258
3139		- if (!sdev) {
3140		- pr_info("%s(%s): nothing to do.\n", __func__, device->name);
3141		- return;
3142		- }
3143		-
3144		- srpt_unregister_mad_agent(sdev);
	3259	+ srpt_unregister_mad_agent(sdev, sdev->device->phys_port_cnt);
3145	3260
3146	3261	ib_unregister_event_handler(&sdev->event_handler);
3147	3262
..	..	@@ -3170,7 +3285,7 @@
3170	3285
3171	3286	ib_dealloc_pd(sdev->pd);
3172	3287
3173		- kfree(sdev);
	3288	+ srpt_sdev_put(sdev);
3174	3289	}
3175	3290
3176	3291	static struct ib_client srpt_client = {
..	..	@@ -3189,24 +3304,28 @@
3189	3304	return 0;
3190	3305	}
3191	3306
3192		-static char *srpt_get_fabric_name(void)
3193		-{
3194		- return "srpt";
3195		-}
3196		-
3197	3307	static struct srpt_port srpt_tpg_to_sport(struct se_portal_group tpg)
3198	3308	{
3199	3309	return tpg->se_tpg_wwn->priv;
3200	3310	}
3201	3311
	3312	+static struct srpt_port_id srpt_wwn_to_sport_id(struct se_wwn wwn)
	3313	+{
	3314	+ struct srpt_port *sport = wwn->priv;
	3315	+
	3316	+ if (sport->guid_id && &sport->guid_id->wwn == wwn)
	3317	+ return sport->guid_id;
	3318	+ if (sport->gid_id && &sport->gid_id->wwn == wwn)
	3319	+ return sport->gid_id;
	3320	+ WARN_ON_ONCE(true);
	3321	+ return NULL;
	3322	+}
	3323	+
3202	3324	static char srpt_get_fabric_wwn(struct se_portal_group tpg)
3203	3325	{
3204		- struct srpt_port *sport = srpt_tpg_to_sport(tpg);
	3326	+ struct srpt_tpg stpg = container_of(tpg, typeof(stpg), tpg);
3205	3327
3206		- WARN_ON_ONCE(tpg != &sport->port_guid_tpg &&
3207		- tpg != &sport->port_gid_tpg);
3208		- return tpg == &sport->port_guid_tpg ? sport->port_guid :
3209		- sport->port_gid;
	3328	+ return stpg->sport_id->name;
3210	3329	}
3211	3330
3212	3331	static u16 srpt_get_tag(struct se_portal_group *tpg)
..	..	@@ -3224,19 +3343,23 @@
3224	3343	struct srpt_send_ioctx *ioctx = container_of(se_cmd,
3225	3344	struct srpt_send_ioctx, cmd);
3226	3345	struct srpt_rdma_ch *ch = ioctx->ch;
3227		- unsigned long flags;
	3346	+ struct srpt_recv_ioctx *recv_ioctx = ioctx->recv_ioctx;
3228	3347
3229	3348	WARN_ON_ONCE(ioctx->state != SRPT_STATE_DONE &&
3230	3349	!(ioctx->cmd.transport_state & CMD_T_ABORTED));
	3350	+
	3351	+ if (recv_ioctx) {
	3352	+ WARN_ON_ONCE(!list_empty(&recv_ioctx->wait_list));
	3353	+ ioctx->recv_ioctx = NULL;
	3354	+ srpt_post_recv(ch->sport->sdev, ch, recv_ioctx);
	3355	+ }
3231	3356
3232	3357	if (ioctx->n_rw_ctx) {
3233	3358	srpt_free_rw_ctxs(ch, ioctx);
3234	3359	ioctx->n_rw_ctx = 0;
3235	3360	}
3236	3361
3237		- spin_lock_irqsave(&ch->spinlock, flags);
3238		- list_add(&ioctx->free_list, &ch->free_list);
3239		- spin_unlock_irqrestore(&ch->spinlock, flags);
	3362	+ target_free_tag(se_cmd->se_sess, se_cmd);
3240	3363	}
3241	3364
3242	3365	/**
..	..	@@ -3614,7 +3737,7 @@
3614	3737	struct se_portal_group *se_tpg = to_tpg(item);
3615	3738	struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3616	3739
3617		- return snprintf(page, PAGE_SIZE, "%d\n", (sport->enabled) ? 1: 0);
	3740	+ return snprintf(page, PAGE_SIZE, "%d\n", sport->enabled);
3618	3741	}
3619	3742
3620	3743	static ssize_t srpt_tpg_enable_store(struct config_item *item,
..	..	@@ -3623,7 +3746,7 @@
3623	3746	struct se_portal_group *se_tpg = to_tpg(item);
3624	3747	struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
3625	3748	unsigned long tmp;
3626		- int ret;
	3749	+ int ret;
3627	3750
3628	3751	ret = kstrtoul(page, 0, &tmp);
3629	3752	if (ret < 0) {
..	..	@@ -3658,19 +3781,25 @@
3658	3781	static struct se_portal_group srpt_make_tpg(struct se_wwn wwn,
3659	3782	const char *name)
3660	3783	{
3661		- struct srpt_port *sport = wwn->priv;
3662		- static struct se_portal_group *tpg;
3663		- int res;
	3784	+ struct srpt_port_id *sport_id = srpt_wwn_to_sport_id(wwn);
	3785	+ struct srpt_tpg *stpg;
	3786	+ int res = -ENOMEM;
3664	3787
3665		- WARN_ON_ONCE(wwn != &sport->port_guid_wwn &&
3666		- wwn != &sport->port_gid_wwn);
3667		- tpg = wwn == &sport->port_guid_wwn ? &sport->port_guid_tpg :
3668		- &sport->port_gid_tpg;
3669		- res = core_tpg_register(wwn, tpg, SCSI_PROTOCOL_SRP);
3670		- if (res)
	3788	+ stpg = kzalloc(sizeof(*stpg), GFP_KERNEL);
	3789	+ if (!stpg)
3671	3790	return ERR_PTR(res);
	3791	+ stpg->sport_id = sport_id;
	3792	+ res = core_tpg_register(wwn, &stpg->tpg, SCSI_PROTOCOL_SRP);
	3793	+ if (res) {
	3794	+ kfree(stpg);
	3795	+ return ERR_PTR(res);
	3796	+ }
3672	3797
3673		- return tpg;
	3798	+ mutex_lock(&sport_id->mutex);
	3799	+ list_add_tail(&stpg->entry, &sport_id->tpg_list);
	3800	+ mutex_unlock(&sport_id->mutex);
	3801	+
	3802	+ return &stpg->tpg;
3674	3803	}
3675	3804
3676	3805	/**
..	..	@@ -3679,10 +3808,17 @@
3679	3808	*/
3680	3809	static void srpt_drop_tpg(struct se_portal_group *tpg)
3681	3810	{
	3811	+ struct srpt_tpg stpg = container_of(tpg, typeof(stpg), tpg);
	3812	+ struct srpt_port_id *sport_id = stpg->sport_id;
3682	3813	struct srpt_port *sport = srpt_tpg_to_sport(tpg);
	3814	+
	3815	+ mutex_lock(&sport_id->mutex);
	3816	+ list_del(&stpg->entry);
	3817	+ mutex_unlock(&sport_id->mutex);
3683	3818
3684	3819	sport->enabled = false;
3685	3820	core_tpg_deregister(tpg);
	3821	+ kfree(stpg);
3686	3822	}
3687	3823
3688	3824	/**
..	..	@@ -3695,7 +3831,31 @@
3695	3831	struct config_group *group,
3696	3832	const char *name)
3697	3833	{
3698		- return srpt_lookup_wwn(name) ? : ERR_PTR(-EINVAL);
	3834	+ struct port_and_port_id papi = srpt_lookup_port(name);
	3835	+ struct srpt_port *sport = papi.sport;
	3836	+ struct srpt_port_id *port_id;
	3837	+
	3838	+ if (!papi.port_id)
	3839	+ return ERR_PTR(-EINVAL);
	3840	+ if (*papi.port_id) {
	3841	+ /* Attempt to create a directory that already exists. */
	3842	+ WARN_ON_ONCE(true);
	3843	+ return &(*papi.port_id)->wwn;
	3844	+ }
	3845	+ port_id = kzalloc(sizeof(*port_id), GFP_KERNEL);
	3846	+ if (!port_id) {
	3847	+ srpt_sdev_put(sport->sdev);
	3848	+ return ERR_PTR(-ENOMEM);
	3849	+ }
	3850	+ mutex_init(&port_id->mutex);
	3851	+ INIT_LIST_HEAD(&port_id->tpg_list);
	3852	+ port_id->wwn.priv = sport;
	3853	+ memcpy(port_id->name, port_id == sport->guid_id ? sport->guid_name :
	3854	+ sport->gid_name, ARRAY_SIZE(port_id->name));
	3855	+
	3856	+ *papi.port_id = port_id;
	3857	+
	3858	+ return &port_id->wwn;
3699	3859	}
3700	3860
3701	3861	/**
..	..	@@ -3704,11 +3864,23 @@
3704	3864	*/
3705	3865	static void srpt_drop_tport(struct se_wwn *wwn)
3706	3866	{
	3867	+ struct srpt_port_id port_id = container_of(wwn, typeof(port_id), wwn);
	3868	+ struct srpt_port *sport = wwn->priv;
	3869	+
	3870	+ if (sport->guid_id == port_id)
	3871	+ sport->guid_id = NULL;
	3872	+ else if (sport->gid_id == port_id)
	3873	+ sport->gid_id = NULL;
	3874	+ else
	3875	+ WARN_ON_ONCE(true);
	3876	+
	3877	+ srpt_sdev_put(sport->sdev);
	3878	+ kfree(port_id);
3707	3879	}
3708	3880
3709	3881	static ssize_t srpt_wwn_version_show(struct config_item item, char buf)
3710	3882	{
3711		- return scnprintf(buf, PAGE_SIZE, "%s\n", DRV_VERSION);
	3883	+ return scnprintf(buf, PAGE_SIZE, "\n");
3712	3884	}
3713	3885
3714	3886	CONFIGFS_ATTR_RO(srpt_wwn_, version);
..	..	@@ -3720,8 +3892,7 @@
3720	3892
3721	3893	static const struct target_core_fabric_ops srpt_template = {
3722	3894	.module = THIS_MODULE,
3723		- .name = "srpt",
3724		- .get_fabric_name = srpt_get_fabric_name,
	3895	+ .fabric_name = "srpt",
3725	3896	.tpg_get_wwn = srpt_get_fabric_wwn,
3726	3897	.tpg_get_tag = srpt_get_tag,
3727	3898	.tpg_check_demo_mode = srpt_check_false,
..	..	@@ -3735,7 +3906,6 @@
3735	3906	.sess_get_index = srpt_sess_get_index,
3736	3907	.sess_get_initiator_sid = NULL,
3737	3908	.write_pending = srpt_write_pending,
3738		- .write_pending_status = srpt_write_pending_status,
3739	3909	.set_default_node_attributes = srpt_set_default_node_attrs,
3740	3910	.get_cmd_state = srpt_get_tcm_cmd_state,
3741	3911	.queue_data_in = srpt_queue_data_in,
..	..	@@ -3772,16 +3942,14 @@
3772	3942
3773	3943	ret = -EINVAL;
3774	3944	if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
3775		- pr_err("invalid value %d for kernel module parameter"
3776		- " srp_max_req_size -- must be at least %d.\n",
	3945	+ pr_err("invalid value %d for kernel module parameter srp_max_req_size -- must be at least %d.\n",
3777	3946	srp_max_req_size, MIN_MAX_REQ_SIZE);
3778	3947	goto out;
3779	3948	}
3780	3949
3781	3950	if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3782	3951	\|\| srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
3783		- pr_err("invalid value %d for kernel module parameter"
3784		- " srpt_srq_size -- must be in the range [%d..%d].\n",
	3952	+ pr_err("invalid value %d for kernel module parameter srpt_srq_size -- must be in the range [%d..%d].\n",
3785	3953	srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3786	3954	goto out;
3787	3955	}