forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc 
 kernel/net/rds/ib_cm.c
....@@ -1,5 +1,5 @@
11 /*
2
- * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved.
2
+ * Copyright (c) 2006, 2019 Oracle and/or its affiliates. All rights reserved.
33  *
44  * This software is available to you under a choice of one of two
55  * licenses.  You may choose to be licensed under the terms of the GNU
....@@ -36,10 +36,12 @@
3636 #include <linux/vmalloc.h>
3737 #include <linux/ratelimit.h>
3838 #include <net/addrconf.h>
39
+#include <rdma/ib_cm.h>
3940 
4041 #include "rds_single_path.h"
4142 #include "rds.h"
4243 #include "ib.h"
44
+#include "ib_mr.h"
4345 
4446 /*
4547  * Set the selected protocol version
....@@ -133,22 +135,26 @@
133135 		rds_ib_set_flow_control(conn, be32_to_cpu(credit));
134136 	}
135137 
136
-	if (conn->c_version < RDS_PROTOCOL(3, 1)) {
137
-		pr_notice("RDS/IB: Connection <%pI6c,%pI6c> version %u.%u no longer supported\n",
138
-			  &conn->c_laddr, &conn->c_faddr,
139
-			  RDS_PROTOCOL_MAJOR(conn->c_version),
140
-			  RDS_PROTOCOL_MINOR(conn->c_version));
141
-		set_bit(RDS_DESTROY_PENDING, &conn->c_path[0].cp_flags);
142
-		rds_conn_destroy(conn);
143
-		return;
144
-	} else {
145
-		pr_notice("RDS/IB: %s conn connected <%pI6c,%pI6c> version %u.%u%s\n",
146
-			  ic->i_active_side ? "Active" : "Passive",
147
-			  &conn->c_laddr, &conn->c_faddr,
148
-			  RDS_PROTOCOL_MAJOR(conn->c_version),
149
-			  RDS_PROTOCOL_MINOR(conn->c_version),
150
-			  ic->i_flowctl ? ", flow control" : "");
138
+	if (conn->c_version < RDS_PROTOCOL_VERSION) {
139
+		if (conn->c_version != RDS_PROTOCOL_COMPAT_VERSION) {
140
+			pr_notice("RDS/IB: Connection <%pI6c,%pI6c> version %u.%u no longer supported\n",
141
+				  &conn->c_laddr, &conn->c_faddr,
142
+				  RDS_PROTOCOL_MAJOR(conn->c_version),
143
+				  RDS_PROTOCOL_MINOR(conn->c_version));
144
+			rds_conn_destroy(conn);
145
+			return;
146
+		}
151147 	}
148
+
149
+	pr_notice("RDS/IB: %s conn connected <%pI6c,%pI6c,%d> version %u.%u%s\n",
150
+		  ic->i_active_side ? "Active" : "Passive",
151
+		  &conn->c_laddr, &conn->c_faddr, conn->c_tos,
152
+		  RDS_PROTOCOL_MAJOR(conn->c_version),
153
+		  RDS_PROTOCOL_MINOR(conn->c_version),
154
+		  ic->i_flowctl ? ", flow control" : "");
155
+
156
+	/* receive sl from the peer */
157
+	ic->i_sl = ic->i_cm_id->route.path_rec->sl;
152158 
153159 	atomic_set(&ic->i_cq_quiesce, 0);
154160 
....@@ -184,6 +190,7 @@
184190 					    NULL);
185191 	}
186192 
193
+	conn->c_proposed_version = conn->c_version;
187194 	rds_connect_complete(conn);
188195 }
189196 
....@@ -220,6 +227,7 @@
220227 			    cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
221228 			dp->ricp_v6.dp_ack_seq =
222229 			    cpu_to_be64(rds_ib_piggyb_ack(ic));
230
+			dp->ricp_v6.dp_cmn.ricpc_dp_toss = conn->c_tos;
223231 
224232 			conn_param->private_data = &dp->ricp_v6;
225233 			conn_param->private_data_len = sizeof(dp->ricp_v6);
....@@ -234,6 +242,7 @@
234242 			    cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
235243 			dp->ricp_v4.dp_ack_seq =
236244 			    cpu_to_be64(rds_ib_piggyb_ack(ic));
245
+			dp->ricp_v4.dp_cmn.ricpc_dp_toss = conn->c_tos;
237246 
238247 			conn_param->private_data = &dp->ricp_v4;
239248 			conn_param->private_data_len = sizeof(dp->ricp_v4);
....@@ -389,10 +398,9 @@
389398 		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
390399 		break;
391400 	default:
392
-		rdsdebug("Fatal QP Event %u (%s) "
393
-			"- connection %pI6c->%pI6c, reconnecting\n",
394
-			event->event, ib_event_msg(event->event),
395
-			&conn->c_laddr, &conn->c_faddr);
401
+		rdsdebug("Fatal QP Event %u (%s) - connection %pI6c->%pI6c, reconnecting\n",
402
+			 event->event, ib_event_msg(event->event),
403
+			 &conn->c_laddr, &conn->c_faddr);
396404 		rds_conn_drop(conn);
397405 		break;
398406 	}
....@@ -432,6 +440,95 @@
432440 	rds_ibdev->vector_load[index]--;
433441 }
434442 
443
+static void rds_dma_hdr_free(struct ib_device *dev, struct rds_header *hdr,
444
+		dma_addr_t dma_addr, enum dma_data_direction dir)
445
+{
446
+	ib_dma_unmap_single(dev, dma_addr, sizeof(*hdr), dir);
447
+	kfree(hdr);
448
+}
449
+
450
+static struct rds_header *rds_dma_hdr_alloc(struct ib_device *dev,
451
+		dma_addr_t *dma_addr, enum dma_data_direction dir)
452
+{
453
+	struct rds_header *hdr;
454
+
455
+	hdr = kzalloc_node(sizeof(*hdr), GFP_KERNEL, ibdev_to_node(dev));
456
+	if (!hdr)
457
+		return NULL;
458
+
459
+	*dma_addr = ib_dma_map_single(dev, hdr, sizeof(*hdr),
460
+				      DMA_BIDIRECTIONAL);
461
+	if (ib_dma_mapping_error(dev, *dma_addr)) {
462
+		kfree(hdr);
463
+		return NULL;
464
+	}
465
+
466
+	return hdr;
467
+}
468
+
469
+/* Free the DMA memory used to store struct rds_header.
470
+ *
471
+ * @dev: the RDS IB device
472
+ * @hdrs: pointer to the array storing DMA memory pointers
473
+ * @dma_addrs: pointer to the array storing DMA addresses
474
+ * @num_hdars: number of headers to free.
475
+ */
476
+static void rds_dma_hdrs_free(struct rds_ib_device *dev,
477
+		struct rds_header **hdrs, dma_addr_t *dma_addrs, u32 num_hdrs,
478
+		enum dma_data_direction dir)
479
+{
480
+	u32 i;
481
+
482
+	for (i = 0; i < num_hdrs; i++)
483
+		rds_dma_hdr_free(dev->dev, hdrs[i], dma_addrs[i], dir);
484
+	kvfree(hdrs);
485
+	kvfree(dma_addrs);
486
+}
487
+
488
+
489
+/* Allocate DMA coherent memory to be used to store struct rds_header for
490
+ * sending/receiving packets.  The pointers to the DMA memory and the
491
+ * associated DMA addresses are stored in two arrays.
492
+ *
493
+ * @dev: the RDS IB device
494
+ * @dma_addrs: pointer to the array for storing DMA addresses
495
+ * @num_hdrs: number of headers to allocate
496
+ *
497
+ * It returns the pointer to the array storing the DMA memory pointers.  On
498
+ * error, NULL pointer is returned.
499
+ */
500
+static struct rds_header **rds_dma_hdrs_alloc(struct rds_ib_device *dev,
501
+		dma_addr_t **dma_addrs, u32 num_hdrs,
502
+		enum dma_data_direction dir)
503
+{
504
+	struct rds_header **hdrs;
505
+	dma_addr_t *hdr_daddrs;
506
+	u32 i;
507
+
508
+	hdrs = kvmalloc_node(sizeof(*hdrs) * num_hdrs, GFP_KERNEL,
509
+			     ibdev_to_node(dev->dev));
510
+	if (!hdrs)
511
+		return NULL;
512
+
513
+	hdr_daddrs = kvmalloc_node(sizeof(*hdr_daddrs) * num_hdrs, GFP_KERNEL,
514
+				   ibdev_to_node(dev->dev));
515
+	if (!hdr_daddrs) {
516
+		kvfree(hdrs);
517
+		return NULL;
518
+	}
519
+
520
+	for (i = 0; i < num_hdrs; i++) {
521
+		hdrs[i] = rds_dma_hdr_alloc(dev->dev, &hdr_daddrs[i], dir);
522
+		if (!hdrs[i]) {
523
+			rds_dma_hdrs_free(dev, hdrs, hdr_daddrs, i, dir);
524
+			return NULL;
525
+		}
526
+	}
527
+
528
+	*dma_addrs = hdr_daddrs;
529
+	return hdrs;
530
+}
531
+
435532 /*
436533  * This needs to be very careful to not leave IS_ERR pointers around for
437534  * cleanup to trip over.
....@@ -443,6 +540,7 @@
443540 	struct ib_qp_init_attr attr;
444541 	struct ib_cq_init_attr cq_attr = {};
445542 	struct rds_ib_device *rds_ibdev;
543
+	unsigned long max_wrs;
446544 	int ret, fr_queue_space;
447545 
448546 	/*
....@@ -454,21 +552,23 @@
454552 		return -EOPNOTSUPP;
455553 
456554 	/* The fr_queue_space is currently set to 512, to add extra space on
457
-	 * completion queue and send queue. This extra space is used for FRMR
555
+	 * completion queue and send queue. This extra space is used for FRWR
458556 	 * registration and invalidation work requests
459557 	 */
460
-	fr_queue_space = rds_ibdev->use_fastreg ?
461
-			 (RDS_IB_DEFAULT_FR_WR + 1) +
462
-			 (RDS_IB_DEFAULT_FR_INV_WR + 1)
463
-			 : 0;
558
+	fr_queue_space = RDS_IB_DEFAULT_FR_WR;
464559 
465560 	/* add the conn now so that connection establishment has the dev */
466561 	rds_ib_add_conn(rds_ibdev, conn);
467562 
468
-	if (rds_ibdev->max_wrs < ic->i_send_ring.w_nr + 1)
469
-		rds_ib_ring_resize(&ic->i_send_ring, rds_ibdev->max_wrs - 1);
470
-	if (rds_ibdev->max_wrs < ic->i_recv_ring.w_nr + 1)
471
-		rds_ib_ring_resize(&ic->i_recv_ring, rds_ibdev->max_wrs - 1);
563
+	max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_send_wr + 1 ?
564
+		rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_send_wr;
565
+	if (ic->i_send_ring.w_nr != max_wrs)
566
+		rds_ib_ring_resize(&ic->i_send_ring, max_wrs);
567
+
568
+	max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_recv_wr + 1 ?
569
+		rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_recv_wr;
570
+	if (ic->i_recv_ring.w_nr != max_wrs)
571
+		rds_ib_ring_resize(&ic->i_recv_ring, max_wrs);
472572 
473573 	/* Protection domain and memory range */
474574 	ic->i_pd = rds_ibdev->pd;
....@@ -526,8 +626,6 @@
526626 	attr.qp_type = IB_QPT_RC;
527627 	attr.send_cq = ic->i_send_cq;
528628 	attr.recv_cq = ic->i_recv_cq;
529
-	atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);
530
-	atomic_set(&ic->i_fastunreg_wrs, RDS_IB_DEFAULT_FR_INV_WR);
531629 
532630 	/*
533631 	 * XXX this can fail if max_*_wr is too large?  Are we supposed
....@@ -539,31 +637,29 @@
539637 		goto recv_cq_out;
540638 	}
541639 
542
-	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,
543
-					   ic->i_send_ring.w_nr *
544
-						sizeof(struct rds_header),
545
-					   &ic->i_send_hdrs_dma, GFP_KERNEL);
640
+	ic->i_send_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_send_hdrs_dma,
641
+					     ic->i_send_ring.w_nr,
642
+					     DMA_TO_DEVICE);
546643 	if (!ic->i_send_hdrs) {
547644 		ret = -ENOMEM;
548
-		rdsdebug("ib_dma_alloc_coherent send failed\n");
645
+		rdsdebug("DMA send hdrs alloc failed\n");
549646 		goto qp_out;
550647 	}
551648 
552
-	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,
553
-					   ic->i_recv_ring.w_nr *
554
-						sizeof(struct rds_header),
555
-					   &ic->i_recv_hdrs_dma, GFP_KERNEL);
649
+	ic->i_recv_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_recv_hdrs_dma,
650
+					     ic->i_recv_ring.w_nr,
651
+					     DMA_FROM_DEVICE);
556652 	if (!ic->i_recv_hdrs) {
557653 		ret = -ENOMEM;
558
-		rdsdebug("ib_dma_alloc_coherent recv failed\n");
654
+		rdsdebug("DMA recv hdrs alloc failed\n");
559655 		goto send_hdrs_dma_out;
560656 	}
561657 
562
-	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),
563
-				       &ic->i_ack_dma, GFP_KERNEL);
658
+	ic->i_ack = rds_dma_hdr_alloc(rds_ibdev->dev, &ic->i_ack_dma,
659
+				      DMA_TO_DEVICE);
564660 	if (!ic->i_ack) {
565661 		ret = -ENOMEM;
566
-		rdsdebug("ib_dma_alloc_coherent ack failed\n");
662
+		rdsdebug("DMA ack header alloc failed\n");
567663 		goto recv_hdrs_dma_out;
568664 	}
569665 
....@@ -594,25 +690,32 @@
594690 
595691 sends_out:
596692 	vfree(ic->i_sends);
693
+
597694 ack_dma_out:
598
-	ib_dma_free_coherent(dev, sizeof(struct rds_header),
599
-			     ic->i_ack, ic->i_ack_dma);
695
+	rds_dma_hdr_free(rds_ibdev->dev, ic->i_ack, ic->i_ack_dma,
696
+			 DMA_TO_DEVICE);
697
+	ic->i_ack = NULL;
698
+
600699 recv_hdrs_dma_out:
601
-	ib_dma_free_coherent(dev, ic->i_recv_ring.w_nr *
602
-					sizeof(struct rds_header),
603
-					ic->i_recv_hdrs, ic->i_recv_hdrs_dma);
700
+	rds_dma_hdrs_free(rds_ibdev, ic->i_recv_hdrs, ic->i_recv_hdrs_dma,
701
+			  ic->i_recv_ring.w_nr, DMA_FROM_DEVICE);
702
+	ic->i_recv_hdrs = NULL;
703
+	ic->i_recv_hdrs_dma = NULL;
704
+
604705 send_hdrs_dma_out:
605
-	ib_dma_free_coherent(dev, ic->i_send_ring.w_nr *
606
-					sizeof(struct rds_header),
607
-					ic->i_send_hdrs, ic->i_send_hdrs_dma);
706
+	rds_dma_hdrs_free(rds_ibdev, ic->i_send_hdrs, ic->i_send_hdrs_dma,
707
+			  ic->i_send_ring.w_nr, DMA_TO_DEVICE);
708
+	ic->i_send_hdrs = NULL;
709
+	ic->i_send_hdrs_dma = NULL;
710
+
608711 qp_out:
609712 	rdma_destroy_qp(ic->i_cm_id);
610713 recv_cq_out:
611
-	if (!ib_destroy_cq(ic->i_recv_cq))
612
-		ic->i_recv_cq = NULL;
714
+	ib_destroy_cq(ic->i_recv_cq);
715
+	ic->i_recv_cq = NULL;
613716 send_cq_out:
614
-	if (!ib_destroy_cq(ic->i_send_cq))
615
-		ic->i_send_cq = NULL;
717
+	ib_destroy_cq(ic->i_send_cq);
718
+	ic->i_send_cq = NULL;
616719 rds_ibdev_out:
617720 	rds_ib_remove_conn(rds_ibdev, conn);
618721 out:
....@@ -635,7 +738,7 @@
635738 	 * original size. The only way to tell the difference is by looking at
636739 	 * the contents, which are initialized to zero.
637740 	 * If the protocol version fields aren't set, this is a connection attempt
638
-	 * from an older version. This could could be 3.0 or 2.0 - we can't tell.
741
+	 * from an older version. This could be 3.0 or 2.0 - we can't tell.
639742 	 * We really should have changed this for OFED 1.3 :-(
640743 	 */
641744 
....@@ -660,13 +763,16 @@
660763 
661764 	/* Even if len is crap *now* I still want to check it. -ASG */
662765 	if (event->param.conn.private_data_len < data_len || major == 0)
663
-		return RDS_PROTOCOL_3_0;
766
+		return RDS_PROTOCOL_4_0;
664767 
665768 	common = be16_to_cpu(mask) & RDS_IB_SUPPORTED_PROTOCOLS;
666
-	if (major == 3 && common) {
667
-		version = RDS_PROTOCOL_3_0;
769
+	if (major == 4 && common) {
770
+		version = RDS_PROTOCOL_4_0;
668771 		while ((common >>= 1) != 0)
669772 			version++;
773
+	} else if (RDS_PROTOCOL_COMPAT_VERSION ==
774
+		   RDS_PROTOCOL(major, minor)) {
775
+		version = RDS_PROTOCOL_COMPAT_VERSION;
670776 	} else {
671777 		if (isv6)
672778 			printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI6c using incompatible protocol version %u.%u\n",
....@@ -729,8 +835,10 @@
729835 
730836 	/* Check whether the remote protocol version matches ours. */
731837 	version = rds_ib_protocol_compatible(event, isv6);
732
-	if (!version)
838
+	if (!version) {
839
+		err = RDS_RDMA_REJ_INCOMPAT;
733840 		goto out;
841
+	}
734842 
735843 	dp = event->param.conn.private_data;
736844 	if (isv6) {
....@@ -771,15 +879,16 @@
771879 		daddr6 = &d_mapped_addr;
772880 	}
773881 
774
-	rdsdebug("saddr %pI6c daddr %pI6c RDSv%u.%u lguid 0x%llx fguid "
775
-		 "0x%llx\n", saddr6, daddr6,
776
-		 RDS_PROTOCOL_MAJOR(version), RDS_PROTOCOL_MINOR(version),
882
+	rdsdebug("saddr %pI6c daddr %pI6c RDSv%u.%u lguid 0x%llx fguid 0x%llx, tos:%d\n",
883
+		 saddr6, daddr6, RDS_PROTOCOL_MAJOR(version),
884
+		 RDS_PROTOCOL_MINOR(version),
777885 		 (unsigned long long)be64_to_cpu(lguid),
778
-		 (unsigned long long)be64_to_cpu(fguid));
886
+		 (unsigned long long)be64_to_cpu(fguid), dp_cmn->ricpc_dp_toss);
779887 
780888 	/* RDS/IB is not currently netns aware, thus init_net */
781889 	conn = rds_conn_create(&init_net, daddr6, saddr6,
782
-			       &rds_ib_transport, GFP_KERNEL, ifindex);
890
+			       &rds_ib_transport, dp_cmn->ricpc_dp_toss,
891
+			       GFP_KERNEL, ifindex);
783892 	if (IS_ERR(conn)) {
784893 		rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn));
785894 		conn = NULL;
....@@ -846,7 +955,8 @@
846955 	if (conn)
847956 		mutex_unlock(&conn->c_cm_lock);
848957 	if (err)
849
-		rdma_reject(cm_id, NULL, 0);
958
+		rdma_reject(cm_id, &err, sizeof(int),
959
+			    IB_CM_REJ_CONSUMER_DEFINED);
850960 	return destroy;
851961 }
852962 
....@@ -861,7 +971,7 @@
861971 
862972 	/* If the peer doesn't do protocol negotiation, we must
863973 	 * default to RDSv3.0 */
864
-	rds_ib_set_protocol(conn, RDS_PROTOCOL_3_0);
974
+	rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1);
865975 	ic->i_flowctl = rds_ib_sysctl_flow_control;	/* advertise flow control */
866976 
867977 	ret = rds_ib_setup_qp(conn);
....@@ -870,11 +980,13 @@
870980 		goto out;
871981 	}
872982 
873
-	rds_ib_cm_fill_conn_param(conn, &conn_param, &dp, RDS_PROTOCOL_VERSION,
983
+	rds_ib_cm_fill_conn_param(conn, &conn_param, &dp,
984
+				  conn->c_proposed_version,
874985 				  UINT_MAX, UINT_MAX, isv6);
875
-	ret = rdma_connect(cm_id, &conn_param);
986
+	ret = rdma_connect_locked(cm_id, &conn_param);
876987 	if (ret)
877
-		rds_ib_conn_error(conn, "rdma_connect failed (%d)\n", ret);
988
+		rds_ib_conn_error(conn, "rdma_connect_locked failed (%d)\n",
989
+				  ret);
878990 
879991 out:
880992 	/* Beware - returning non-zero tells the rdma_cm to destroy
....@@ -975,8 +1087,6 @@
9751087 		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);
9761088 
9771089 	if (ic->i_cm_id) {
978
-		struct ib_device *dev = ic->i_cm_id->device;
979
-
9801090 		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);
9811091 		err = rdma_disconnect(ic->i_cm_id);
9821092 		if (err) {
....@@ -986,6 +1096,11 @@
9861096 			rdsdebug("failed to disconnect, cm: %p err %d\n",
9871097 				ic->i_cm_id, err);
9881098 		}
1099
+
1100
+		/* kick off "flush_worker" for all pools in order to reap
1101
+		 * all FRMR registrations that are still marked "FRMR_IS_INUSE"
1102
+		 */
1103
+		rds_ib_flush_mrs();
9891104 
9901105 		/*
9911106 		 * We want to wait for tx and rx completion to finish
....@@ -999,8 +1114,8 @@
9991114 		wait_event(rds_ib_ring_empty_wait,
10001115 			   rds_ib_ring_empty(&ic->i_recv_ring) &&
10011116 			   (atomic_read(&ic->i_signaled_sends) == 0) &&
1002
-			   (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR) &&
1003
-			   (atomic_read(&ic->i_fastunreg_wrs) == RDS_IB_DEFAULT_FR_INV_WR));
1117
+			   (atomic_read(&ic->i_fastreg_inuse_count) == 0) &&
1118
+			   (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR));
10041119 		tasklet_kill(&ic->i_send_tasklet);
10051120 		tasklet_kill(&ic->i_recv_tasklet);
10061121 
....@@ -1021,24 +1136,40 @@
10211136 			ib_destroy_cq(ic->i_recv_cq);
10221137 		}
10231138 
1024
-		/* then free the resources that ib callbacks use */
1025
-		if (ic->i_send_hdrs)
1026
-			ib_dma_free_coherent(dev,
1027
-					   ic->i_send_ring.w_nr *
1028
-						sizeof(struct rds_header),
1029
-					   ic->i_send_hdrs,
1030
-					   ic->i_send_hdrs_dma);
1139
+		if (ic->rds_ibdev) {
1140
+			/* then free the resources that ib callbacks use */
1141
+			if (ic->i_send_hdrs) {
1142
+				rds_dma_hdrs_free(ic->rds_ibdev,
1143
+						  ic->i_send_hdrs,
1144
+						  ic->i_send_hdrs_dma,
1145
+						  ic->i_send_ring.w_nr,
1146
+						  DMA_TO_DEVICE);
1147
+				ic->i_send_hdrs = NULL;
1148
+				ic->i_send_hdrs_dma = NULL;
1149
+			}
10311150 
1032
-		if (ic->i_recv_hdrs)
1033
-			ib_dma_free_coherent(dev,
1034
-					   ic->i_recv_ring.w_nr *
1035
-						sizeof(struct rds_header),
1036
-					   ic->i_recv_hdrs,
1037
-					   ic->i_recv_hdrs_dma);
1151
+			if (ic->i_recv_hdrs) {
1152
+				rds_dma_hdrs_free(ic->rds_ibdev,
1153
+						  ic->i_recv_hdrs,
1154
+						  ic->i_recv_hdrs_dma,
1155
+						  ic->i_recv_ring.w_nr,
1156
+						  DMA_FROM_DEVICE);
1157
+				ic->i_recv_hdrs = NULL;
1158
+				ic->i_recv_hdrs_dma = NULL;
1159
+			}
10381160 
1039
-		if (ic->i_ack)
1040
-			ib_dma_free_coherent(dev, sizeof(struct rds_header),
1041
-					     ic->i_ack, ic->i_ack_dma);
1161
+			if (ic->i_ack) {
1162
+				rds_dma_hdr_free(ic->rds_ibdev->dev, ic->i_ack,
1163
+						 ic->i_ack_dma, DMA_TO_DEVICE);
1164
+				ic->i_ack = NULL;
1165
+			}
1166
+		} else {
1167
+			WARN_ON(ic->i_send_hdrs);
1168
+			WARN_ON(ic->i_send_hdrs_dma);
1169
+			WARN_ON(ic->i_recv_hdrs);
1170
+			WARN_ON(ic->i_recv_hdrs_dma);
1171
+			WARN_ON(ic->i_ack);
1172
+		}
10421173 
10431174 		if (ic->i_sends)
10441175 			rds_ib_send_clear_ring(ic);
....@@ -1057,9 +1188,6 @@
10571188 		ic->i_pd = NULL;
10581189 		ic->i_send_cq = NULL;
10591190 		ic->i_recv_cq = NULL;
1060
-		ic->i_send_hdrs = NULL;
1061
-		ic->i_recv_hdrs = NULL;
1062
-		ic->i_ack = NULL;
10631191 	}
10641192 	BUG_ON(ic->rds_ibdev);
10651193 
....@@ -1085,8 +1213,9 @@
10851213 	ic->i_flowctl = 0;
10861214 	atomic_set(&ic->i_credits, 0);
10871215 
1088
-	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);
1089
-	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);
1216
+	/* Re-init rings, but retain sizes. */
1217
+	rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr);
1218
+	rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr);
10901219 
10911220 	if (ic->i_ibinc) {
10921221 		rds_inc_put(&ic->i_ibinc->ii_inc);
....@@ -1127,13 +1256,14 @@
11271256 	spin_lock_init(&ic->i_ack_lock);
11281257 #endif
11291258 	atomic_set(&ic->i_signaled_sends, 0);
1259
+	atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);
11301260 
11311261 	/*
11321262 	 * rds_ib_conn_shutdown() waits for these to be emptied so they
11331263 	 * must be initialized before it can be called.
11341264 	 */
1135
-	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);
1136
-	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);
1265
+	rds_ib_ring_init(&ic->i_send_ring, 0);
1266
+	rds_ib_ring_init(&ic->i_recv_ring, 0);
11371267 
11381268 	ic->conn = conn;
11391269 	conn->c_transport_data = ic;