~hc/RK356X_SDK_RELEASE.git

..	..	@@ -33,11 +33,14 @@
33	33	#include <linux/acpi.h>
34	34	#include <linux/etherdevice.h>
35	35	#include <linux/interrupt.h>
	36	+#include <linux/iopoll.h>
36	37	#include <linux/kernel.h>
37	38	#include <linux/types.h>
38	39	#include <net/addrconf.h>
39	40	#include <rdma/ib_addr.h>
	41	+#include <rdma/ib_cache.h>
40	42	#include <rdma/ib_umem.h>
	43	+#include <rdma/uverbs_ioctl.h>
41	44
42	45	#include "hnae3.h"
43	46	#include "hns_roce_common.h"
..	..	@@ -54,166 +57,604 @@
54	57	dseg->len = cpu_to_le32(sg->length);
55	58	}
56	59
57		-static void set_extend_sge(struct hns_roce_qp qp, const struct ib_send_wr wr,
58		- unsigned int *sge_ind)
	60	+/*
	61	+ * mapped-value = 1 + real-value
	62	+ * The hns wr opcode real value is start from 0, In order to distinguish between
	63	+ * initialized and uninitialized map values, we plus 1 to the actual value when
	64	+ * defining the mapping, so that the validity can be identified by checking the
	65	+ * mapped value is greater than 0.
	66	+ */
	67	+#define HR_OPC_MAP(ib_key, hr_key) \
	68	+ [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key
	69	+
	70	+static const u32 hns_roce_op_code[] = {
	71	+ HR_OPC_MAP(RDMA_WRITE, RDMA_WRITE),
	72	+ HR_OPC_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE_WITH_IMM),
	73	+ HR_OPC_MAP(SEND, SEND),
	74	+ HR_OPC_MAP(SEND_WITH_IMM, SEND_WITH_IMM),
	75	+ HR_OPC_MAP(RDMA_READ, RDMA_READ),
	76	+ HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP),
	77	+ HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD),
	78	+ HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV),
	79	+ HR_OPC_MAP(LOCAL_INV, LOCAL_INV),
	80	+ HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP),
	81	+ HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD),
	82	+ HR_OPC_MAP(REG_MR, FAST_REG_PMR),
	83	+};
	84	+
	85	+static u32 to_hr_opcode(u32 ib_opcode)
59	86	{
60		- struct hns_roce_v2_wqe_data_seg *dseg;
61		- struct ib_sge *sg;
62		- int num_in_wqe = 0;
63		- int extend_sge_num;
64		- int fi_sge_num;
65		- int se_sge_num;
66		- int shift;
67		- int i;
	87	+ if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code))
	88	+ return HNS_ROCE_V2_WQE_OP_MASK;
68	89
69		- if (qp->ibqp.qp_type == IB_QPT_RC \|\| qp->ibqp.qp_type == IB_QPT_UC)
70		- num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
71		- extend_sge_num = wr->num_sge - num_in_wqe;
72		- sg = wr->sg_list + num_in_wqe;
73		- shift = qp->hr_buf.page_shift;
	90	+ return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :
	91	+ HNS_ROCE_V2_WQE_OP_MASK;
	92	+}
74	93
75		- /*
76		- * Check whether wr->num_sge sges are in the same page. If not, we
77		- * should calculate how many sges in the first page and the second
78		- * page.
79		- */
80		- dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
81		- fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
82		- (uintptr_t)dseg) /
83		- sizeof(struct hns_roce_v2_wqe_data_seg);
84		- if (extend_sge_num > fi_sge_num) {
85		- se_sge_num = extend_sge_num - fi_sge_num;
86		- for (i = 0; i < fi_sge_num; i++) {
87		- set_data_seg_v2(dseg++, sg + i);
88		- (*sge_ind)++;
89		- }
90		- dseg = get_send_extend_sge(qp,
91		- (*sge_ind) & (qp->sge.sge_cnt - 1));
92		- for (i = 0; i < se_sge_num; i++) {
93		- set_data_seg_v2(dseg++, sg + fi_sge_num + i);
94		- (*sge_ind)++;
95		- }
	94	+static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
	95	+ const struct ib_reg_wr *wr)
	96	+{
	97	+ struct hns_roce_wqe_frmr_seg *fseg =
	98	+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
	99	+ struct hns_roce_mr *mr = to_hr_mr(wr->mr);
	100	+ u64 pbl_ba;
	101	+
	102	+ /* use ib_access_flags */
	103	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
	104	+ wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
	105	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
	106	+ wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
	107	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RR_S,
	108	+ wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
	109	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RW_S,
	110	+ wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
	111	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_LW_S,
	112	+ wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
	113	+
	114	+ /* Data structure reuse may lead to confusion */
	115	+ pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
	116	+ rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));
	117	+ rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));
	118	+
	119	+ rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
	120	+ rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
	121	+ rc_sq_wqe->rkey = cpu_to_le32(wr->key);
	122	+ rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
	123	+
	124	+ fseg->pbl_size = cpu_to_le32(mr->npages);
	125	+ roce_set_field(fseg->mode_buf_pg_sz,
	126	+ V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
	127	+ V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
	128	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
	129	+ roce_set_bit(fseg->mode_buf_pg_sz,
	130	+ V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
	131	+}
	132	+
	133	+static void set_atomic_seg(const struct ib_send_wr *wr,
	134	+ struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
	135	+ unsigned int valid_num_sge)
	136	+{
	137	+ struct hns_roce_v2_wqe_data_seg *dseg =
	138	+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
	139	+ struct hns_roce_wqe_atomic_seg *aseg =
	140	+ (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);
	141	+
	142	+ set_data_seg_v2(dseg, wr->sg_list);
	143	+
	144	+ if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
	145	+ aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
	146	+ aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add);
96	147	} else {
97		- for (i = 0; i < extend_sge_num; i++) {
98		- set_data_seg_v2(dseg++, sg + i);
99		- (*sge_ind)++;
	148	+ aseg->fetchadd_swap_data =
	149	+ cpu_to_le64(atomic_wr(wr)->compare_add);
	150	+ aseg->cmp_data = 0;
	151	+ }
	152	+
	153	+ roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
	154	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
	155	+}
	156	+
	157	+static int fill_ext_sge_inl_data(struct hns_roce_qp *qp,
	158	+ const struct ib_send_wr *wr,
	159	+ unsigned int *sge_idx, u32 msg_len)
	160	+{
	161	+ struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;
	162	+ unsigned int dseg_len = sizeof(struct hns_roce_v2_wqe_data_seg);
	163	+ unsigned int ext_sge_sz = qp->sq.max_gs * dseg_len;
	164	+ unsigned int left_len_in_pg;
	165	+ unsigned int idx = *sge_idx;
	166	+ unsigned int i = 0;
	167	+ unsigned int len;
	168	+ void *addr;
	169	+ void *dseg;
	170	+
	171	+ if (msg_len > ext_sge_sz) {
	172	+ ibdev_err(ibdev,
	173	+ "no enough extended sge space for inline data.\n");
	174	+ return -EINVAL;
	175	+ }
	176	+
	177	+ dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
	178	+ left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg;
	179	+ len = wr->sg_list[0].length;
	180	+ addr = (void *)(unsigned long)(wr->sg_list[0].addr);
	181	+
	182	+ /* When copying data to extended sge space, the left length in page may
	183	+ * not long enough for current user's sge. So the data should be
	184	+ * splited into several parts, one in the first page, and the others in
	185	+ * the subsequent pages.
	186	+ */
	187	+ while (1) {
	188	+ if (len <= left_len_in_pg) {
	189	+ memcpy(dseg, addr, len);
	190	+
	191	+ idx += len / dseg_len;
	192	+
	193	+ i++;
	194	+ if (i >= wr->num_sge)
	195	+ break;
	196	+
	197	+ left_len_in_pg -= len;
	198	+ len = wr->sg_list[i].length;
	199	+ addr = (void *)(unsigned long)(wr->sg_list[i].addr);
	200	+ dseg += len;
	201	+ } else {
	202	+ memcpy(dseg, addr, left_len_in_pg);
	203	+
	204	+ len -= left_len_in_pg;
	205	+ addr += left_len_in_pg;
	206	+ idx += left_len_in_pg / dseg_len;
	207	+ dseg = hns_roce_get_extend_sge(qp,
	208	+ idx & (qp->sge.sge_cnt - 1));
	209	+ left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT;
100	210	}
101	211	}
	212	+
	213	+ *sge_idx = idx;
	214	+
	215	+ return 0;
	216	+}
	217	+
	218	+static void set_extend_sge(struct hns_roce_qp qp, struct ib_sge sge,
	219	+ unsigned int *sge_ind, unsigned int cnt)
	220	+{
	221	+ struct hns_roce_v2_wqe_data_seg *dseg;
	222	+ unsigned int idx = *sge_ind;
	223	+
	224	+ while (cnt > 0) {
	225	+ dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
	226	+ if (likely(sge->length)) {
	227	+ set_data_seg_v2(dseg, sge);
	228	+ idx++;
	229	+ cnt--;
	230	+ }
	231	+ sge++;
	232	+ }
	233	+
	234	+ *sge_ind = idx;
	235	+}
	236	+
	237	+static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len)
	238	+{
	239	+ struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
	240	+ int mtu = ib_mtu_enum_to_int(qp->path_mtu);
	241	+
	242	+ if (len > qp->max_inline_data \|\| len > mtu) {
	243	+ ibdev_err(&hr_dev->ib_dev,
	244	+ "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n",
	245	+ len, qp->max_inline_data, mtu);
	246	+ return false;
	247	+ }
	248	+
	249	+ return true;
	250	+}
	251	+
	252	+static int set_rc_inl(struct hns_roce_qp qp, const struct ib_send_wr wr,
	253	+ struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
	254	+ unsigned int *sge_idx)
	255	+{
	256	+ struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
	257	+ u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len);
	258	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	259	+ unsigned int curr_idx = *sge_idx;
	260	+ void *dseg = rc_sq_wqe;
	261	+ unsigned int i;
	262	+ int ret;
	263	+
	264	+ if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {
	265	+ ibdev_err(ibdev, "invalid inline parameters!\n");
	266	+ return -EINVAL;
	267	+ }
	268	+
	269	+ if (!check_inl_data_len(qp, msg_len))
	270	+ return -EINVAL;
	271	+
	272	+ dseg += sizeof(struct hns_roce_v2_rc_send_wqe);
	273	+
	274	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, 1);
	275	+
	276	+ if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) {
	277	+ roce_set_bit(rc_sq_wqe->byte_20,
	278	+ V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 0);
	279	+
	280	+ for (i = 0; i < wr->num_sge; i++) {
	281	+ memcpy(dseg, ((void *)wr->sg_list[i].addr),
	282	+ wr->sg_list[i].length);
	283	+ dseg += wr->sg_list[i].length;
	284	+ }
	285	+ } else {
	286	+ roce_set_bit(rc_sq_wqe->byte_20,
	287	+ V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 1);
	288	+
	289	+ ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len);
	290	+ if (ret)
	291	+ return ret;
	292	+
	293	+ roce_set_field(rc_sq_wqe->byte_16,
	294	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
	295	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
	296	+ curr_idx - *sge_idx);
	297	+ }
	298	+
	299	+ *sge_idx = curr_idx;
	300	+
	301	+ return 0;
102	302	}
103	303
104	304	static int set_rwqe_data_seg(struct ib_qp ibqp, const struct ib_send_wr wr,
105	305	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
106		- void wqe, unsigned int sge_ind,
107		- const struct ib_send_wr **bad_wr)
	306	+ unsigned int *sge_ind,
	307	+ unsigned int valid_num_sge)
108	308	{
109		- struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
110		- struct hns_roce_v2_wqe_data_seg *dseg = wqe;
	309	+ struct hns_roce_v2_wqe_data_seg *dseg =
	310	+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
111	311	struct hns_roce_qp *qp = to_hr_qp(ibqp);
	312	+ int j = 0;
112	313	int i;
113	314
114		- if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
115		- if (le32_to_cpu(rc_sq_wqe->msg_len) >
116		- hr_dev->caps.max_sq_inline) {
117		- *bad_wr = wr;
118		- dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
119		- rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
120		- return -EINVAL;
121		- }
	315	+ roce_set_field(rc_sq_wqe->byte_20,
	316	+ V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
	317	+ V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
	318	+ (*sge_ind) & (qp->sge.sge_cnt - 1));
122	319
123		- if (wr->opcode == IB_WR_RDMA_READ) {
124		- *bad_wr = wr;
125		- dev_err(hr_dev->dev, "Not support inline data!\n");
126		- return -EINVAL;
127		- }
	320	+ if (wr->send_flags & IB_SEND_INLINE)
	321	+ return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind);
128	322
	323	+ if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {
129	324	for (i = 0; i < wr->num_sge; i++) {
130		- memcpy(wqe, ((void *)wr->sg_list[i].addr),
131		- wr->sg_list[i].length);
132		- wqe += wr->sg_list[i].length;
	325	+ if (likely(wr->sg_list[i].length)) {
	326	+ set_data_seg_v2(dseg, wr->sg_list + i);
	327	+ dseg++;
	328	+ }
133	329	}
134		-
135		- roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
136		- 1);
137	330	} else {
138		- if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
139		- for (i = 0; i < wr->num_sge; i++) {
140		- if (likely(wr->sg_list[i].length)) {
141		- set_data_seg_v2(dseg, wr->sg_list + i);
142		- dseg++;
143		- }
	331	+ for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) {
	332	+ if (likely(wr->sg_list[i].length)) {
	333	+ set_data_seg_v2(dseg, wr->sg_list + i);
	334	+ dseg++;
	335	+ j++;
144	336	}
145		- } else {
146		- roce_set_field(rc_sq_wqe->byte_20,
147		- V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
148		- V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
149		- (*sge_ind) & (qp->sge.sge_cnt - 1));
150		-
151		- for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
152		- if (likely(wr->sg_list[i].length)) {
153		- set_data_seg_v2(dseg, wr->sg_list + i);
154		- dseg++;
155		- }
156		- }
157		-
158		- set_extend_sge(qp, wr, sge_ind);
159	337	}
160	338
161		- roce_set_field(rc_sq_wqe->byte_16,
162		- V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
163		- V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
	339	+ set_extend_sge(qp, wr->sg_list + i, sge_ind,
	340	+ valid_num_sge - HNS_ROCE_SGE_IN_WQE);
	341	+ }
	342	+
	343	+ roce_set_field(rc_sq_wqe->byte_16,
	344	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
	345	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
	346	+
	347	+ return 0;
	348	+}
	349	+
	350	+static int check_send_valid(struct hns_roce_dev *hr_dev,
	351	+ struct hns_roce_qp *hr_qp)
	352	+{
	353	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	354	+ struct ib_qp *ibqp = &hr_qp->ibqp;
	355	+
	356	+ if (unlikely(ibqp->qp_type != IB_QPT_RC &&
	357	+ ibqp->qp_type != IB_QPT_GSI &&
	358	+ ibqp->qp_type != IB_QPT_UD)) {
	359	+ ibdev_err(ibdev, "Not supported QP(0x%x)type!\n",
	360	+ ibqp->qp_type);
	361	+ return -EOPNOTSUPP;
	362	+ } else if (unlikely(hr_qp->state == IB_QPS_RESET \|\|
	363	+ hr_qp->state == IB_QPS_INIT \|\|
	364	+ hr_qp->state == IB_QPS_RTR)) {
	365	+ ibdev_err(ibdev, "failed to post WQE, QP state %hhu!\n",
	366	+ hr_qp->state);
	367	+ return -EINVAL;
	368	+ } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {
	369	+ ibdev_err(ibdev, "failed to post WQE, dev state %d!\n",
	370	+ hr_dev->state);
	371	+ return -EIO;
164	372	}
165	373
166	374	return 0;
167	375	}
168	376
169		-static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
170		- const struct ib_qp_attr *attr,
171		- int attr_mask, enum ib_qp_state cur_state,
172		- enum ib_qp_state new_state);
	377	+static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,
	378	+ unsigned int *sge_len)
	379	+{
	380	+ unsigned int valid_num = 0;
	381	+ unsigned int len = 0;
	382	+ int i;
	383	+
	384	+ for (i = 0; i < wr->num_sge; i++) {
	385	+ if (likely(wr->sg_list[i].length)) {
	386	+ len += wr->sg_list[i].length;
	387	+ valid_num++;
	388	+ }
	389	+ }
	390	+
	391	+ *sge_len = len;
	392	+ return valid_num;
	393	+}
	394	+
	395	+static __le32 get_immtdata(const struct ib_send_wr *wr)
	396	+{
	397	+ switch (wr->opcode) {
	398	+ case IB_WR_SEND_WITH_IMM:
	399	+ case IB_WR_RDMA_WRITE_WITH_IMM:
	400	+ return cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
	401	+ default:
	402	+ return 0;
	403	+ }
	404	+}
	405	+
	406	+static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
	407	+ const struct ib_send_wr *wr)
	408	+{
	409	+ u32 ib_op = wr->opcode;
	410	+
	411	+ if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM)
	412	+ return -EINVAL;
	413	+
	414	+ ud_sq_wqe->immtdata = get_immtdata(wr);
	415	+
	416	+ roce_set_field(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
	417	+ V2_UD_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op));
	418	+
	419	+ return 0;
	420	+}
	421	+
	422	+static inline int set_ud_wqe(struct hns_roce_qp *qp,
	423	+ const struct ib_send_wr *wr,
	424	+ void wqe, unsigned int sge_idx,
	425	+ unsigned int owner_bit)
	426	+{
	427	+ struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
	428	+ struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
	429	+ struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
	430	+ unsigned int curr_idx = *sge_idx;
	431	+ int valid_num_sge;
	432	+ u32 msg_len = 0;
	433	+ int ret;
	434	+
	435	+ valid_num_sge = calc_wr_sge_num(wr, &msg_len);
	436	+ memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
	437	+
	438	+ ret = set_ud_opcode(ud_sq_wqe, wr);
	439	+ if (WARN_ON(ret))
	440	+ return ret;
	441	+
	442	+ roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
	443	+ V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
	444	+ roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
	445	+ V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
	446	+ roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
	447	+ V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
	448	+ roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
	449	+ V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
	450	+ roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
	451	+ V2_UD_SEND_WQE_BYTE_48_DMAC_4_S, ah->av.mac[4]);
	452	+ roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
	453	+ V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, ah->av.mac[5]);
	454	+
	455	+ ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
	456	+
	457	+ /* Set sig attr */
	458	+ roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S,
	459	+ (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
	460	+
	461	+ /* Set se attr */
	462	+ roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S,
	463	+ (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
	464	+
	465	+ roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S,
	466	+ owner_bit);
	467	+
	468	+ roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M,
	469	+ V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn);
	470	+
	471	+ roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
	472	+ V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
	473	+
	474	+ roce_set_field(ud_sq_wqe->byte_20,
	475	+ V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
	476	+ V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
	477	+ curr_idx & (qp->sge.sge_cnt - 1));
	478	+
	479	+ roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
	480	+ V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
	481	+ ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
	482	+ qp->qkey : ud_wr(wr)->remote_qkey);
	483	+ roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
	484	+ V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn);
	485	+
	486	+ roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
	487	+ V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
	488	+ roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
	489	+ V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
	490	+ roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
	491	+ V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
	492	+ roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
	493	+ V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
	494	+ roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M,
	495	+ V2_UD_SEND_WQE_BYTE_40_PORTN_S, qp->port);
	496	+
	497	+ roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
	498	+ V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index);
	499	+
	500	+ if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) {
	501	+ roce_set_bit(ud_sq_wqe->byte_40,
	502	+ V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
	503	+ ah->av.vlan_en);
	504	+ roce_set_field(ud_sq_wqe->byte_36,
	505	+ V2_UD_SEND_WQE_BYTE_36_VLAN_M,
	506	+ V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
	507	+ }
	508	+
	509	+ memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2);
	510	+
	511	+ set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
	512	+
	513	+ *sge_idx = curr_idx;
	514	+
	515	+ return 0;
	516	+}
	517	+
	518	+static int set_rc_opcode(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
	519	+ const struct ib_send_wr *wr)
	520	+{
	521	+ u32 ib_op = wr->opcode;
	522	+
	523	+ rc_sq_wqe->immtdata = get_immtdata(wr);
	524	+
	525	+ switch (ib_op) {
	526	+ case IB_WR_RDMA_READ:
	527	+ case IB_WR_RDMA_WRITE:
	528	+ case IB_WR_RDMA_WRITE_WITH_IMM:
	529	+ rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
	530	+ rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
	531	+ break;
	532	+ case IB_WR_SEND:
	533	+ case IB_WR_SEND_WITH_IMM:
	534	+ break;
	535	+ case IB_WR_ATOMIC_CMP_AND_SWP:
	536	+ case IB_WR_ATOMIC_FETCH_AND_ADD:
	537	+ rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey);
	538	+ rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr);
	539	+ break;
	540	+ case IB_WR_REG_MR:
	541	+ set_frmr_seg(rc_sq_wqe, reg_wr(wr));
	542	+ break;
	543	+ case IB_WR_LOCAL_INV:
	544	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
	545	+ fallthrough;
	546	+ case IB_WR_SEND_WITH_INV:
	547	+ rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
	548	+ break;
	549	+ default:
	550	+ return -EINVAL;
	551	+ }
	552	+
	553	+ roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
	554	+ V2_RC_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op));
	555	+
	556	+ return 0;
	557	+}
	558	+static inline int set_rc_wqe(struct hns_roce_qp *qp,
	559	+ const struct ib_send_wr *wr,
	560	+ void wqe, unsigned int sge_idx,
	561	+ unsigned int owner_bit)
	562	+{
	563	+ struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
	564	+ unsigned int curr_idx = *sge_idx;
	565	+ unsigned int valid_num_sge;
	566	+ u32 msg_len = 0;
	567	+ int ret;
	568	+
	569	+ valid_num_sge = calc_wr_sge_num(wr, &msg_len);
	570	+ memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
	571	+
	572	+ rc_sq_wqe->msg_len = cpu_to_le32(msg_len);
	573	+
	574	+ ret = set_rc_opcode(rc_sq_wqe, wr);
	575	+ if (WARN_ON(ret))
	576	+ return ret;
	577	+
	578	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S,
	579	+ (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
	580	+
	581	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S,
	582	+ (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
	583	+
	584	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S,
	585	+ (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
	586	+
	587	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
	588	+ owner_bit);
	589	+
	590	+ if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP \|\|
	591	+ wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
	592	+ set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
	593	+ else if (wr->opcode != IB_WR_REG_MR)
	594	+ ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
	595	+ &curr_idx, valid_num_sge);
	596	+
	597	+ *sge_idx = curr_idx;
	598	+
	599	+ return ret;
	600	+}
	601	+
	602	+static inline void update_sq_db(struct hns_roce_dev *hr_dev,
	603	+ struct hns_roce_qp *qp)
	604	+{
	605	+ /*
	606	+ * Hip08 hardware cannot flush the WQEs in SQ if the QP state
	607	+ * gets into errored mode. Hence, as a workaround to this
	608	+ * hardware limitation, driver needs to assist in flushing. But
	609	+ * the flushing operation uses mailbox to convey the QP state to
	610	+ * the hardware and which can sleep due to the mutex protection
	611	+ * around the mailbox calls. Hence, use the deferred flush for
	612	+ * now.
	613	+ */
	614	+ if (qp->state == IB_QPS_ERR) {
	615	+ if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
	616	+ init_flush_work(hr_dev, qp);
	617	+ } else {
	618	+ struct hns_roce_v2_db sq_db = {};
	619	+
	620	+ roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
	621	+ V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
	622	+ roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
	623	+ V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
	624	+ roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
	625	+ V2_DB_PARAMETER_IDX_S, qp->sq.head);
	626	+ roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
	627	+ V2_DB_PARAMETER_SL_S, qp->sl);
	628	+
	629	+ hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
	630	+ }
	631	+}
173	632
174	633	static int hns_roce_v2_post_send(struct ib_qp *ibqp,
175	634	const struct ib_send_wr *wr,
176	635	const struct ib_send_wr **bad_wr)
177	636	{
178	637	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
179		- struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
180		- struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
181		- struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
	638	+ struct ib_device *ibdev = &hr_dev->ib_dev;
182	639	struct hns_roce_qp *qp = to_hr_qp(ibqp);
183		- struct device *dev = hr_dev->dev;
184		- struct hns_roce_v2_db sq_db;
185		- struct ib_qp_attr attr;
186		- unsigned int sge_ind = 0;
	640	+ unsigned long flags = 0;
187	641	unsigned int owner_bit;
188		- unsigned long flags;
189		- unsigned int ind;
	642	+ unsigned int sge_idx;
	643	+ unsigned int wqe_idx;
190	644	void *wqe = NULL;
191		- bool loopback;
192		- int attr_mask;
193		- u32 tmp_len;
194		- int ret = 0;
195		- u8 *smac;
196	645	int nreq;
197		- int i;
198		-
199		- if (unlikely(ibqp->qp_type != IB_QPT_RC &&
200		- ibqp->qp_type != IB_QPT_GSI &&
201		- ibqp->qp_type != IB_QPT_UD)) {
202		- dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
203		- *bad_wr = wr;
204		- return -EOPNOTSUPP;
205		- }
206		-
207		- if (unlikely(qp->state == IB_QPS_RESET \|\| qp->state == IB_QPS_INIT \|\|
208		- qp->state == IB_QPS_RTR)) {
209		- dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
210		- *bad_wr = wr;
211		- return -EINVAL;
212		- }
	646	+ int ret;
213	647
214	648	spin_lock_irqsave(&qp->sq.lock, flags);
215		- ind = qp->sq_next_wqe;
216		- sge_ind = qp->next_sge;
	649	+
	650	+ ret = check_send_valid(hr_dev, qp);
	651	+ if (unlikely(ret)) {
	652	+ *bad_wr = wr;
	653	+ nreq = 0;
	654	+ goto out;
	655	+ }
	656	+
	657	+ sge_idx = qp->next_sge;
217	658
218	659	for (nreq = 0; wr; ++nreq, wr = wr->next) {
219	660	if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
..	..	@@ -222,334 +663,56 @@
222	663	goto out;
223	664	}
224	665
	666	+ wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
	667	+
225	668	if (unlikely(wr->num_sge > qp->sq.max_gs)) {
226		- dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
227		- wr->num_sge, qp->sq.max_gs);
	669	+ ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n",
	670	+ wr->num_sge, qp->sq.max_gs);
228	671	ret = -EINVAL;
229	672	*bad_wr = wr;
230	673	goto out;
231	674	}
232	675
233		- wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
234		- qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
235		- wr->wr_id;
236		-
	676	+ wqe = hns_roce_get_send_wqe(qp, wqe_idx);
	677	+ qp->sq.wrid[wqe_idx] = wr->wr_id;
237	678	owner_bit =
238	679	~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
239		- tmp_len = 0;
240	680
241	681	/* Corresponding to the QP type, wqe process separately */
242		- if (ibqp->qp_type == IB_QPT_GSI) {
243		- ud_sq_wqe = wqe;
244		- memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
	682	+ if (ibqp->qp_type == IB_QPT_GSI)
	683	+ ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
	684	+ else if (ibqp->qp_type == IB_QPT_RC)
	685	+ ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
245	686
246		- roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
247		- V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
248		- roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
249		- V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
250		- roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
251		- V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
252		- roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
253		- V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
254		- roce_set_field(ud_sq_wqe->byte_48,
255		- V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
256		- V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
257		- ah->av.mac[4]);
258		- roce_set_field(ud_sq_wqe->byte_48,
259		- V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
260		- V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
261		- ah->av.mac[5]);
262		-
263		- /* MAC loopback */
264		- smac = (u8 *)hr_dev->dev_addr[qp->port];
265		- loopback = ether_addr_equal_unaligned(ah->av.mac,
266		- smac) ? 1 : 0;
267		-
268		- roce_set_bit(ud_sq_wqe->byte_40,
269		- V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
270		-
271		- roce_set_field(ud_sq_wqe->byte_4,
272		- V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
273		- V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
274		- HNS_ROCE_V2_WQE_OP_SEND);
275		-
276		- for (i = 0; i < wr->num_sge; i++)
277		- tmp_len += wr->sg_list[i].length;
278		-
279		- ud_sq_wqe->msg_len =
280		- cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
281		-
282		- switch (wr->opcode) {
283		- case IB_WR_SEND_WITH_IMM:
284		- case IB_WR_RDMA_WRITE_WITH_IMM:
285		- ud_sq_wqe->immtdata =
286		- cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
287		- break;
288		- default:
289		- ud_sq_wqe->immtdata = 0;
290		- break;
291		- }
292		-
293		- /* Set sig attr */
294		- roce_set_bit(ud_sq_wqe->byte_4,
295		- V2_UD_SEND_WQE_BYTE_4_CQE_S,
296		- (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
297		-
298		- /* Set se attr */
299		- roce_set_bit(ud_sq_wqe->byte_4,
300		- V2_UD_SEND_WQE_BYTE_4_SE_S,
301		- (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
302		-
303		- roce_set_bit(ud_sq_wqe->byte_4,
304		- V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
305		-
306		- roce_set_field(ud_sq_wqe->byte_16,
307		- V2_UD_SEND_WQE_BYTE_16_PD_M,
308		- V2_UD_SEND_WQE_BYTE_16_PD_S,
309		- to_hr_pd(ibqp->pd)->pdn);
310		-
311		- roce_set_field(ud_sq_wqe->byte_16,
312		- V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
313		- V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
314		- wr->num_sge);
315		-
316		- roce_set_field(ud_sq_wqe->byte_20,
317		- V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
318		- V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
319		- sge_ind & (qp->sge.sge_cnt - 1));
320		-
321		- roce_set_field(ud_sq_wqe->byte_24,
322		- V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
323		- V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
324		- ud_sq_wqe->qkey =
325		- cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
326		- qp->qkey : ud_wr(wr)->remote_qkey);
327		- roce_set_field(ud_sq_wqe->byte_32,
328		- V2_UD_SEND_WQE_BYTE_32_DQPN_M,
329		- V2_UD_SEND_WQE_BYTE_32_DQPN_S,
330		- ud_wr(wr)->remote_qpn);
331		-
332		- roce_set_field(ud_sq_wqe->byte_36,
333		- V2_UD_SEND_WQE_BYTE_36_VLAN_M,
334		- V2_UD_SEND_WQE_BYTE_36_VLAN_S,
335		- le16_to_cpu(ah->av.vlan));
336		- roce_set_field(ud_sq_wqe->byte_36,
337		- V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
338		- V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
339		- ah->av.hop_limit);
340		- roce_set_field(ud_sq_wqe->byte_36,
341		- V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
342		- V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
343		- ah->av.sl_tclass_flowlabel >>
344		- HNS_ROCE_TCLASS_SHIFT);
345		- roce_set_field(ud_sq_wqe->byte_40,
346		- V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
347		- V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,
348		- ah->av.sl_tclass_flowlabel &
349		- HNS_ROCE_FLOW_LABEL_MASK);
350		- roce_set_field(ud_sq_wqe->byte_40,
351		- V2_UD_SEND_WQE_BYTE_40_SL_M,
352		- V2_UD_SEND_WQE_BYTE_40_SL_S,
353		- le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
354		- HNS_ROCE_SL_SHIFT);
355		- roce_set_field(ud_sq_wqe->byte_40,
356		- V2_UD_SEND_WQE_BYTE_40_PORTN_M,
357		- V2_UD_SEND_WQE_BYTE_40_PORTN_S,
358		- qp->port);
359		-
360		- roce_set_field(ud_sq_wqe->byte_48,
361		- V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
362		- V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
363		- hns_get_gid_index(hr_dev, qp->phy_port,
364		- ah->av.gid_index));
365		-
366		- memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
367		- GID_LEN_V2);
368		-
369		- set_extend_sge(qp, wr, &sge_ind);
370		- ind++;
371		- } else if (ibqp->qp_type == IB_QPT_RC) {
372		- rc_sq_wqe = wqe;
373		- memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
374		- for (i = 0; i < wr->num_sge; i++)
375		- tmp_len += wr->sg_list[i].length;
376		-
377		- rc_sq_wqe->msg_len =
378		- cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
379		-
380		- switch (wr->opcode) {
381		- case IB_WR_SEND_WITH_IMM:
382		- case IB_WR_RDMA_WRITE_WITH_IMM:
383		- rc_sq_wqe->immtdata =
384		- cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
385		- break;
386		- case IB_WR_SEND_WITH_INV:
387		- rc_sq_wqe->inv_key =
388		- cpu_to_le32(wr->ex.invalidate_rkey);
389		- break;
390		- default:
391		- rc_sq_wqe->immtdata = 0;
392		- break;
393		- }
394		-
395		- roce_set_bit(rc_sq_wqe->byte_4,
396		- V2_RC_SEND_WQE_BYTE_4_FENCE_S,
397		- (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
398		-
399		- roce_set_bit(rc_sq_wqe->byte_4,
400		- V2_RC_SEND_WQE_BYTE_4_SE_S,
401		- (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
402		-
403		- roce_set_bit(rc_sq_wqe->byte_4,
404		- V2_RC_SEND_WQE_BYTE_4_CQE_S,
405		- (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
406		-
407		- roce_set_bit(rc_sq_wqe->byte_4,
408		- V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
409		-
410		- switch (wr->opcode) {
411		- case IB_WR_RDMA_READ:
412		- roce_set_field(rc_sq_wqe->byte_4,
413		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
414		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
415		- HNS_ROCE_V2_WQE_OP_RDMA_READ);
416		- rc_sq_wqe->rkey =
417		- cpu_to_le32(rdma_wr(wr)->rkey);
418		- rc_sq_wqe->va =
419		- cpu_to_le64(rdma_wr(wr)->remote_addr);
420		- break;
421		- case IB_WR_RDMA_WRITE:
422		- roce_set_field(rc_sq_wqe->byte_4,
423		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
424		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
425		- HNS_ROCE_V2_WQE_OP_RDMA_WRITE);
426		- rc_sq_wqe->rkey =
427		- cpu_to_le32(rdma_wr(wr)->rkey);
428		- rc_sq_wqe->va =
429		- cpu_to_le64(rdma_wr(wr)->remote_addr);
430		- break;
431		- case IB_WR_RDMA_WRITE_WITH_IMM:
432		- roce_set_field(rc_sq_wqe->byte_4,
433		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
434		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
435		- HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM);
436		- rc_sq_wqe->rkey =
437		- cpu_to_le32(rdma_wr(wr)->rkey);
438		- rc_sq_wqe->va =
439		- cpu_to_le64(rdma_wr(wr)->remote_addr);
440		- break;
441		- case IB_WR_SEND:
442		- roce_set_field(rc_sq_wqe->byte_4,
443		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
444		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
445		- HNS_ROCE_V2_WQE_OP_SEND);
446		- break;
447		- case IB_WR_SEND_WITH_INV:
448		- roce_set_field(rc_sq_wqe->byte_4,
449		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
450		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
451		- HNS_ROCE_V2_WQE_OP_SEND_WITH_INV);
452		- break;
453		- case IB_WR_SEND_WITH_IMM:
454		- roce_set_field(rc_sq_wqe->byte_4,
455		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
456		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
457		- HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM);
458		- break;
459		- case IB_WR_LOCAL_INV:
460		- roce_set_field(rc_sq_wqe->byte_4,
461		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
462		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
463		- HNS_ROCE_V2_WQE_OP_LOCAL_INV);
464		- break;
465		- case IB_WR_ATOMIC_CMP_AND_SWP:
466		- roce_set_field(rc_sq_wqe->byte_4,
467		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
468		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
469		- HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP);
470		- break;
471		- case IB_WR_ATOMIC_FETCH_AND_ADD:
472		- roce_set_field(rc_sq_wqe->byte_4,
473		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
474		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
475		- HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD);
476		- break;
477		- case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
478		- roce_set_field(rc_sq_wqe->byte_4,
479		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
480		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
481		- HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP);
482		- break;
483		- case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
484		- roce_set_field(rc_sq_wqe->byte_4,
485		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
486		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
487		- HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD);
488		- break;
489		- default:
490		- roce_set_field(rc_sq_wqe->byte_4,
491		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
492		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
493		- HNS_ROCE_V2_WQE_OP_MASK);
494		- break;
495		- }
496		-
497		- wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
498		-
499		- ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe, wqe,
500		- &sge_ind, bad_wr);
501		- if (ret)
502		- goto out;
503		- ind++;
504		- } else {
505		- dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
506		- spin_unlock_irqrestore(&qp->sq.lock, flags);
	687	+ if (unlikely(ret)) {
507	688	*bad_wr = wr;
508		- return -EOPNOTSUPP;
	689	+ goto out;
509	690	}
510	691	}
511	692
512	693	out:
513	694	if (likely(nreq)) {
514	695	qp->sq.head += nreq;
	696	+ qp->next_sge = sge_idx;
515	697	/* Memory barrier */
516	698	wmb();
517		-
518		- sq_db.byte_4 = 0;
519		- sq_db.parameter = 0;
520		-
521		- roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
522		- V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
523		- roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
524		- V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
525		- roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
526		- V2_DB_PARAMETER_IDX_S,
527		- qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
528		- roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
529		- V2_DB_PARAMETER_SL_S, qp->sl);
530		-
531		- hns_roce_write64_k((__le32 *)&sq_db, qp->sq.db_reg_l);
532		-
533		- qp->sq_next_wqe = ind;
534		- qp->next_sge = sge_ind;
535		-
536		- if (qp->state == IB_QPS_ERR) {
537		- attr_mask = IB_QP_STATE;
538		- attr.qp_state = IB_QPS_ERR;
539		-
540		- ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask,
541		- qp->state, IB_QPS_ERR);
542		- if (ret) {
543		- spin_unlock_irqrestore(&qp->sq.lock, flags);
544		- *bad_wr = wr;
545		- return ret;
546		- }
547		- }
	699	+ update_sq_db(hr_dev, qp);
548	700	}
549	701
550	702	spin_unlock_irqrestore(&qp->sq.lock, flags);
551	703
552	704	return ret;
	705	+}
	706	+
	707	+static int check_recv_valid(struct hns_roce_dev *hr_dev,
	708	+ struct hns_roce_qp *hr_qp)
	709	+{
	710	+ if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
	711	+ return -EIO;
	712	+ else if (hr_qp->state == IB_QPS_RESET)
	713	+ return -EINVAL;
	714	+
	715	+ return 0;
553	716	}
554	717
555	718	static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
..	..	@@ -558,44 +721,44 @@
558	721	{
559	722	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
560	723	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
	724	+ struct ib_device *ibdev = &hr_dev->ib_dev;
561	725	struct hns_roce_v2_wqe_data_seg *dseg;
562	726	struct hns_roce_rinl_sge *sge_list;
563		- struct device *dev = hr_dev->dev;
564		- struct ib_qp_attr attr;
565	727	unsigned long flags;
566	728	void *wqe = NULL;
567		- int attr_mask;
568		- int ret = 0;
	729	+ u32 wqe_idx;
569	730	int nreq;
570		- int ind;
	731	+ int ret;
571	732	int i;
572	733
573	734	spin_lock_irqsave(&hr_qp->rq.lock, flags);
574		- ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
575	735
576		- if (hr_qp->state == IB_QPS_RESET) {
577		- spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
	736	+ ret = check_recv_valid(hr_dev, hr_qp);
	737	+ if (unlikely(ret)) {
578	738	*bad_wr = wr;
579		- return -EINVAL;
	739	+ nreq = 0;
	740	+ goto out;
580	741	}
581	742
582	743	for (nreq = 0; wr; ++nreq, wr = wr->next) {
583		- if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
584		- hr_qp->ibqp.recv_cq)) {
	744	+ if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,
	745	+ hr_qp->ibqp.recv_cq))) {
585	746	ret = -ENOMEM;
586	747	*bad_wr = wr;
587	748	goto out;
588	749	}
589	750
	751	+ wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
	752	+
590	753	if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
591		- dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
592		- wr->num_sge, hr_qp->rq.max_gs);
	754	+ ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n",
	755	+ wr->num_sge, hr_qp->rq.max_gs);
593	756	ret = -EINVAL;
594	757	*bad_wr = wr;
595	758	goto out;
596	759	}
597	760
598		- wqe = get_recv_wqe(hr_qp, ind);
	761	+ wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
599	762	dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
600	763	for (i = 0; i < wr->num_sge; i++) {
601	764	if (!wr->sg_list[i].length)
..	..	@@ -604,15 +767,15 @@
604	767	dseg++;
605	768	}
606	769
607		- if (i < hr_qp->rq.max_gs) {
	770	+ if (wr->num_sge < hr_qp->rq.max_gs) {
608	771	dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
609	772	dseg->addr = 0;
610	773	}
611	774
612	775	/* rq support inline data */
613		- if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
614		- sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
615		- hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt =
	776	+ if (hr_qp->rq_inl_buf.wqe_cnt) {
	777	+ sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
	778	+ hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt =
616	779	(u32)wr->num_sge;
617	780	for (i = 0; i < wr->num_sge; i++) {
618	781	sge_list[i].addr =
..	..	@@ -621,9 +784,7 @@
621	784	}
622	785	}
623	786
624		- hr_qp->rq.wrid[ind] = wr->wr_id;
625		-
626		- ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
	787	+ hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
627	788	}
628	789
629	790	out:
..	..	@@ -632,25 +793,265 @@
632	793	/* Memory barrier */
633	794	wmb();
634	795
635		- *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
636		-
	796	+ /*
	797	+ * Hip08 hardware cannot flush the WQEs in RQ if the QP state
	798	+ * gets into errored mode. Hence, as a workaround to this
	799	+ * hardware limitation, driver needs to assist in flushing. But
	800	+ * the flushing operation uses mailbox to convey the QP state to
	801	+ * the hardware and which can sleep due to the mutex protection
	802	+ * around the mailbox calls. Hence, use the deferred flush for
	803	+ * now.
	804	+ */
637	805	if (hr_qp->state == IB_QPS_ERR) {
638		- attr_mask = IB_QP_STATE;
639		- attr.qp_state = IB_QPS_ERR;
640		-
641		- ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr,
642		- attr_mask, hr_qp->state,
643		- IB_QPS_ERR);
644		- if (ret) {
645		- spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
646		- *bad_wr = wr;
647		- return ret;
648		- }
	806	+ if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG,
	807	+ &hr_qp->flush_flag))
	808	+ init_flush_work(hr_dev, hr_qp);
	809	+ } else {
	810	+ *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
649	811	}
650	812	}
651	813	spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
652	814
653	815	return ret;
	816	+}
	817	+
	818	+static void get_srq_wqe(struct hns_roce_srq srq, int n)
	819	+{
	820	+ return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
	821	+}
	822	+
	823	+static void get_idx_buf(struct hns_roce_idx_que idx_que, int n)
	824	+{
	825	+ return hns_roce_buf_offset(idx_que->mtr.kmem,
	826	+ n << idx_que->entry_shift);
	827	+}
	828	+
	829	+static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
	830	+{
	831	+ /* always called with interrupts disabled. */
	832	+ spin_lock(&srq->lock);
	833	+
	834	+ bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
	835	+ srq->tail++;
	836	+
	837	+ spin_unlock(&srq->lock);
	838	+}
	839	+
	840	+static int find_empty_entry(struct hns_roce_idx_que *idx_que,
	841	+ unsigned long size)
	842	+{
	843	+ int wqe_idx;
	844	+
	845	+ if (unlikely(bitmap_full(idx_que->bitmap, size)))
	846	+ return -ENOSPC;
	847	+
	848	+ wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
	849	+
	850	+ bitmap_set(idx_que->bitmap, wqe_idx, 1);
	851	+
	852	+ return wqe_idx;
	853	+}
	854	+
	855	+static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
	856	+ const struct ib_recv_wr *wr,
	857	+ const struct ib_recv_wr **bad_wr)
	858	+{
	859	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	860	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	861	+ struct hns_roce_v2_wqe_data_seg *dseg;
	862	+ struct hns_roce_v2_db srq_db;
	863	+ unsigned long flags;
	864	+ __le32 *srq_idx;
	865	+ int ret = 0;
	866	+ int wqe_idx;
	867	+ void *wqe;
	868	+ int nreq;
	869	+ int ind;
	870	+ int i;
	871	+
	872	+ spin_lock_irqsave(&srq->lock, flags);
	873	+
	874	+ ind = srq->head & (srq->wqe_cnt - 1);
	875	+
	876	+ for (nreq = 0; wr; ++nreq, wr = wr->next) {
	877	+ if (unlikely(wr->num_sge >= srq->max_gs)) {
	878	+ ret = -EINVAL;
	879	+ *bad_wr = wr;
	880	+ break;
	881	+ }
	882	+
	883	+ if (unlikely(srq->head == srq->tail)) {
	884	+ ret = -ENOMEM;
	885	+ *bad_wr = wr;
	886	+ break;
	887	+ }
	888	+
	889	+ wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt);
	890	+ if (unlikely(wqe_idx < 0)) {
	891	+ ret = -ENOMEM;
	892	+ *bad_wr = wr;
	893	+ break;
	894	+ }
	895	+
	896	+ wqe = get_srq_wqe(srq, wqe_idx);
	897	+ dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
	898	+
	899	+ for (i = 0; i < wr->num_sge; ++i) {
	900	+ dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
	901	+ dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
	902	+ dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
	903	+ }
	904	+
	905	+ if (wr->num_sge < srq->max_gs) {
	906	+ dseg[i].len = 0;
	907	+ dseg[i].lkey = cpu_to_le32(0x100);
	908	+ dseg[i].addr = 0;
	909	+ }
	910	+
	911	+ srq_idx = get_idx_buf(&srq->idx_que, ind);
	912	+ *srq_idx = cpu_to_le32(wqe_idx);
	913	+
	914	+ srq->wrid[wqe_idx] = wr->wr_id;
	915	+ ind = (ind + 1) & (srq->wqe_cnt - 1);
	916	+ }
	917	+
	918	+ if (likely(nreq)) {
	919	+ srq->head += nreq;
	920	+
	921	+ /*
	922	+ * Make sure that descriptors are written before
	923	+ * doorbell record.
	924	+ */
	925	+ wmb();
	926	+
	927	+ srq_db.byte_4 =
	928	+ cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S \|
	929	+ (srq->srqn & V2_DB_BYTE_4_TAG_M));
	930	+ srq_db.parameter =
	931	+ cpu_to_le32(srq->head & V2_DB_PARAMETER_IDX_M);
	932	+
	933	+ hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
	934	+ }
	935	+
	936	+ spin_unlock_irqrestore(&srq->lock, flags);
	937	+
	938	+ return ret;
	939	+}
	940	+
	941	+static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
	942	+ unsigned long instance_stage,
	943	+ unsigned long reset_stage)
	944	+{
	945	+ /* When hardware reset has been completed once or more, we should stop
	946	+ * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
	947	+ * function, we should exit with error. If now at HNAE3_INIT_CLIENT
	948	+ * stage of soft reset process, we should exit with error, and then
	949	+ * HNAE3_INIT_CLIENT related process can rollback the operation like
	950	+ * notifing hardware to free resources, HNAE3_INIT_CLIENT related
	951	+ * process will exit with error to notify NIC driver to reschedule soft
	952	+ * reset process once again.
	953	+ */
	954	+ hr_dev->is_reset = true;
	955	+ hr_dev->dis_db = true;
	956	+
	957	+ if (reset_stage == HNS_ROCE_STATE_RST_INIT \|\|
	958	+ instance_stage == HNS_ROCE_STATE_INIT)
	959	+ return CMD_RST_PRC_EBUSY;
	960	+
	961	+ return CMD_RST_PRC_SUCCESS;
	962	+}
	963	+
	964	+static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
	965	+ unsigned long instance_stage,
	966	+ unsigned long reset_stage)
	967	+{
	968	+#define HW_RESET_TIMEOUT_US 1000000
	969	+#define HW_RESET_SLEEP_US 1000
	970	+
	971	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	972	+ struct hnae3_handle *handle = priv->handle;
	973	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	974	+ unsigned long val;
	975	+ int ret;
	976	+
	977	+ /* When hardware reset is detected, we should stop sending mailbox&cmq&
	978	+ * doorbell to hardware. If now in .init_instance() function, we should
	979	+ * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
	980	+ * process, we should exit with error, and then HNAE3_INIT_CLIENT
	981	+ * related process can rollback the operation like notifing hardware to
	982	+ * free resources, HNAE3_INIT_CLIENT related process will exit with
	983	+ * error to notify NIC driver to reschedule soft reset process once
	984	+ * again.
	985	+ */
	986	+ hr_dev->dis_db = true;
	987	+
	988	+ ret = read_poll_timeout(ops->ae_dev_reset_cnt, val,
	989	+ val > hr_dev->reset_cnt, HW_RESET_SLEEP_US,
	990	+ HW_RESET_TIMEOUT_US, false, handle);
	991	+ if (!ret)
	992	+ hr_dev->is_reset = true;
	993	+
	994	+ if (!hr_dev->is_reset \|\| reset_stage == HNS_ROCE_STATE_RST_INIT \|\|
	995	+ instance_stage == HNS_ROCE_STATE_INIT)
	996	+ return CMD_RST_PRC_EBUSY;
	997	+
	998	+ return CMD_RST_PRC_SUCCESS;
	999	+}
	1000	+
	1001	+static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
	1002	+{
	1003	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	1004	+ struct hnae3_handle *handle = priv->handle;
	1005	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	1006	+
	1007	+ /* When software reset is detected at .init_instance() function, we
	1008	+ * should stop sending mailbox&cmq&doorbell to hardware, and exit
	1009	+ * with error.
	1010	+ */
	1011	+ hr_dev->dis_db = true;
	1012	+ if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
	1013	+ hr_dev->is_reset = true;
	1014	+
	1015	+ return CMD_RST_PRC_EBUSY;
	1016	+}
	1017	+
	1018	+static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
	1019	+{
	1020	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	1021	+ struct hnae3_handle *handle = priv->handle;
	1022	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	1023	+ unsigned long instance_stage; /* the current instance stage */
	1024	+ unsigned long reset_stage; /* the current reset stage */
	1025	+ unsigned long reset_cnt;
	1026	+ bool sw_resetting;
	1027	+ bool hw_resetting;
	1028	+
	1029	+ if (hr_dev->is_reset)
	1030	+ return CMD_RST_PRC_SUCCESS;
	1031	+
	1032	+ /* Get information about reset from NIC driver or RoCE driver itself,
	1033	+ * the meaning of the following variables from NIC driver are described
	1034	+ * as below:
	1035	+ * reset_cnt -- The count value of completed hardware reset.
	1036	+ * hw_resetting -- Whether hardware device is resetting now.
	1037	+ * sw_resetting -- Whether NIC's software reset process is running now.
	1038	+ */
	1039	+ instance_stage = handle->rinfo.instance_state;
	1040	+ reset_stage = handle->rinfo.reset_state;
	1041	+ reset_cnt = ops->ae_dev_reset_cnt(handle);
	1042	+ hw_resetting = ops->get_cmdq_stat(handle);
	1043	+ sw_resetting = ops->ae_dev_resetting(handle);
	1044	+
	1045	+ if (reset_cnt != hr_dev->reset_cnt)
	1046	+ return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
	1047	+ reset_stage);
	1048	+ else if (hw_resetting)
	1049	+ return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
	1050	+ reset_stage);
	1051	+ else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
	1052	+ return hns_roce_v2_cmd_sw_resetting(hr_dev);
	1053	+
	1054	+ return 0;
654	1055	}
655	1056
656	1057	static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
..	..	@@ -696,7 +1097,7 @@
696	1097
697	1098	static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
698	1099	{
699		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1100	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
700	1101	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
701	1102	&priv->cmq.csq : &priv->cmq.crq;
702	1103
..	..	@@ -709,7 +1110,7 @@
709	1110
710	1111	static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
711	1112	{
712		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1113	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
713	1114	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
714	1115	&priv->cmq.csq : &priv->cmq.crq;
715	1116	dma_addr_t dma = ring->desc_dma_addr;
..	..	@@ -719,8 +1120,7 @@
719	1120	roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
720	1121	upper_32_bits(dma));
721	1122	roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
722		- (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) \|
723		- HNS_ROCE_CMQ_ENABLE);
	1123	+ ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
724	1124	roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
725	1125	roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
726	1126	} else {
..	..	@@ -728,8 +1128,7 @@
728	1128	roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
729	1129	upper_32_bits(dma));
730	1130	roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
731		- (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) \|
732		- HNS_ROCE_CMQ_ENABLE);
	1131	+ ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
733	1132	roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
734	1133	roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
735	1134	}
..	..	@@ -737,7 +1136,7 @@
737	1136
738	1137	static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
739	1138	{
740		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1139	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
741	1140	int ret;
742	1141
743	1142	/* Setup the queue entries for command queue */
..	..	@@ -781,7 +1180,7 @@
781	1180
782	1181	static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
783	1182	{
784		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1183	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
785	1184
786	1185	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
787	1186	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
..	..	@@ -803,15 +1202,15 @@
803	1202
804	1203	static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
805	1204	{
806		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
807	1205	u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
	1206	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
808	1207
809	1208	return head == priv->cmq.csq.next_to_use;
810	1209	}
811	1210
812	1211	static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
813	1212	{
814		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1213	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
815	1214	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
816	1215	struct hns_roce_cmq_desc *desc;
817	1216	u16 ntc = csq->next_to_clean;
..	..	@@ -833,21 +1232,18 @@
833	1232	return clean;
834	1233	}
835	1234
836		-static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
837		- struct hns_roce_cmq_desc *desc, int num)
	1235	+static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
	1236	+ struct hns_roce_cmq_desc *desc, int num)
838	1237	{
839		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1238	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
840	1239	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
841	1240	struct hns_roce_cmq_desc *desc_to_use;
842	1241	bool complete = false;
843	1242	u32 timeout = 0;
844	1243	int handle = 0;
845	1244	u16 desc_ret;
846		- int ret = 0;
	1245	+ int ret;
847	1246	int ntc;
848		-
849		- if (hr_dev->is_reset)
850		- return 0;
851	1247
852	1248	spin_lock_bh(&csq->lock);
853	1249
..	..	@@ -879,7 +1275,7 @@
879	1275	* If the command is sync, wait for the firmware to write back,
880	1276	* if multi descriptors to be sent, use the first one to check
881	1277	*/
882		- if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
	1278	+ if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
883	1279	do {
884	1280	if (hns_roce_cmq_csq_done(hr_dev))
885	1281	break;
..	..	@@ -891,15 +1287,14 @@
891	1287	if (hns_roce_cmq_csq_done(hr_dev)) {
892	1288	complete = true;
893	1289	handle = 0;
	1290	+ ret = 0;
894	1291	while (handle < num) {
895	1292	/* get the result of hardware write back */
896	1293	desc_to_use = &csq->desc[ntc];
897	1294	desc[handle] = *desc_to_use;
898	1295	dev_dbg(hr_dev->dev, "Get cmq desc:\n");
899		- desc_ret = desc[handle].retval;
900		- if (desc_ret == CMD_EXEC_SUCCESS)
901		- ret = 0;
902		- else
	1296	+ desc_ret = le16_to_cpu(desc[handle].retval);
	1297	+ if (unlikely(desc_ret != CMD_EXEC_SUCCESS))
903	1298	ret = -EIO;
904	1299	priv->cmq.last_status = desc_ret;
905	1300	ntc++;
..	..	@@ -923,6 +1318,30 @@
923	1318	return ret;
924	1319	}
925	1320
	1321	+static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
	1322	+ struct hns_roce_cmq_desc *desc, int num)
	1323	+{
	1324	+ int retval;
	1325	+ int ret;
	1326	+
	1327	+ ret = hns_roce_v2_rst_process_cmd(hr_dev);
	1328	+ if (ret == CMD_RST_PRC_SUCCESS)
	1329	+ return 0;
	1330	+ if (ret == CMD_RST_PRC_EBUSY)
	1331	+ return -EBUSY;
	1332	+
	1333	+ ret = __hns_roce_cmq_send(hr_dev, desc, num);
	1334	+ if (ret) {
	1335	+ retval = hns_roce_v2_rst_process_cmd(hr_dev);
	1336	+ if (retval == CMD_RST_PRC_SUCCESS)
	1337	+ return 0;
	1338	+ else if (retval == CMD_RST_PRC_EBUSY)
	1339	+ return -EBUSY;
	1340	+ }
	1341	+
	1342	+ return ret;
	1343	+}
	1344	+
926	1345	static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
927	1346	{
928	1347	struct hns_roce_query_version *resp;
..	..	@@ -935,8 +1354,157 @@
935	1354	return ret;
936	1355
937	1356	resp = (struct hns_roce_query_version *)desc.data;
938		- hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version);
939		- hr_dev->vendor_id = le32_to_cpu(resp->rocee_vendor_id);
	1357	+ hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
	1358	+ hr_dev->vendor_id = hr_dev->pci_dev->vendor;
	1359	+
	1360	+ return 0;
	1361	+}
	1362	+
	1363	+static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
	1364	+{
	1365	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	1366	+ struct hnae3_handle *handle = priv->handle;
	1367	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	1368	+ unsigned long reset_cnt;
	1369	+ bool sw_resetting;
	1370	+ bool hw_resetting;
	1371	+
	1372	+ reset_cnt = ops->ae_dev_reset_cnt(handle);
	1373	+ hw_resetting = ops->get_hw_reset_stat(handle);
	1374	+ sw_resetting = ops->ae_dev_resetting(handle);
	1375	+
	1376	+ if (reset_cnt != hr_dev->reset_cnt \|\| hw_resetting \|\| sw_resetting)
	1377	+ return true;
	1378	+
	1379	+ return false;
	1380	+}
	1381	+
	1382	+static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval,
	1383	+ int flag)
	1384	+{
	1385	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	1386	+ struct hnae3_handle *handle = priv->handle;
	1387	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	1388	+ unsigned long instance_stage;
	1389	+ unsigned long reset_cnt;
	1390	+ unsigned long end;
	1391	+ bool sw_resetting;
	1392	+ bool hw_resetting;
	1393	+
	1394	+ instance_stage = handle->rinfo.instance_state;
	1395	+ reset_cnt = ops->ae_dev_reset_cnt(handle);
	1396	+ hw_resetting = ops->get_hw_reset_stat(handle);
	1397	+ sw_resetting = ops->ae_dev_resetting(handle);
	1398	+
	1399	+ if (reset_cnt != hr_dev->reset_cnt) {
	1400	+ hr_dev->dis_db = true;
	1401	+ hr_dev->is_reset = true;
	1402	+ dev_info(hr_dev->dev, "Func clear success after reset.\n");
	1403	+ } else if (hw_resetting) {
	1404	+ hr_dev->dis_db = true;
	1405	+
	1406	+ dev_warn(hr_dev->dev,
	1407	+ "Func clear is pending, device in resetting state.\n");
	1408	+ end = HNS_ROCE_V2_HW_RST_TIMEOUT;
	1409	+ while (end) {
	1410	+ if (!ops->get_hw_reset_stat(handle)) {
	1411	+ hr_dev->is_reset = true;
	1412	+ dev_info(hr_dev->dev,
	1413	+ "Func clear success after reset.\n");
	1414	+ return;
	1415	+ }
	1416	+ msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
	1417	+ end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
	1418	+ }
	1419	+
	1420	+ dev_warn(hr_dev->dev, "Func clear failed.\n");
	1421	+ } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) {
	1422	+ hr_dev->dis_db = true;
	1423	+
	1424	+ dev_warn(hr_dev->dev,
	1425	+ "Func clear is pending, device in resetting state.\n");
	1426	+ end = HNS_ROCE_V2_HW_RST_TIMEOUT;
	1427	+ while (end) {
	1428	+ if (ops->ae_dev_reset_cnt(handle) !=
	1429	+ hr_dev->reset_cnt) {
	1430	+ hr_dev->is_reset = true;
	1431	+ dev_info(hr_dev->dev,
	1432	+ "Func clear success after sw reset\n");
	1433	+ return;
	1434	+ }
	1435	+ msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
	1436	+ end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
	1437	+ }
	1438	+
	1439	+ dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n");
	1440	+ } else {
	1441	+ if (retval && !flag)
	1442	+ dev_warn(hr_dev->dev,
	1443	+ "Func clear read failed, ret = %d.\n", retval);
	1444	+
	1445	+ dev_warn(hr_dev->dev, "Func clear failed.\n");
	1446	+ }
	1447	+}
	1448	+static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
	1449	+{
	1450	+ bool fclr_write_fail_flag = false;
	1451	+ struct hns_roce_func_clear *resp;
	1452	+ struct hns_roce_cmq_desc desc;
	1453	+ unsigned long end;
	1454	+ int ret = 0;
	1455	+
	1456	+ if (hns_roce_func_clr_chk_rst(hr_dev))
	1457	+ goto out;
	1458	+
	1459	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
	1460	+ resp = (struct hns_roce_func_clear *)desc.data;
	1461	+
	1462	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1463	+ if (ret) {
	1464	+ fclr_write_fail_flag = true;
	1465	+ dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
	1466	+ ret);
	1467	+ goto out;
	1468	+ }
	1469	+
	1470	+ msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
	1471	+ end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
	1472	+ while (end) {
	1473	+ if (hns_roce_func_clr_chk_rst(hr_dev))
	1474	+ goto out;
	1475	+ msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
	1476	+ end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
	1477	+
	1478	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
	1479	+ true);
	1480	+
	1481	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1482	+ if (ret)
	1483	+ continue;
	1484	+
	1485	+ if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
	1486	+ hr_dev->is_reset = true;
	1487	+ return;
	1488	+ }
	1489	+ }
	1490	+
	1491	+out:
	1492	+ hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag);
	1493	+}
	1494	+
	1495	+static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
	1496	+{
	1497	+ struct hns_roce_query_fw_info *resp;
	1498	+ struct hns_roce_cmq_desc desc;
	1499	+ int ret;
	1500	+
	1501	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
	1502	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1503	+ if (ret)
	1504	+ return ret;
	1505	+
	1506	+ resp = (struct hns_roce_query_fw_info *)desc.data;
	1507	+ hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
940	1508
941	1509	return 0;
942	1510	}
..	..	@@ -1002,8 +1570,63 @@
1002	1570	hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1003	1571	PF_RES_DATA_3_PF_SL_NUM_M,
1004	1572	PF_RES_DATA_3_PF_SL_NUM_S);
	1573	+ hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
	1574	+ PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
	1575	+ PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
1005	1576
1006	1577	return 0;
	1578	+}
	1579	+
	1580	+static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
	1581	+{
	1582	+ struct hns_roce_pf_timer_res_a *req_a;
	1583	+ struct hns_roce_cmq_desc desc;
	1584	+ int ret;
	1585	+
	1586	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
	1587	+ true);
	1588	+
	1589	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1590	+ if (ret)
	1591	+ return ret;
	1592	+
	1593	+ req_a = (struct hns_roce_pf_timer_res_a *)desc.data;
	1594	+
	1595	+ hr_dev->caps.qpc_timer_bt_num =
	1596	+ roce_get_field(req_a->qpc_timer_bt_idx_num,
	1597	+ PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
	1598	+ PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
	1599	+ hr_dev->caps.cqc_timer_bt_num =
	1600	+ roce_get_field(req_a->cqc_timer_bt_idx_num,
	1601	+ PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
	1602	+ PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
	1603	+
	1604	+ return 0;
	1605	+}
	1606	+
	1607	+static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id)
	1608	+{
	1609	+ struct hns_roce_cmq_desc desc;
	1610	+ struct hns_roce_vf_switch *swt;
	1611	+ int ret;
	1612	+
	1613	+ swt = (struct hns_roce_vf_switch *)desc.data;
	1614	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
	1615	+ swt->rocee_sel \|= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
	1616	+ roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M,
	1617	+ VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id);
	1618	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1619	+ if (ret)
	1620	+ return ret;
	1621	+
	1622	+ desc.flag =
	1623	+ cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR \| HNS_ROCE_CMD_FLAG_IN);
	1624	+ desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
	1625	+ roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
	1626	+ roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0);
	1627	+ roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
	1628	+
	1629	+ return hns_roce_cmq_send(hr_dev, &desc, 1);
1007	1630	}
1008	1631
1009	1632	static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
..	..	@@ -1015,8 +1638,6 @@
1015	1638
1016	1639	req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1017	1640	req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1018		- memset(req_a, 0, sizeof(*req_a));
1019		- memset(req_b, 0, sizeof(*req_b));
1020	1641	for (i = 0; i < 2; i++) {
1021	1642	hns_roce_cmq_setup_basic_desc(&desc[i],
1022	1643	HNS_ROCE_OPC_ALLOC_VF_RES, false);
..	..	@@ -1025,73 +1646,62 @@
1025	1646	desc[i].flag \|= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1026	1647	else
1027	1648	desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1028		-
1029		- if (i == 0) {
1030		- roce_set_field(req_a->vf_qpc_bt_idx_num,
1031		- VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1032		- VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1033		- roce_set_field(req_a->vf_qpc_bt_idx_num,
1034		- VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1035		- VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
1036		- HNS_ROCE_VF_QPC_BT_NUM);
1037		-
1038		- roce_set_field(req_a->vf_srqc_bt_idx_num,
1039		- VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1040		- VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1041		- roce_set_field(req_a->vf_srqc_bt_idx_num,
1042		- VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1043		- VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1044		- HNS_ROCE_VF_SRQC_BT_NUM);
1045		-
1046		- roce_set_field(req_a->vf_cqc_bt_idx_num,
1047		- VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1048		- VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1049		- roce_set_field(req_a->vf_cqc_bt_idx_num,
1050		- VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1051		- VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
1052		- HNS_ROCE_VF_CQC_BT_NUM);
1053		-
1054		- roce_set_field(req_a->vf_mpt_bt_idx_num,
1055		- VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1056		- VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1057		- roce_set_field(req_a->vf_mpt_bt_idx_num,
1058		- VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1059		- VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
1060		- HNS_ROCE_VF_MPT_BT_NUM);
1061		-
1062		- roce_set_field(req_a->vf_eqc_bt_idx_num,
1063		- VF_RES_A_DATA_5_VF_EQC_IDX_M,
1064		- VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1065		- roce_set_field(req_a->vf_eqc_bt_idx_num,
1066		- VF_RES_A_DATA_5_VF_EQC_NUM_M,
1067		- VF_RES_A_DATA_5_VF_EQC_NUM_S,
1068		- HNS_ROCE_VF_EQC_NUM);
1069		- } else {
1070		- roce_set_field(req_b->vf_smac_idx_num,
1071		- VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1072		- VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1073		- roce_set_field(req_b->vf_smac_idx_num,
1074		- VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1075		- VF_RES_B_DATA_1_VF_SMAC_NUM_S,
1076		- HNS_ROCE_VF_SMAC_NUM);
1077		-
1078		- roce_set_field(req_b->vf_sgid_idx_num,
1079		- VF_RES_B_DATA_2_VF_SGID_IDX_M,
1080		- VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1081		- roce_set_field(req_b->vf_sgid_idx_num,
1082		- VF_RES_B_DATA_2_VF_SGID_NUM_M,
1083		- VF_RES_B_DATA_2_VF_SGID_NUM_S,
1084		- HNS_ROCE_VF_SGID_NUM);
1085		-
1086		- roce_set_field(req_b->vf_qid_idx_sl_num,
1087		- VF_RES_B_DATA_3_VF_QID_IDX_M,
1088		- VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1089		- roce_set_field(req_b->vf_qid_idx_sl_num,
1090		- VF_RES_B_DATA_3_VF_SL_NUM_M,
1091		- VF_RES_B_DATA_3_VF_SL_NUM_S,
1092		- HNS_ROCE_VF_SL_NUM);
1093		- }
1094	1649	}
	1650	+
	1651	+ roce_set_field(req_a->vf_qpc_bt_idx_num,
	1652	+ VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
	1653	+ VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
	1654	+ roce_set_field(req_a->vf_qpc_bt_idx_num,
	1655	+ VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
	1656	+ VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, HNS_ROCE_VF_QPC_BT_NUM);
	1657	+
	1658	+ roce_set_field(req_a->vf_srqc_bt_idx_num,
	1659	+ VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
	1660	+ VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
	1661	+ roce_set_field(req_a->vf_srqc_bt_idx_num,
	1662	+ VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
	1663	+ VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
	1664	+ HNS_ROCE_VF_SRQC_BT_NUM);
	1665	+
	1666	+ roce_set_field(req_a->vf_cqc_bt_idx_num,
	1667	+ VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
	1668	+ VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
	1669	+ roce_set_field(req_a->vf_cqc_bt_idx_num,
	1670	+ VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
	1671	+ VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, HNS_ROCE_VF_CQC_BT_NUM);
	1672	+
	1673	+ roce_set_field(req_a->vf_mpt_bt_idx_num,
	1674	+ VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
	1675	+ VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
	1676	+ roce_set_field(req_a->vf_mpt_bt_idx_num,
	1677	+ VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
	1678	+ VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, HNS_ROCE_VF_MPT_BT_NUM);
	1679	+
	1680	+ roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_IDX_M,
	1681	+ VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
	1682	+ roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_NUM_M,
	1683	+ VF_RES_A_DATA_5_VF_EQC_NUM_S, HNS_ROCE_VF_EQC_NUM);
	1684	+
	1685	+ roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_IDX_M,
	1686	+ VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
	1687	+ roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_NUM_M,
	1688	+ VF_RES_B_DATA_1_VF_SMAC_NUM_S, HNS_ROCE_VF_SMAC_NUM);
	1689	+
	1690	+ roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_IDX_M,
	1691	+ VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
	1692	+ roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_NUM_M,
	1693	+ VF_RES_B_DATA_2_VF_SGID_NUM_S, HNS_ROCE_VF_SGID_NUM);
	1694	+
	1695	+ roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_QID_IDX_M,
	1696	+ VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
	1697	+ roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_SL_NUM_M,
	1698	+ VF_RES_B_DATA_3_VF_SL_NUM_S, HNS_ROCE_VF_SL_NUM);
	1699	+
	1700	+ roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
	1701	+ VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
	1702	+ roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
	1703	+ VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
	1704	+ HNS_ROCE_VF_SCCC_BT_NUM);
1095	1705
1096	1706	return hns_roce_cmq_send(hr_dev, desc, 2);
1097	1707	}
..	..	@@ -1102,6 +1712,7 @@
1102	1712	u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1103	1713	u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1104	1714	u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
	1715	+ u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
1105	1716	struct hns_roce_cfg_bt_attr *req;
1106	1717	struct hns_roce_cmq_desc desc;
1107	1718
..	..	@@ -1149,7 +1760,455 @@
1149	1760	CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1150	1761	mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1151	1762
	1763	+ roce_set_field(req->vf_sccc_cfg,
	1764	+ CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
	1765	+ CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
	1766	+ hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
	1767	+ roce_set_field(req->vf_sccc_cfg,
	1768	+ CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
	1769	+ CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
	1770	+ hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
	1771	+ roce_set_field(req->vf_sccc_cfg,
	1772	+ CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
	1773	+ CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
	1774	+ sccc_hop_num ==
	1775	+ HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);
	1776	+
1152	1777	return hns_roce_cmq_send(hr_dev, &desc, 1);
	1778	+}
	1779	+
	1780	+static void set_default_caps(struct hns_roce_dev *hr_dev)
	1781	+{
	1782	+ struct hns_roce_caps *caps = &hr_dev->caps;
	1783	+
	1784	+ caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM;
	1785	+ caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM;
	1786	+ caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM;
	1787	+ caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM;
	1788	+ caps->min_cqes = HNS_ROCE_MIN_CQE_NUM;
	1789	+ caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM;
	1790	+ caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
	1791	+ caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
	1792	+ caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
	1793	+ caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
	1794	+ caps->num_uars = HNS_ROCE_V2_UAR_NUM;
	1795	+ caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM;
	1796	+ caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM;
	1797	+ caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM;
	1798	+ caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
	1799	+ caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM;
	1800	+ caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
	1801	+ caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
	1802	+ caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
	1803	+ caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
	1804	+ caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM;
	1805	+ caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
	1806	+ caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
	1807	+ caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
	1808	+ caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
	1809	+ caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
	1810	+ caps->qpc_sz = HNS_ROCE_V2_QPC_SZ;
	1811	+ caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ;
	1812	+ caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ;
	1813	+ caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ;
	1814	+ caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ;
	1815	+ caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ;
	1816	+ caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
	1817	+ caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ;
	1818	+ caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE;
	1819	+ caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
	1820	+ caps->reserved_lkey = 0;
	1821	+ caps->reserved_pds = 0;
	1822	+ caps->reserved_mrws = 1;
	1823	+ caps->reserved_uars = 0;
	1824	+ caps->reserved_cqs = 0;
	1825	+ caps->reserved_srqs = 0;
	1826	+ caps->reserved_qps = HNS_ROCE_V2_RSV_QPS;
	1827	+
	1828	+ caps->qpc_ba_pg_sz = 0;
	1829	+ caps->qpc_buf_pg_sz = 0;
	1830	+ caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
	1831	+ caps->srqc_ba_pg_sz = 0;
	1832	+ caps->srqc_buf_pg_sz = 0;
	1833	+ caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
	1834	+ caps->cqc_ba_pg_sz = 0;
	1835	+ caps->cqc_buf_pg_sz = 0;
	1836	+ caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
	1837	+ caps->mpt_ba_pg_sz = 0;
	1838	+ caps->mpt_buf_pg_sz = 0;
	1839	+ caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
	1840	+ caps->mtt_ba_pg_sz = 0;
	1841	+ caps->mtt_buf_pg_sz = 0;
	1842	+ caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
	1843	+ caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM;
	1844	+ caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM;
	1845	+ caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM;
	1846	+ caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;
	1847	+ caps->cqe_buf_pg_sz = 0;
	1848	+ caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
	1849	+ caps->srqwqe_ba_pg_sz = 0;
	1850	+ caps->srqwqe_buf_pg_sz = 0;
	1851	+ caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM;
	1852	+ caps->idx_ba_pg_sz = 0;
	1853	+ caps->idx_buf_pg_sz = 0;
	1854	+ caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM;
	1855	+ caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
	1856	+
	1857	+ caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR \|
	1858	+ HNS_ROCE_CAP_FLAG_ROCE_V1_V2 \|
	1859	+ HNS_ROCE_CAP_FLAG_RECORD_DB \|
	1860	+ HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
	1861	+
	1862	+ caps->pkey_table_len[0] = 1;
	1863	+ caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
	1864	+ caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
	1865	+ caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM;
	1866	+ caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
	1867	+ caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
	1868	+ caps->local_ca_ack_delay = 0;
	1869	+ caps->max_mtu = IB_MTU_4096;
	1870	+
	1871	+ caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;
	1872	+ caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;
	1873	+
	1874	+ caps->flags \|= HNS_ROCE_CAP_FLAG_ATOMIC \| HNS_ROCE_CAP_FLAG_MW \|
	1875	+ HNS_ROCE_CAP_FLAG_SRQ \| HNS_ROCE_CAP_FLAG_FRMR \|
	1876	+ HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
	1877	+
	1878	+ caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
	1879	+ caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
	1880	+ caps->qpc_timer_ba_pg_sz = 0;
	1881	+ caps->qpc_timer_buf_pg_sz = 0;
	1882	+ caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
	1883	+ caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
	1884	+ caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
	1885	+ caps->cqc_timer_ba_pg_sz = 0;
	1886	+ caps->cqc_timer_buf_pg_sz = 0;
	1887	+ caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
	1888	+
	1889	+ caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ;
	1890	+ caps->sccc_ba_pg_sz = 0;
	1891	+ caps->sccc_buf_pg_sz = 0;
	1892	+ caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
	1893	+
	1894	+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
	1895	+ caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
	1896	+ caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
	1897	+ caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
	1898	+ caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
	1899	+ }
	1900	+}
	1901	+
	1902	+static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num,
	1903	+ int buf_page_size, int bt_page_size, u32 hem_type)
	1904	+{
	1905	+ u64 obj_per_chunk;
	1906	+ u64 bt_chunk_size = PAGE_SIZE;
	1907	+ u64 buf_chunk_size = PAGE_SIZE;
	1908	+ u64 obj_per_chunk_default = buf_chunk_size / obj_size;
	1909	+
	1910	+ *buf_page_size = 0;
	1911	+ *bt_page_size = 0;
	1912	+
	1913	+ switch (hop_num) {
	1914	+ case 3:
	1915	+ obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
	1916	+ (bt_chunk_size / BA_BYTE_LEN) *
	1917	+ (bt_chunk_size / BA_BYTE_LEN) *
	1918	+ obj_per_chunk_default;
	1919	+ break;
	1920	+ case 2:
	1921	+ obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
	1922	+ (bt_chunk_size / BA_BYTE_LEN) *
	1923	+ obj_per_chunk_default;
	1924	+ break;
	1925	+ case 1:
	1926	+ obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
	1927	+ obj_per_chunk_default;
	1928	+ break;
	1929	+ case HNS_ROCE_HOP_NUM_0:
	1930	+ obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
	1931	+ break;
	1932	+ default:
	1933	+ pr_err("table %u not support hop_num = %u!\n", hem_type,
	1934	+ hop_num);
	1935	+ return;
	1936	+ }
	1937	+
	1938	+ if (hem_type >= HEM_TYPE_MTT)
	1939	+ *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
	1940	+ else
	1941	+ *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
	1942	+}
	1943	+
	1944	+static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
	1945	+{
	1946	+ struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
	1947	+ struct hns_roce_caps *caps = &hr_dev->caps;
	1948	+ struct hns_roce_query_pf_caps_a *resp_a;
	1949	+ struct hns_roce_query_pf_caps_b *resp_b;
	1950	+ struct hns_roce_query_pf_caps_c *resp_c;
	1951	+ struct hns_roce_query_pf_caps_d *resp_d;
	1952	+ struct hns_roce_query_pf_caps_e *resp_e;
	1953	+ int ctx_hop_num;
	1954	+ int pbl_hop_num;
	1955	+ int ret;
	1956	+ int i;
	1957	+
	1958	+ for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
	1959	+ hns_roce_cmq_setup_basic_desc(&desc[i],
	1960	+ HNS_ROCE_OPC_QUERY_PF_CAPS_NUM,
	1961	+ true);
	1962	+ if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
	1963	+ desc[i].flag \|= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
	1964	+ else
	1965	+ desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
	1966	+ }
	1967	+
	1968	+ ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);
	1969	+ if (ret)
	1970	+ return ret;
	1971	+
	1972	+ resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;
	1973	+ resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;
	1974	+ resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
	1975	+ resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
	1976	+ resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
	1977	+
	1978	+ caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;
	1979	+ caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);
	1980	+ caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline);
	1981	+ caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg);
	1982	+ caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg);
	1983	+ caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer);
	1984	+ caps->num_cqc_timer = le16_to_cpu(resp_a->num_cqc_timer);
	1985	+ caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges);
	1986	+ caps->num_aeq_vectors = resp_a->num_aeq_vectors;
	1987	+ caps->num_other_vectors = resp_a->num_other_vectors;
	1988	+ caps->max_sq_desc_sz = resp_a->max_sq_desc_sz;
	1989	+ caps->max_rq_desc_sz = resp_a->max_rq_desc_sz;
	1990	+ caps->max_srq_desc_sz = resp_a->max_srq_desc_sz;
	1991	+ caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE;
	1992	+
	1993	+ caps->mtpt_entry_sz = resp_b->mtpt_entry_sz;
	1994	+ caps->irrl_entry_sz = resp_b->irrl_entry_sz;
	1995	+ caps->trrl_entry_sz = resp_b->trrl_entry_sz;
	1996	+ caps->cqc_entry_sz = resp_b->cqc_entry_sz;
	1997	+ caps->srqc_entry_sz = resp_b->srqc_entry_sz;
	1998	+ caps->idx_entry_sz = resp_b->idx_entry_sz;
	1999	+ caps->sccc_sz = resp_b->sccc_sz;
	2000	+ caps->max_mtu = resp_b->max_mtu;
	2001	+ caps->qpc_sz = HNS_ROCE_V2_QPC_SZ;
	2002	+ caps->min_cqes = resp_b->min_cqes;
	2003	+ caps->min_wqes = resp_b->min_wqes;
	2004	+ caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap);
	2005	+ caps->pkey_table_len[0] = resp_b->pkey_table_len;
	2006	+ caps->phy_num_uars = resp_b->phy_num_uars;
	2007	+ ctx_hop_num = resp_b->ctx_hop_num;
	2008	+ pbl_hop_num = resp_b->pbl_hop_num;
	2009	+
	2010	+ caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds,
	2011	+ V2_QUERY_PF_CAPS_C_NUM_PDS_M,
	2012	+ V2_QUERY_PF_CAPS_C_NUM_PDS_S);
	2013	+ caps->flags = roce_get_field(resp_c->cap_flags_num_pds,
	2014	+ V2_QUERY_PF_CAPS_C_CAP_FLAGS_M,
	2015	+ V2_QUERY_PF_CAPS_C_CAP_FLAGS_S);
	2016	+ caps->flags \|= le16_to_cpu(resp_d->cap_flags_ex) <<
	2017	+ HNS_ROCE_CAP_FLAGS_EX_SHIFT;
	2018	+
	2019	+ caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs,
	2020	+ V2_QUERY_PF_CAPS_C_NUM_CQS_M,
	2021	+ V2_QUERY_PF_CAPS_C_NUM_CQS_S);
	2022	+ caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs,
	2023	+ V2_QUERY_PF_CAPS_C_MAX_GID_M,
	2024	+ V2_QUERY_PF_CAPS_C_MAX_GID_S);
	2025	+ caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth,
	2026	+ V2_QUERY_PF_CAPS_C_CQ_DEPTH_M,
	2027	+ V2_QUERY_PF_CAPS_C_CQ_DEPTH_S);
	2028	+ caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws,
	2029	+ V2_QUERY_PF_CAPS_C_NUM_MRWS_M,
	2030	+ V2_QUERY_PF_CAPS_C_NUM_MRWS_S);
	2031	+ caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps,
	2032	+ V2_QUERY_PF_CAPS_C_NUM_QPS_M,
	2033	+ V2_QUERY_PF_CAPS_C_NUM_QPS_S);
	2034	+ caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps,
	2035	+ V2_QUERY_PF_CAPS_C_MAX_ORD_M,
	2036	+ V2_QUERY_PF_CAPS_C_MAX_ORD_S);
	2037	+ caps->max_qp_dest_rdma = caps->max_qp_init_rdma;
	2038	+ caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
	2039	+ caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs,
	2040	+ V2_QUERY_PF_CAPS_D_NUM_SRQS_M,
	2041	+ V2_QUERY_PF_CAPS_D_NUM_SRQS_S);
	2042	+ caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
	2043	+ caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth,
	2044	+ V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M,
	2045	+ V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S);
	2046	+ caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth,
	2047	+ V2_QUERY_PF_CAPS_D_NUM_CEQS_M,
	2048	+ V2_QUERY_PF_CAPS_D_NUM_CEQS_S);
	2049	+ caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth,
	2050	+ V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M,
	2051	+ V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S);
	2052	+ caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
	2053	+ V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M,
	2054	+ V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S);
	2055	+ caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
	2056	+ V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M,
	2057	+ V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S);
	2058	+ caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds,
	2059	+ V2_QUERY_PF_CAPS_D_RSV_PDS_M,
	2060	+ V2_QUERY_PF_CAPS_D_RSV_PDS_S);
	2061	+ caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds,
	2062	+ V2_QUERY_PF_CAPS_D_NUM_UARS_M,
	2063	+ V2_QUERY_PF_CAPS_D_NUM_UARS_S);
	2064	+ caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps,
	2065	+ V2_QUERY_PF_CAPS_D_RSV_QPS_M,
	2066	+ V2_QUERY_PF_CAPS_D_RSV_QPS_S);
	2067	+ caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps,
	2068	+ V2_QUERY_PF_CAPS_D_RSV_UARS_M,
	2069	+ V2_QUERY_PF_CAPS_D_RSV_UARS_S);
	2070	+ caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
	2071	+ V2_QUERY_PF_CAPS_E_RSV_MRWS_M,
	2072	+ V2_QUERY_PF_CAPS_E_RSV_MRWS_S);
	2073	+ caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
	2074	+ V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M,
	2075	+ V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S);
	2076	+ caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs,
	2077	+ V2_QUERY_PF_CAPS_E_RSV_CQS_M,
	2078	+ V2_QUERY_PF_CAPS_E_RSV_CQS_S);
	2079	+ caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs,
	2080	+ V2_QUERY_PF_CAPS_E_RSV_SRQS_M,
	2081	+ V2_QUERY_PF_CAPS_E_RSV_SRQS_S);
	2082	+ caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey,
	2083	+ V2_QUERY_PF_CAPS_E_RSV_LKEYS_M,
	2084	+ V2_QUERY_PF_CAPS_E_RSV_LKEYS_S);
	2085	+ caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
	2086	+ caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
	2087	+ caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
	2088	+ caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
	2089	+
	2090	+ caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
	2091	+ caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
	2092	+ caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
	2093	+ caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
	2094	+ caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
	2095	+ caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
	2096	+ caps->mtt_ba_pg_sz = 0;
	2097	+ caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
	2098	+ caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
	2099	+ caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
	2100	+
	2101	+ caps->qpc_hop_num = ctx_hop_num;
	2102	+ caps->srqc_hop_num = ctx_hop_num;
	2103	+ caps->cqc_hop_num = ctx_hop_num;
	2104	+ caps->mpt_hop_num = ctx_hop_num;
	2105	+ caps->mtt_hop_num = pbl_hop_num;
	2106	+ caps->cqe_hop_num = pbl_hop_num;
	2107	+ caps->srqwqe_hop_num = pbl_hop_num;
	2108	+ caps->idx_hop_num = pbl_hop_num;
	2109	+ caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
	2110	+ V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M,
	2111	+ V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S);
	2112	+ caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
	2113	+ V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M,
	2114	+ V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S);
	2115	+ caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
	2116	+ V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M,
	2117	+ V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S);
	2118	+
	2119	+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
	2120	+ caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
	2121	+ caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
	2122	+ caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
	2123	+ caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
	2124	+ caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
	2125	+ }
	2126	+
	2127	+ calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,
	2128	+ caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,
	2129	+ HEM_TYPE_QPC);
	2130	+ calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,
	2131	+ caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,
	2132	+ HEM_TYPE_MTPT);
	2133	+ calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,
	2134	+ caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,
	2135	+ HEM_TYPE_CQC);
	2136	+ calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num,
	2137	+ caps->srqc_bt_num, &caps->srqc_buf_pg_sz,
	2138	+ &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC);
	2139	+
	2140	+ caps->sccc_hop_num = ctx_hop_num;
	2141	+ caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
	2142	+ caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
	2143	+
	2144	+ calc_pg_sz(caps->num_qps, caps->sccc_sz,
	2145	+ caps->sccc_hop_num, caps->sccc_bt_num,
	2146	+ &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,
	2147	+ HEM_TYPE_SCCC);
	2148	+ calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,
	2149	+ caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,
	2150	+ &caps->cqc_timer_buf_pg_sz,
	2151	+ &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);
	2152	+
	2153	+ calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num,
	2154	+ 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
	2155	+ calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,
	2156	+ caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,
	2157	+ &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);
	2158	+ calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num,
	2159	+ 1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX);
	2160	+
	2161	+ return 0;
	2162	+}
	2163	+
	2164	+static int hns_roce_config_qpc_size(struct hns_roce_dev *hr_dev)
	2165	+{
	2166	+ struct hns_roce_cmq_desc desc;
	2167	+ struct hns_roce_cfg_entry_size *cfg_size =
	2168	+ (struct hns_roce_cfg_entry_size *)desc.data;
	2169	+
	2170	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
	2171	+ false);
	2172	+
	2173	+ cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_QPC_SIZE);
	2174	+ cfg_size->size = cpu_to_le32(hr_dev->caps.qpc_sz);
	2175	+
	2176	+ return hns_roce_cmq_send(hr_dev, &desc, 1);
	2177	+}
	2178	+
	2179	+static int hns_roce_config_sccc_size(struct hns_roce_dev *hr_dev)
	2180	+{
	2181	+ struct hns_roce_cmq_desc desc;
	2182	+ struct hns_roce_cfg_entry_size *cfg_size =
	2183	+ (struct hns_roce_cfg_entry_size *)desc.data;
	2184	+
	2185	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
	2186	+ false);
	2187	+
	2188	+ cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_SCCC_SIZE);
	2189	+ cfg_size->size = cpu_to_le32(hr_dev->caps.sccc_sz);
	2190	+
	2191	+ return hns_roce_cmq_send(hr_dev, &desc, 1);
	2192	+}
	2193	+
	2194	+static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev)
	2195	+{
	2196	+ int ret;
	2197	+
	2198	+ if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
	2199	+ return 0;
	2200	+
	2201	+ ret = hns_roce_config_qpc_size(hr_dev);
	2202	+ if (ret) {
	2203	+ dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret);
	2204	+ return ret;
	2205	+ }
	2206	+
	2207	+ ret = hns_roce_config_sccc_size(hr_dev);
	2208	+ if (ret)
	2209	+ dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret);
	2210	+
	2211	+ return ret;
1153	2212	}
1154	2213
1155	2214	static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
..	..	@@ -1158,6 +2217,13 @@
1158	2217	int ret;
1159	2218
1160	2219	ret = hns_roce_cmq_query_hw_info(hr_dev);
	2220	+ if (ret) {
	2221	+ dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
	2222	+ ret);
	2223	+ return ret;
	2224	+ }
	2225	+
	2226	+ ret = hns_roce_query_fw_ver(hr_dev);
1161	2227	if (ret) {
1162	2228	dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1163	2229	ret);
..	..	@@ -1179,6 +2245,36 @@
1179	2245	return ret;
1180	2246	}
1181	2247
	2248	+ ret = hns_roce_query_pf_timer_resource(hr_dev);
	2249	+ if (ret) {
	2250	+ dev_err(hr_dev->dev,
	2251	+ "failed to query pf timer resource, ret = %d.\n", ret);
	2252	+ return ret;
	2253	+ }
	2254	+
	2255	+ ret = hns_roce_set_vf_switch_param(hr_dev, 0);
	2256	+ if (ret) {
	2257	+ dev_err(hr_dev->dev,
	2258	+ "failed to set function switch param, ret = %d.\n",
	2259	+ ret);
	2260	+ return ret;
	2261	+ }
	2262	+
	2263	+ hr_dev->vendor_part_id = hr_dev->pci_dev->device;
	2264	+ hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
	2265	+
	2266	+ caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
	2267	+ caps->pbl_buf_pg_sz = 0;
	2268	+ caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
	2269	+ caps->eqe_ba_pg_sz = 0;
	2270	+ caps->eqe_buf_pg_sz = 0;
	2271	+ caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
	2272	+ caps->tsq_buf_pg_sz = 0;
	2273	+
	2274	+ ret = hns_roce_query_pf_caps(hr_dev);
	2275	+ if (ret)
	2276	+ set_default_caps(hr_dev);
	2277	+
1182	2278	ret = hns_roce_alloc_vf_resource(hr_dev);
1183	2279	if (ret) {
1184	2280	dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
..	..	@@ -1186,89 +2282,15 @@
1186	2282	return ret;
1187	2283	}
1188	2284
1189		- hr_dev->vendor_part_id = 0;
1190		- hr_dev->sys_image_guid = 0;
1191		-
1192		- caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM;
1193		- caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM;
1194		- caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM;
1195		- caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM;
1196		- caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1197		- caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1198		- caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1199		- caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
1200		- caps->num_uars = HNS_ROCE_V2_UAR_NUM;
1201		- caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM;
1202		- caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM;
1203		- caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM;
1204		- caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1205		- caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM;
1206		- caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
1207		- caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
1208		- caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM;
1209		- caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1210		- caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1211		- caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1212		- caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1213		- caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1214		- caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ;
1215		- caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1216		- caps->trrl_entry_sz = HNS_ROCE_V2_TRRL_ENTRY_SZ;
1217		- caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ;
1218		- caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1219		- caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
1220		- caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1221		- caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1222		- caps->reserved_lkey = 0;
1223		- caps->reserved_pds = 0;
1224		- caps->reserved_mrws = 1;
1225		- caps->reserved_uars = 0;
1226		- caps->reserved_cqs = 0;
1227		- caps->reserved_qps = HNS_ROCE_V2_RSV_QPS;
1228		-
1229		- caps->qpc_ba_pg_sz = 0;
1230		- caps->qpc_buf_pg_sz = 0;
1231		- caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1232		- caps->srqc_ba_pg_sz = 0;
1233		- caps->srqc_buf_pg_sz = 0;
1234		- caps->srqc_hop_num = HNS_ROCE_HOP_NUM_0;
1235		- caps->cqc_ba_pg_sz = 0;
1236		- caps->cqc_buf_pg_sz = 0;
1237		- caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1238		- caps->mpt_ba_pg_sz = 0;
1239		- caps->mpt_buf_pg_sz = 0;
1240		- caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1241		- caps->pbl_ba_pg_sz = 0;
1242		- caps->pbl_buf_pg_sz = 0;
1243		- caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
1244		- caps->mtt_ba_pg_sz = 0;
1245		- caps->mtt_buf_pg_sz = 0;
1246		- caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
1247		- caps->cqe_ba_pg_sz = 0;
1248		- caps->cqe_buf_pg_sz = 0;
1249		- caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
1250		- caps->eqe_ba_pg_sz = 0;
1251		- caps->eqe_buf_pg_sz = 0;
1252		- caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
1253		- caps->tsq_buf_pg_sz = 0;
1254		- caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1255		-
1256		- caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR \|
1257		- HNS_ROCE_CAP_FLAG_ROCE_V1_V2 \|
1258		- HNS_ROCE_CAP_FLAG_RQ_INLINE \|
1259		- HNS_ROCE_CAP_FLAG_RECORD_DB \|
1260		- HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
1261		- caps->pkey_table_len[0] = 1;
1262		- caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1263		- caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
1264		- caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM;
1265		- caps->local_ca_ack_delay = 0;
1266		- caps->max_mtu = IB_MTU_4096;
1267		-
1268	2285	ret = hns_roce_v2_set_bt(hr_dev);
1269		- if (ret)
1270		- dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1271		- ret);
	2286	+ if (ret) {
	2287	+ dev_err(hr_dev->dev,
	2288	+ "Configure bt attribute fail, ret = %d.\n", ret);
	2289	+ return ret;
	2290	+ }
	2291	+
	2292	+ /* Configure the size of QPC, SCCC, etc. */
	2293	+ ret = hns_roce_config_entry_size(hr_dev);
1272	2294
1273	2295	return ret;
1274	2296	}
..	..	@@ -1303,8 +2325,6 @@
1303	2325
1304	2326	page_num = link_tbl->npages;
1305	2327	entry = link_tbl->table.buf;
1306		- memset(req_a, 0, sizeof(*req_a));
1307		- memset(req_b, 0, sizeof(*req_b));
1308	2328
1309	2329	for (i = 0; i < 2; i++) {
1310	2330	hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
..	..	@@ -1313,41 +2333,30 @@
1313	2333	desc[i].flag \|= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1314	2334	else
1315	2335	desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1316		-
1317		- if (i == 0) {
1318		- req_a->base_addr_l = link_tbl->table.map & 0xffffffff;
1319		- req_a->base_addr_h = (link_tbl->table.map >> 32) &
1320		- 0xffffffff;
1321		- roce_set_field(req_a->depth_pgsz_init_en,
1322		- CFG_LLM_QUE_DEPTH_M,
1323		- CFG_LLM_QUE_DEPTH_S,
1324		- link_tbl->npages);
1325		- roce_set_field(req_a->depth_pgsz_init_en,
1326		- CFG_LLM_QUE_PGSZ_M,
1327		- CFG_LLM_QUE_PGSZ_S,
1328		- link_tbl->pg_sz);
1329		- req_a->head_ba_l = entry[0].blk_ba0;
1330		- req_a->head_ba_h_nxtptr = entry[0].blk_ba1_nxt_ptr;
1331		- roce_set_field(req_a->head_ptr,
1332		- CFG_LLM_HEAD_PTR_M,
1333		- CFG_LLM_HEAD_PTR_S, 0);
1334		- } else {
1335		- req_b->tail_ba_l = entry[page_num - 1].blk_ba0;
1336		- roce_set_field(req_b->tail_ba_h,
1337		- CFG_LLM_TAIL_BA_H_M,
1338		- CFG_LLM_TAIL_BA_H_S,
1339		- entry[page_num - 1].blk_ba1_nxt_ptr &
1340		- HNS_ROCE_LINK_TABLE_BA1_M);
1341		- roce_set_field(req_b->tail_ptr,
1342		- CFG_LLM_TAIL_PTR_M,
1343		- CFG_LLM_TAIL_PTR_S,
1344		- (entry[page_num - 2].blk_ba1_nxt_ptr &
1345		- HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
1346		- HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1347		- }
1348	2336	}
1349		- roce_set_field(req_a->depth_pgsz_init_en,
1350		- CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1);
	2337	+
	2338	+ req_a->base_addr_l = cpu_to_le32(link_tbl->table.map & 0xffffffff);
	2339	+ req_a->base_addr_h = cpu_to_le32(link_tbl->table.map >> 32);
	2340	+ roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_DEPTH_M,
	2341	+ CFG_LLM_QUE_DEPTH_S, link_tbl->npages);
	2342	+ roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_PGSZ_M,
	2343	+ CFG_LLM_QUE_PGSZ_S, link_tbl->pg_sz);
	2344	+ roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M,
	2345	+ CFG_LLM_INIT_EN_S, 1);
	2346	+ req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
	2347	+ req_a->head_ba_h_nxtptr = cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
	2348	+ roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M, CFG_LLM_HEAD_PTR_S,
	2349	+ 0);
	2350	+
	2351	+ req_b->tail_ba_l = cpu_to_le32(entry[page_num - 1].blk_ba0);
	2352	+ roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M,
	2353	+ CFG_LLM_TAIL_BA_H_S,
	2354	+ entry[page_num - 1].blk_ba1_nxt_ptr &
	2355	+ HNS_ROCE_LINK_TABLE_BA1_M);
	2356	+ roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M, CFG_LLM_TAIL_PTR_S,
	2357	+ (entry[page_num - 2].blk_ba1_nxt_ptr &
	2358	+ HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
	2359	+ HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1351	2360
1352	2361	return hns_roce_cmq_send(hr_dev, desc, 2);
1353	2362	}
..	..	@@ -1407,19 +2416,14 @@
1407	2416	goto err_alloc_buf_failed;
1408	2417
1409	2418	link_tbl->pg_list[i].map = t;
1410		- memset(link_tbl->pg_list[i].buf, 0, buf_chk_sz);
1411	2419
1412		- entry[i].blk_ba0 = (t >> 12) & 0xffffffff;
1413		- roce_set_field(entry[i].blk_ba1_nxt_ptr,
1414		- HNS_ROCE_LINK_TABLE_BA1_M,
1415		- HNS_ROCE_LINK_TABLE_BA1_S,
1416		- t >> 44);
	2420	+ entry[i].blk_ba0 = (u32)(t >> 12);
	2421	+ entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44);
1417	2422
1418	2423	if (i < (pg_num - 1))
1419		- roce_set_field(entry[i].blk_ba1_nxt_ptr,
1420		- HNS_ROCE_LINK_TABLE_NXT_PTR_M,
1421		- HNS_ROCE_LINK_TABLE_NXT_PTR_S,
1422		- i + 1);
	2424	+ entry[i].blk_ba1_nxt_ptr \|=
	2425	+ (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S;
	2426	+
1423	2427	}
1424	2428	link_tbl->npages = pg_num;
1425	2429	link_tbl->pg_sz = buf_chk_sz;
..	..	@@ -1464,7 +2468,8 @@
1464	2468	static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
1465	2469	{
1466	2470	struct hns_roce_v2_priv *priv = hr_dev->priv;
1467		- int ret;
	2471	+ int qpc_count, cqc_count;
	2472	+ int ret, i;
1468	2473
1469	2474	/* TSQ includes SQ doorbell and ack doorbell */
1470	2475	ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
..	..	@@ -1479,7 +2484,39 @@
1479	2484	goto err_tpq_init_failed;
1480	2485	}
1481	2486
	2487	+ /* Alloc memory for QPC Timer buffer space chunk */
	2488	+ for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
	2489	+ qpc_count++) {
	2490	+ ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
	2491	+ qpc_count);
	2492	+ if (ret) {
	2493	+ dev_err(hr_dev->dev, "QPC Timer get failed\n");
	2494	+ goto err_qpc_timer_failed;
	2495	+ }
	2496	+ }
	2497	+
	2498	+ /* Alloc memory for CQC Timer buffer space chunk */
	2499	+ for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
	2500	+ cqc_count++) {
	2501	+ ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
	2502	+ cqc_count);
	2503	+ if (ret) {
	2504	+ dev_err(hr_dev->dev, "CQC Timer get failed\n");
	2505	+ goto err_cqc_timer_failed;
	2506	+ }
	2507	+ }
	2508	+
1482	2509	return 0;
	2510	+
	2511	+err_cqc_timer_failed:
	2512	+ for (i = 0; i < cqc_count; i++)
	2513	+ hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
	2514	+
	2515	+err_qpc_timer_failed:
	2516	+ for (i = 0; i < qpc_count; i++)
	2517	+ hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
	2518	+
	2519	+ hns_roce_free_link_table(hr_dev, &priv->tpq);
1483	2520
1484	2521	err_tpq_init_failed:
1485	2522	hns_roce_free_link_table(hr_dev, &priv->tsq);
..	..	@@ -1491,22 +2528,59 @@
1491	2528	{
1492	2529	struct hns_roce_v2_priv *priv = hr_dev->priv;
1493	2530
	2531	+ hns_roce_function_clear(hr_dev);
	2532	+
1494	2533	hns_roce_free_link_table(hr_dev, &priv->tpq);
1495	2534	hns_roce_free_link_table(hr_dev, &priv->tsq);
1496	2535	}
1497	2536
	2537	+static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
	2538	+{
	2539	+ struct hns_roce_cmq_desc desc;
	2540	+ struct hns_roce_mbox_status *mb_st =
	2541	+ (struct hns_roce_mbox_status *)desc.data;
	2542	+ enum hns_roce_cmd_return_status status;
	2543	+
	2544	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
	2545	+
	2546	+ status = hns_roce_cmq_send(hr_dev, &desc, 1);
	2547	+ if (status)
	2548	+ return status;
	2549	+
	2550	+ return le32_to_cpu(mb_st->mb_status_hw_run);
	2551	+}
	2552	+
1498	2553	static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
1499	2554	{
1500		- u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
	2555	+ u32 status = hns_roce_query_mbox_status(hr_dev);
1501	2556
1502	2557	return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
1503	2558	}
1504	2559
1505	2560	static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
1506	2561	{
1507		- u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
	2562	+ u32 status = hns_roce_query_mbox_status(hr_dev);
1508	2563
1509	2564	return status & HNS_ROCE_HW_MB_STATUS_MASK;
	2565	+}
	2566	+
	2567	+static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
	2568	+ u64 out_param, u32 in_modifier, u8 op_modifier,
	2569	+ u16 op, u16 token, int event)
	2570	+{
	2571	+ struct hns_roce_cmq_desc desc;
	2572	+ struct hns_roce_post_mbox mb = (struct hns_roce_post_mbox )desc.data;
	2573	+
	2574	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
	2575	+
	2576	+ mb->in_param_l = cpu_to_le32(in_param);
	2577	+ mb->in_param_h = cpu_to_le32(in_param >> 32);
	2578	+ mb->out_param_l = cpu_to_le32(out_param);
	2579	+ mb->out_param_h = cpu_to_le32(out_param >> 32);
	2580	+ mb->cmd_tag = cpu_to_le32(in_modifier << 8 \| op);
	2581	+ mb->token_event_en = cpu_to_le32(event << 16 \| token);
	2582	+
	2583	+ return hns_roce_cmq_send(hr_dev, &desc, 1);
1510	2584	}
1511	2585
1512	2586	static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
..	..	@@ -1514,11 +2588,8 @@
1514	2588	u16 op, u16 token, int event)
1515	2589	{
1516	2590	struct device *dev = hr_dev->dev;
1517		- u32 __iomem hcr = (u32 __iomem )(hr_dev->reg_base +
1518		- ROCEE_VF_MB_CFG0_REG);
1519	2591	unsigned long end;
1520		- u32 val0 = 0;
1521		- u32 val1 = 0;
	2592	+ int ret;
1522	2593
1523	2594	end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
1524	2595	while (hns_roce_v2_cmd_pending(hr_dev)) {
..	..	@@ -1530,34 +2601,19 @@
1530	2601	cond_resched();
1531	2602	}
1532	2603
1533		- roce_set_field(val0, HNS_ROCE_VF_MB4_TAG_MASK,
1534		- HNS_ROCE_VF_MB4_TAG_SHIFT, in_modifier);
1535		- roce_set_field(val0, HNS_ROCE_VF_MB4_CMD_MASK,
1536		- HNS_ROCE_VF_MB4_CMD_SHIFT, op);
1537		- roce_set_field(val1, HNS_ROCE_VF_MB5_EVENT_MASK,
1538		- HNS_ROCE_VF_MB5_EVENT_SHIFT, event);
1539		- roce_set_field(val1, HNS_ROCE_VF_MB5_TOKEN_MASK,
1540		- HNS_ROCE_VF_MB5_TOKEN_SHIFT, token);
	2604	+ ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
	2605	+ op_modifier, op, token, event);
	2606	+ if (ret)
	2607	+ dev_err(dev, "Post mailbox fail(%d)\n", ret);
1541	2608
1542		- writeq(in_param, hcr + 0);
1543		- writeq(out_param, hcr + 2);
1544		-
1545		- /* Memory barrier */
1546		- wmb();
1547		-
1548		- writel(val0, hcr + 4);
1549		- writel(val1, hcr + 5);
1550		-
1551		- mmiowb();
1552		-
1553		- return 0;
	2609	+ return ret;
1554	2610	}
1555	2611
1556	2612	static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
1557	2613	unsigned long timeout)
1558	2614	{
1559	2615	struct device *dev = hr_dev->dev;
1560		- unsigned long end = 0;
	2616	+ unsigned long end;
1561	2617	u32 status;
1562	2618
1563	2619	end = msecs_to_jiffies(timeout) + jiffies;
..	..	@@ -1571,6 +2627,9 @@
1571	2627
1572	2628	status = hns_roce_v2_cmd_complete(hr_dev);
1573	2629	if (status != 0x1) {
	2630	+ if (status == CMD_RST_PRC_EBUSY)
	2631	+ return status;
	2632	+
1574	2633	dev_err(dev, "mailbox status 0x%x!\n", status);
1575	2634	return -EBUSY;
1576	2635	}
..	..	@@ -1589,11 +2648,9 @@
1589	2648
1590	2649	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
1591	2650
1592		- roce_set_field(sgid_tb->table_idx_rsv,
1593		- CFG_SGID_TB_TABLE_IDX_M,
	2651	+ roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M,
1594	2652	CFG_SGID_TB_TABLE_IDX_S, gid_index);
1595		- roce_set_field(sgid_tb->vf_sgid_type_rsv,
1596		- CFG_SGID_TB_VF_SGID_TYPE_M,
	2653	+ roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M,
1597	2654	CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
1598	2655
1599	2656	p = (u32 *)&gid->raw[0];
..	..	@@ -1633,7 +2690,9 @@
1633	2690
1634	2691	ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
1635	2692	if (ret)
1636		- dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret);
	2693	+ ibdev_err(&hr_dev->ib_dev,
	2694	+ "failed to configure sgid table, ret = %d!\n",
	2695	+ ret);
1637	2696
1638	2697	return ret;
1639	2698	}
..	..	@@ -1652,52 +2711,42 @@
1652	2711	reg_smac_l = (u32 )(&addr[0]);
1653	2712	reg_smac_h = (u16 )(&addr[4]);
1654	2713
1655		- memset(smac_tb, 0, sizeof(*smac_tb));
1656		- roce_set_field(smac_tb->tb_idx_rsv,
1657		- CFG_SMAC_TB_IDX_M,
	2714	+ roce_set_field(smac_tb->tb_idx_rsv, CFG_SMAC_TB_IDX_M,
1658	2715	CFG_SMAC_TB_IDX_S, phy_port);
1659		- roce_set_field(smac_tb->vf_smac_h_rsv,
1660		- CFG_SMAC_TB_VF_SMAC_H_M,
	2716	+ roce_set_field(smac_tb->vf_smac_h_rsv, CFG_SMAC_TB_VF_SMAC_H_M,
1661	2717	CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
1662		- smac_tb->vf_smac_l = reg_smac_l;
	2718	+ smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
1663	2719
1664	2720	return hns_roce_cmq_send(hr_dev, &desc, 1);
1665	2721	}
1666	2722
1667		-static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
	2723	+static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,
	2724	+ struct hns_roce_v2_mpt_entry *mpt_entry,
1668	2725	struct hns_roce_mr *mr)
1669	2726	{
1670		- struct scatterlist *sg;
1671		- u64 page_addr;
1672		- u64 *pages;
1673		- int i, j;
1674		- int len;
1675		- int entry;
	2727	+ u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
	2728	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	2729	+ dma_addr_t pbl_ba;
	2730	+ int i, count;
1676	2731
1677		- mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
1678		- mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
	2732	+ count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
	2733	+ ARRAY_SIZE(pages), &pbl_ba);
	2734	+ if (count < 1) {
	2735	+ ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
	2736	+ count);
	2737	+ return -ENOBUFS;
	2738	+ }
	2739	+
	2740	+ /* Aligned to the hardware address access unit */
	2741	+ for (i = 0; i < count; i++)
	2742	+ pages[i] >>= 6;
	2743	+
	2744	+ mpt_entry->pbl_size = cpu_to_le32(mr->npages);
	2745	+ mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3);
1679	2746	roce_set_field(mpt_entry->byte_48_mode_ba,
1680	2747	V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
1681		- upper_32_bits(mr->pbl_ba >> 3));
	2748	+ upper_32_bits(pbl_ba >> 3));
1682	2749
1683		- pages = (u64 *)__get_free_page(GFP_KERNEL);
1684		- if (!pages)
1685		- return -ENOMEM;
1686		-
1687		- i = 0;
1688		- for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1689		- len = sg_dma_len(sg) >> PAGE_SHIFT;
1690		- for (j = 0; j < len; ++j) {
1691		- page_addr = sg_dma_address(sg) +
1692		- (j << mr->umem->page_shift);
1693		- pages[i] = page_addr >> 6;
1694		- /* Record the first 2 entry directly to MTPT table */
1695		- if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
1696		- goto found;
1697		- i++;
1698		- }
1699		- }
1700		-found:
1701	2750	mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
1702	2751	roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
1703	2752	V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
..	..	@@ -1708,14 +2757,13 @@
1708	2757	roce_set_field(mpt_entry->byte_64_buf_pa1,
1709	2758	V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
1710	2759	V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
1711		- mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
1712		-
1713		- free_page((unsigned long)pages);
	2760	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
1714	2761
1715	2762	return 0;
1716	2763	}
1717	2764
1718		-static int hns_roce_v2_write_mtpt(void mb_buf, struct hns_roce_mr mr,
	2765	+static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
	2766	+ void mb_buf, struct hns_roce_mr mr,
1719	2767	unsigned long mtpt_idx)
1720	2768	{
1721	2769	struct hns_roce_v2_mpt_entry *mpt_entry;
..	..	@@ -1732,16 +2780,17 @@
1732	2780	roce_set_field(mpt_entry->byte_4_pd_hop_st,
1733	2781	V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
1734	2782	V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
1735		- mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
	2783	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
1736	2784	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1737	2785	V2_MPT_BYTE_4_PD_S, mr->pd);
1738	2786
1739	2787	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
1740		- roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
1741		- roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 0);
	2788	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
	2789	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
1742	2790	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
1743	2791	(mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1744		- roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S, 0);
	2792	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
	2793	+ mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
1745	2794	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
1746	2795	(mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1747	2796	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
..	..	@@ -1763,7 +2812,7 @@
1763	2812	if (mr->type == MR_TYPE_DMA)
1764	2813	return 0;
1765	2814
1766		- ret = set_mtpt_pbl(mpt_entry, mr);
	2815	+ ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
1767	2816
1768	2817	return ret;
1769	2818	}
..	..	@@ -1809,16 +2858,105 @@
1809	2858	mr->iova = iova;
1810	2859	mr->size = size;
1811	2860
1812		- ret = set_mtpt_pbl(mpt_entry, mr);
	2861	+ ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
1813	2862	}
1814	2863
1815	2864	return ret;
1816	2865	}
1817	2866
	2867	+static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev,
	2868	+ void mb_buf, struct hns_roce_mr mr)
	2869	+{
	2870	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	2871	+ struct hns_roce_v2_mpt_entry *mpt_entry;
	2872	+ dma_addr_t pbl_ba = 0;
	2873	+
	2874	+ mpt_entry = mb_buf;
	2875	+ memset(mpt_entry, 0, sizeof(*mpt_entry));
	2876	+
	2877	+ if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) {
	2878	+ ibdev_err(ibdev, "failed to find frmr mtr.\n");
	2879	+ return -ENOBUFS;
	2880	+ }
	2881	+
	2882	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
	2883	+ V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
	2884	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
	2885	+ V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
	2886	+ roce_set_field(mpt_entry->byte_4_pd_hop_st,
	2887	+ V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
	2888	+ V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
	2889	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
	2890	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
	2891	+ V2_MPT_BYTE_4_PD_S, mr->pd);
	2892	+
	2893	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
	2894	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
	2895	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
	2896	+
	2897	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
	2898	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
	2899	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
	2900	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
	2901	+
	2902	+ mpt_entry->pbl_size = cpu_to_le32(mr->npages);
	2903	+
	2904	+ mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3));
	2905	+ roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
	2906	+ V2_MPT_BYTE_48_PBL_BA_H_S,
	2907	+ upper_32_bits(pbl_ba >> 3));
	2908	+
	2909	+ roce_set_field(mpt_entry->byte_64_buf_pa1,
	2910	+ V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
	2911	+ V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
	2912	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
	2913	+
	2914	+ return 0;
	2915	+}
	2916	+
	2917	+static int hns_roce_v2_mw_write_mtpt(void mb_buf, struct hns_roce_mw mw)
	2918	+{
	2919	+ struct hns_roce_v2_mpt_entry *mpt_entry;
	2920	+
	2921	+ mpt_entry = mb_buf;
	2922	+ memset(mpt_entry, 0, sizeof(*mpt_entry));
	2923	+
	2924	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
	2925	+ V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
	2926	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
	2927	+ V2_MPT_BYTE_4_PD_S, mw->pdn);
	2928	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
	2929	+ V2_MPT_BYTE_4_PBL_HOP_NUM_S,
	2930	+ mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
	2931	+ mw->pbl_hop_num);
	2932	+ roce_set_field(mpt_entry->byte_4_pd_hop_st,
	2933	+ V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
	2934	+ V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
	2935	+ mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
	2936	+
	2937	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
	2938	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
	2939	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 1);
	2940	+
	2941	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
	2942	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
	2943	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
	2944	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
	2945	+ mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
	2946	+
	2947	+ roce_set_field(mpt_entry->byte_64_buf_pa1,
	2948	+ V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
	2949	+ V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
	2950	+ mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
	2951	+
	2952	+ mpt_entry->lkey = cpu_to_le32(mw->rkey);
	2953	+
	2954	+ return 0;
	2955	+}
	2956	+
1818	2957	static void get_cqe_v2(struct hns_roce_cq hr_cq, int n)
1819	2958	{
1820		- return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1821		- n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
	2959	+ return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);
1822	2960	}
1823	2961
1824	2962	static void get_sw_cqe_v2(struct hns_roce_cq hr_cq, int n)
..	..	@@ -1827,17 +2965,12 @@
1827	2965
1828	2966	/* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1829	2967	return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
1830		- !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL;
	2968	+ !!(n & hr_cq->cq_depth)) ? cqe : NULL;
1831	2969	}
1832	2970
1833		-static struct hns_roce_v2_cqe next_cqe_sw_v2(struct hns_roce_cq hr_cq)
	2971	+static inline void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 ci)
1834	2972	{
1835		- return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
1836		-}
1837		-
1838		-static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
1839		-{
1840		- *hr_cq->set_ci_db = cons_index & 0xffffff;
	2973	+ *hr_cq->set_ci_db = ci & V2_CQ_DB_PARAMETER_CONS_IDX_M;
1841	2974	}
1842	2975
1843	2976	static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
..	..	@@ -1846,11 +2979,12 @@
1846	2979	struct hns_roce_v2_cqe cqe, dest;
1847	2980	u32 prod_index;
1848	2981	int nfreed = 0;
	2982	+ int wqe_index;
1849	2983	u8 owner_bit;
1850	2984
1851	2985	for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
1852	2986	++prod_index) {
1853		- if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
	2987	+ if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
1854	2988	break;
1855	2989	}
1856	2990
..	..	@@ -1863,7 +2997,13 @@
1863	2997	if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
1864	2998	V2_CQE_BYTE_16_LCL_QPN_S) &
1865	2999	HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
1866		- /* In v1 engine, not support SRQ */
	3000	+ if (srq &&
	3001	+ roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
	3002	+ wqe_index = roce_get_field(cqe->byte_4,
	3003	+ V2_CQE_BYTE_4_WQE_INDX_M,
	3004	+ V2_CQE_BYTE_4_WQE_INDX_S);
	3005	+ hns_roce_free_srq_wqe(srq, wqe_index);
	3006	+ }
1867	3007	++nfreed;
1868	3008	} else if (nfreed) {
1869	3009	dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
..	..	@@ -1897,8 +3037,7 @@
1897	3037
1898	3038	static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
1899	3039	struct hns_roce_cq hr_cq, void mb_buf,
1900		- u64 *mtts, dma_addr_t dma_handle, int nent,
1901		- u32 vector)
	3040	+ u64 *mtts, dma_addr_t dma_handle)
1902	3041	{
1903	3042	struct hns_roce_v2_cq_context *cq_context;
1904	3043
..	..	@@ -1910,55 +3049,56 @@
1910	3049	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
1911	3050	V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
1912	3051	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
1913		- V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));
	3052	+ V2_CQC_BYTE_4_SHIFT_S, ilog2(hr_cq->cq_depth));
1914	3053	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
1915		- V2_CQC_BYTE_4_CEQN_S, vector);
1916		- cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn);
	3054	+ V2_CQC_BYTE_4_CEQN_S, hr_cq->vector);
1917	3055
1918	3056	roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
1919	3057	V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
1920	3058
1921		- cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
1922		- cq_context->cqe_cur_blk_addr =
1923		- cpu_to_le32(cq_context->cqe_cur_blk_addr);
	3059	+ roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQE_SIZE_M,
	3060	+ V2_CQC_BYTE_8_CQE_SIZE_S, hr_cq->cqe_size ==
	3061	+ HNS_ROCE_V3_CQE_SIZE ? 1 : 0);
	3062	+
	3063	+ cq_context->cqe_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
1924	3064
1925	3065	roce_set_field(cq_context->byte_16_hop_addr,
1926	3066	V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
1927	3067	V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
1928		- cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT)));
	3068	+ upper_32_bits(to_hr_hw_page_addr(mtts[0])));
1929	3069	roce_set_field(cq_context->byte_16_hop_addr,
1930	3070	V2_CQC_BYTE_16_CQE_HOP_NUM_M,
1931	3071	V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
1932	3072	HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
1933	3073
1934		- cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
	3074	+ cq_context->cqe_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
1935	3075	roce_set_field(cq_context->byte_24_pgsz_addr,
1936	3076	V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
1937	3077	V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
1938		- cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT)));
	3078	+ upper_32_bits(to_hr_hw_page_addr(mtts[1])));
1939	3079	roce_set_field(cq_context->byte_24_pgsz_addr,
1940	3080	V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
1941	3081	V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
1942		- hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
	3082	+ to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));
1943	3083	roce_set_field(cq_context->byte_24_pgsz_addr,
1944	3084	V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
1945	3085	V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
1946		- hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
	3086	+ to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));
1947	3087
1948		- cq_context->cqe_ba = (u32)(dma_handle >> 3);
	3088	+ cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3);
1949	3089
1950	3090	roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
1951	3091	V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
1952	3092
1953		- if (hr_cq->db_en)
1954		- roce_set_bit(cq_context->byte_44_db_record,
1955		- V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
	3093	+ roce_set_bit(cq_context->byte_44_db_record,
	3094	+ V2_CQC_BYTE_44_DB_RECORD_EN_S,
	3095	+ (hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB) ? 1 : 0);
1956	3096
1957	3097	roce_set_field(cq_context->byte_44_db_record,
1958	3098	V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
1959	3099	V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
1960	3100	((u32)hr_cq->db.dma) >> 1);
1961		- cq_context->db_record_addr = hr_cq->db.dma >> 32;
	3101	+ cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32);
1962	3102
1963	3103	roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
1964	3104	V2_CQC_BYTE_56_CQ_MAX_CNT_M,
..	..	@@ -1973,9 +3113,10 @@
1973	3113	static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
1974	3114	enum ib_cq_notify_flags flags)
1975	3115	{
	3116	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
1976	3117	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1977	3118	u32 notification_flag;
1978		- u32 doorbell[2];
	3119	+ __le32 doorbell[2];
1979	3120
1980	3121	doorbell[0] = 0;
1981	3122	doorbell[1] = 0;
..	..	@@ -1991,14 +3132,13 @@
1991	3132	roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
1992	3133	HNS_ROCE_V2_CQ_DB_NTR);
1993	3134	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
1994		- V2_CQ_DB_PARAMETER_CONS_IDX_S,
1995		- hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
	3135	+ V2_CQ_DB_PARAMETER_CONS_IDX_S, hr_cq->cons_index);
1996	3136	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
1997	3137	V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
1998	3138	roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
1999	3139	notification_flag);
2000	3140
2001		- hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
	3141	+ hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);
2002	3142
2003	3143	return 0;
2004	3144	}
..	..	@@ -2018,7 +3158,7 @@
2018	3158
2019	3159	sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
2020	3160	sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
2021		- wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
	3161	+ wqe_buf = hns_roce_get_recv_wqe(*cur_qp, wr_cnt);
2022	3162	data_len = wc->byte_len;
2023	3163
2024	3164	for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
..	..	@@ -2029,7 +3169,7 @@
2029	3169	wqe_buf += size;
2030	3170	}
2031	3171
2032		- if (data_len) {
	3172	+ if (unlikely(data_len)) {
2033	3173	wc->status = IB_WC_LOC_LEN_ERR;
2034	3174	return -EAGAIN;
2035	3175	}
..	..	@@ -2037,24 +3177,137 @@
2037	3177	return 0;
2038	3178	}
2039	3179
	3180	+static int sw_comp(struct hns_roce_qp hr_qp, struct hns_roce_wq wq,
	3181	+ int num_entries, struct ib_wc *wc)
	3182	+{
	3183	+ unsigned int left;
	3184	+ int npolled = 0;
	3185	+
	3186	+ left = wq->head - wq->tail;
	3187	+ if (left == 0)
	3188	+ return 0;
	3189	+
	3190	+ left = min_t(unsigned int, (unsigned int)num_entries, left);
	3191	+ while (npolled < left) {
	3192	+ wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
	3193	+ wc->status = IB_WC_WR_FLUSH_ERR;
	3194	+ wc->vendor_err = 0;
	3195	+ wc->qp = &hr_qp->ibqp;
	3196	+
	3197	+ wq->tail++;
	3198	+ wc++;
	3199	+ npolled++;
	3200	+ }
	3201	+
	3202	+ return npolled;
	3203	+}
	3204	+
	3205	+static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,
	3206	+ struct ib_wc *wc)
	3207	+{
	3208	+ struct hns_roce_qp *hr_qp;
	3209	+ int npolled = 0;
	3210	+
	3211	+ list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {
	3212	+ npolled += sw_comp(hr_qp, &hr_qp->sq,
	3213	+ num_entries - npolled, wc + npolled);
	3214	+ if (npolled >= num_entries)
	3215	+ goto out;
	3216	+ }
	3217	+
	3218	+ list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {
	3219	+ npolled += sw_comp(hr_qp, &hr_qp->rq,
	3220	+ num_entries - npolled, wc + npolled);
	3221	+ if (npolled >= num_entries)
	3222	+ goto out;
	3223	+ }
	3224	+
	3225	+out:
	3226	+ return npolled;
	3227	+}
	3228	+
	3229	+static void get_cqe_status(struct hns_roce_dev hr_dev, struct hns_roce_qp qp,
	3230	+ struct hns_roce_cq cq, struct hns_roce_v2_cqe cqe,
	3231	+ struct ib_wc *wc)
	3232	+{
	3233	+ static const struct {
	3234	+ u32 cqe_status;
	3235	+ enum ib_wc_status wc_status;
	3236	+ } map[] = {
	3237	+ { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },
	3238	+ { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },
	3239	+ { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
	3240	+ { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },
	3241	+ { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
	3242	+ { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },
	3243	+ { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
	3244	+ { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
	3245	+ { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
	3246	+ { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
	3247	+ { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },
	3248	+ { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,
	3249	+ IB_WC_RETRY_EXC_ERR },
	3250	+ { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
	3251	+ { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
	3252	+ { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}
	3253	+ };
	3254	+
	3255	+ u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
	3256	+ V2_CQE_BYTE_4_STATUS_S);
	3257	+ int i;
	3258	+
	3259	+ wc->status = IB_WC_GENERAL_ERR;
	3260	+ for (i = 0; i < ARRAY_SIZE(map); i++)
	3261	+ if (cqe_status == map[i].cqe_status) {
	3262	+ wc->status = map[i].wc_status;
	3263	+ break;
	3264	+ }
	3265	+
	3266	+ if (likely(wc->status == IB_WC_SUCCESS \|\|
	3267	+ wc->status == IB_WC_WR_FLUSH_ERR))
	3268	+ return;
	3269	+
	3270	+ ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status);
	3271	+ print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe,
	3272	+ cq->cqe_size, false);
	3273	+
	3274	+ /*
	3275	+ * For hns ROCEE, GENERAL_ERR is an error type that is not defined in
	3276	+ * the standard protocol, the driver must ignore it and needn't to set
	3277	+ * the QP to an error state.
	3278	+ */
	3279	+ if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)
	3280	+ return;
	3281	+
	3282	+ /*
	3283	+ * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets
	3284	+ * into errored mode. Hence, as a workaround to this hardware
	3285	+ * limitation, driver needs to assist in flushing. But the flushing
	3286	+ * operation uses mailbox to convey the QP state to the hardware and
	3287	+ * which can sleep due to the mutex protection around the mailbox calls.
	3288	+ * Hence, use the deferred flush for now. Once wc error detected, the
	3289	+ * flushing operation is needed.
	3290	+ */
	3291	+ if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
	3292	+ init_flush_work(hr_dev, qp);
	3293	+}
	3294	+
2040	3295	static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2041	3296	struct hns_roce_qp *cur_qp, struct ib_wc wc)
2042	3297	{
2043		- struct hns_roce_dev *hr_dev;
	3298	+ struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
	3299	+ struct hns_roce_srq *srq = NULL;
2044	3300	struct hns_roce_v2_cqe *cqe;
2045	3301	struct hns_roce_qp *hr_qp;
2046	3302	struct hns_roce_wq *wq;
2047		- struct ib_qp_attr attr;
2048		- int attr_mask;
2049	3303	int is_send;
2050	3304	u16 wqe_ctr;
2051	3305	u32 opcode;
2052		- u32 status;
2053	3306	int qpn;
2054	3307	int ret;
2055	3308
2056	3309	/* Find cqe according to consumer index */
2057		- cqe = next_cqe_sw_v2(hr_cq);
	3310	+ cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2058	3311	if (!cqe)
2059	3312	return -EAGAIN;
2060	3313
..	..	@@ -2069,11 +3322,11 @@
2069	3322	V2_CQE_BYTE_16_LCL_QPN_S);
2070	3323
2071	3324	if (!cur_qp \|\| (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (cur_qp)->qpn) {
2072		- hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2073	3325	hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2074	3326	if (unlikely(!hr_qp)) {
2075		- dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
2076		- hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
	3327	+ ibdev_err(&hr_dev->ib_dev,
	3328	+ "CQ %06lx with entry for unknown QPN %06x\n",
	3329	+ hr_cq->cqn, qpn & HNS_ROCE_V2_CQE_QPN_MASK);
2077	3330	return -EINVAL;
2078	3331	}
2079	3332	*cur_qp = hr_qp;
..	..	@@ -2082,126 +3335,7 @@
2082	3335	wc->qp = &(*cur_qp)->ibqp;
2083	3336	wc->vendor_err = 0;
2084	3337
2085		- status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2086		- V2_CQE_BYTE_4_STATUS_S);
2087		- switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
2088		- case HNS_ROCE_CQE_V2_SUCCESS:
2089		- wc->status = IB_WC_SUCCESS;
2090		- break;
2091		- case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
2092		- wc->status = IB_WC_LOC_LEN_ERR;
2093		- break;
2094		- case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
2095		- wc->status = IB_WC_LOC_QP_OP_ERR;
2096		- break;
2097		- case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
2098		- wc->status = IB_WC_LOC_PROT_ERR;
2099		- break;
2100		- case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
2101		- wc->status = IB_WC_WR_FLUSH_ERR;
2102		- break;
2103		- case HNS_ROCE_CQE_V2_MW_BIND_ERR:
2104		- wc->status = IB_WC_MW_BIND_ERR;
2105		- break;
2106		- case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
2107		- wc->status = IB_WC_BAD_RESP_ERR;
2108		- break;
2109		- case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
2110		- wc->status = IB_WC_LOC_ACCESS_ERR;
2111		- break;
2112		- case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
2113		- wc->status = IB_WC_REM_INV_REQ_ERR;
2114		- break;
2115		- case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
2116		- wc->status = IB_WC_REM_ACCESS_ERR;
2117		- break;
2118		- case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
2119		- wc->status = IB_WC_REM_OP_ERR;
2120		- break;
2121		- case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
2122		- wc->status = IB_WC_RETRY_EXC_ERR;
2123		- break;
2124		- case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
2125		- wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2126		- break;
2127		- case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
2128		- wc->status = IB_WC_REM_ABORT_ERR;
2129		- break;
2130		- default:
2131		- wc->status = IB_WC_GENERAL_ERR;
2132		- break;
2133		- }
2134		-
2135		- /* flush cqe if wc status is error, excluding flush error */
2136		- if ((wc->status != IB_WC_SUCCESS) &&
2137		- (wc->status != IB_WC_WR_FLUSH_ERR)) {
2138		- attr_mask = IB_QP_STATE;
2139		- attr.qp_state = IB_QPS_ERR;
2140		- return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp,
2141		- &attr, attr_mask,
2142		- (*cur_qp)->state, IB_QPS_ERR);
2143		- }
2144		-
2145		- if (wc->status == IB_WC_WR_FLUSH_ERR)
2146		- return 0;
2147		-
2148	3338	if (is_send) {
2149		- wc->wc_flags = 0;
2150		- /* SQ corresponding to CQE */
2151		- switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2152		- V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
2153		- case HNS_ROCE_SQ_OPCODE_SEND:
2154		- wc->opcode = IB_WC_SEND;
2155		- break;
2156		- case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
2157		- wc->opcode = IB_WC_SEND;
2158		- break;
2159		- case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
2160		- wc->opcode = IB_WC_SEND;
2161		- wc->wc_flags \|= IB_WC_WITH_IMM;
2162		- break;
2163		- case HNS_ROCE_SQ_OPCODE_RDMA_READ:
2164		- wc->opcode = IB_WC_RDMA_READ;
2165		- wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2166		- break;
2167		- case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
2168		- wc->opcode = IB_WC_RDMA_WRITE;
2169		- break;
2170		- case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
2171		- wc->opcode = IB_WC_RDMA_WRITE;
2172		- wc->wc_flags \|= IB_WC_WITH_IMM;
2173		- break;
2174		- case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
2175		- wc->opcode = IB_WC_LOCAL_INV;
2176		- wc->wc_flags \|= IB_WC_WITH_INVALIDATE;
2177		- break;
2178		- case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
2179		- wc->opcode = IB_WC_COMP_SWAP;
2180		- wc->byte_len = 8;
2181		- break;
2182		- case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
2183		- wc->opcode = IB_WC_FETCH_ADD;
2184		- wc->byte_len = 8;
2185		- break;
2186		- case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
2187		- wc->opcode = IB_WC_MASKED_COMP_SWAP;
2188		- wc->byte_len = 8;
2189		- break;
2190		- case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
2191		- wc->opcode = IB_WC_MASKED_FETCH_ADD;
2192		- wc->byte_len = 8;
2193		- break;
2194		- case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
2195		- wc->opcode = IB_WC_REG_MR;
2196		- break;
2197		- case HNS_ROCE_SQ_OPCODE_BIND_MW:
2198		- wc->opcode = IB_WC_REG_MR;
2199		- break;
2200		- default:
2201		- wc->status = IB_WC_GENERAL_ERR;
2202		- break;
2203		- }
2204		-
2205	3339	wq = &(*cur_qp)->sq;
2206	3340	if ((*cur_qp)->sq_signal_bits) {
2207	3341	/*
..	..	@@ -2218,6 +3352,80 @@
2218	3352
2219	3353	wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2220	3354	++wq->tail;
	3355	+ } else if ((*cur_qp)->ibqp.srq) {
	3356	+ srq = to_hr_srq((*cur_qp)->ibqp.srq);
	3357	+ wqe_ctr = (u16)roce_get_field(cqe->byte_4,
	3358	+ V2_CQE_BYTE_4_WQE_INDX_M,
	3359	+ V2_CQE_BYTE_4_WQE_INDX_S);
	3360	+ wc->wr_id = srq->wrid[wqe_ctr];
	3361	+ hns_roce_free_srq_wqe(srq, wqe_ctr);
	3362	+ } else {
	3363	+ /* Update tail pointer, record wr_id */
	3364	+ wq = &(*cur_qp)->rq;
	3365	+ wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
	3366	+ ++wq->tail;
	3367	+ }
	3368	+
	3369	+ get_cqe_status(hr_dev, *cur_qp, hr_cq, cqe, wc);
	3370	+ if (unlikely(wc->status != IB_WC_SUCCESS))
	3371	+ return 0;
	3372	+
	3373	+ if (is_send) {
	3374	+ wc->wc_flags = 0;
	3375	+ /* SQ corresponding to CQE */
	3376	+ switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
	3377	+ V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
	3378	+ case HNS_ROCE_V2_WQE_OP_SEND:
	3379	+ wc->opcode = IB_WC_SEND;
	3380	+ break;
	3381	+ case HNS_ROCE_V2_WQE_OP_SEND_WITH_INV:
	3382	+ wc->opcode = IB_WC_SEND;
	3383	+ break;
	3384	+ case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:
	3385	+ wc->opcode = IB_WC_SEND;
	3386	+ wc->wc_flags \|= IB_WC_WITH_IMM;
	3387	+ break;
	3388	+ case HNS_ROCE_V2_WQE_OP_RDMA_READ:
	3389	+ wc->opcode = IB_WC_RDMA_READ;
	3390	+ wc->byte_len = le32_to_cpu(cqe->byte_cnt);
	3391	+ break;
	3392	+ case HNS_ROCE_V2_WQE_OP_RDMA_WRITE:
	3393	+ wc->opcode = IB_WC_RDMA_WRITE;
	3394	+ break;
	3395	+ case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:
	3396	+ wc->opcode = IB_WC_RDMA_WRITE;
	3397	+ wc->wc_flags \|= IB_WC_WITH_IMM;
	3398	+ break;
	3399	+ case HNS_ROCE_V2_WQE_OP_LOCAL_INV:
	3400	+ wc->opcode = IB_WC_LOCAL_INV;
	3401	+ wc->wc_flags \|= IB_WC_WITH_INVALIDATE;
	3402	+ break;
	3403	+ case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:
	3404	+ wc->opcode = IB_WC_COMP_SWAP;
	3405	+ wc->byte_len = 8;
	3406	+ break;
	3407	+ case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:
	3408	+ wc->opcode = IB_WC_FETCH_ADD;
	3409	+ wc->byte_len = 8;
	3410	+ break;
	3411	+ case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:
	3412	+ wc->opcode = IB_WC_MASKED_COMP_SWAP;
	3413	+ wc->byte_len = 8;
	3414	+ break;
	3415	+ case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:
	3416	+ wc->opcode = IB_WC_MASKED_FETCH_ADD;
	3417	+ wc->byte_len = 8;
	3418	+ break;
	3419	+ case HNS_ROCE_V2_WQE_OP_FAST_REG_PMR:
	3420	+ wc->opcode = IB_WC_REG_MR;
	3421	+ break;
	3422	+ case HNS_ROCE_V2_WQE_OP_BIND_MW:
	3423	+ wc->opcode = IB_WC_REG_MR;
	3424	+ break;
	3425	+ default:
	3426	+ wc->status = IB_WC_GENERAL_ERR;
	3427	+ break;
	3428	+ }
2221	3429	} else {
2222	3430	/* RQ correspond to CQE */
2223	3431	wc->byte_len = le32_to_cpu(cqe->byte_cnt);
..	..	@@ -2258,14 +3466,9 @@
2258	3466	opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
2259	3467	(roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
2260	3468	ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
2261		- if (ret)
	3469	+ if (unlikely(ret))
2262	3470	return -EAGAIN;
2263	3471	}
2264		-
2265		- /* Update tail pointer, record wr_id */
2266		- wq = &(*cur_qp)->rq;
2267		- wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2268		- ++wq->tail;
2269	3472
2270	3473	wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2271	3474	V2_CQE_BYTE_32_SL_S);
..	..	@@ -2279,15 +3482,16 @@
2279	3482	wc->port_num = roce_get_field(cqe->byte_32,
2280	3483	V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
2281	3484	wc->pkey_index = 0;
2282		- memcpy(wc->smac, cqe->smac, 4);
2283		- wc->smac[4] = roce_get_field(cqe->byte_28,
2284		- V2_CQE_BYTE_28_SMAC_4_M,
2285		- V2_CQE_BYTE_28_SMAC_4_S);
2286		- wc->smac[5] = roce_get_field(cqe->byte_28,
2287		- V2_CQE_BYTE_28_SMAC_5_M,
2288		- V2_CQE_BYTE_28_SMAC_5_S);
2289		- wc->vlan_id = 0xffff;
2290		- wc->wc_flags \|= (IB_WC_WITH_VLAN \| IB_WC_WITH_SMAC);
	3485	+
	3486	+ if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
	3487	+ wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
	3488	+ V2_CQE_BYTE_28_VID_M,
	3489	+ V2_CQE_BYTE_28_VID_S);
	3490	+ wc->wc_flags \|= IB_WC_WITH_VLAN;
	3491	+ } else {
	3492	+ wc->vlan_id = 0xffff;
	3493	+ }
	3494	+
2291	3495	wc->network_hdr_type = roce_get_field(cqe->byte_28,
2292	3496	V2_CQE_BYTE_28_PORT_TYPE_M,
2293	3497	V2_CQE_BYTE_28_PORT_TYPE_S);
..	..	@@ -2299,12 +3503,25 @@
2299	3503	static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
2300	3504	struct ib_wc *wc)
2301	3505	{
	3506	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
2302	3507	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2303	3508	struct hns_roce_qp *cur_qp = NULL;
2304	3509	unsigned long flags;
2305	3510	int npolled;
2306	3511
2307	3512	spin_lock_irqsave(&hr_cq->lock, flags);
	3513	+
	3514	+ /*
	3515	+ * When the device starts to reset, the state is RST_DOWN. At this time,
	3516	+ * there may still be some valid CQEs in the hardware that are not
	3517	+ * polled. Therefore, it is not allowed to switch to the software mode
	3518	+ * immediately. When the state changes to UNINIT, CQE no longer exists
	3519	+ * in the hardware, and then switch to software mode.
	3520	+ */
	3521	+ if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) {
	3522	+ npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);
	3523	+ goto out;
	3524	+ }
2308	3525
2309	3526	for (npolled = 0; npolled < num_entries; ++npolled) {
2310	3527	if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
..	..	@@ -2317,17 +3534,78 @@
2317	3534	hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2318	3535	}
2319	3536
	3537	+out:
2320	3538	spin_unlock_irqrestore(&hr_cq->lock, flags);
2321	3539
2322	3540	return npolled;
	3541	+}
	3542	+
	3543	+static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
	3544	+ int step_idx)
	3545	+{
	3546	+ int op;
	3547	+
	3548	+ if (type == HEM_TYPE_SCCC && step_idx)
	3549	+ return -EINVAL;
	3550	+
	3551	+ switch (type) {
	3552	+ case HEM_TYPE_QPC:
	3553	+ op = HNS_ROCE_CMD_WRITE_QPC_BT0;
	3554	+ break;
	3555	+ case HEM_TYPE_MTPT:
	3556	+ op = HNS_ROCE_CMD_WRITE_MPT_BT0;
	3557	+ break;
	3558	+ case HEM_TYPE_CQC:
	3559	+ op = HNS_ROCE_CMD_WRITE_CQC_BT0;
	3560	+ break;
	3561	+ case HEM_TYPE_SRQC:
	3562	+ op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
	3563	+ break;
	3564	+ case HEM_TYPE_SCCC:
	3565	+ op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
	3566	+ break;
	3567	+ case HEM_TYPE_QPC_TIMER:
	3568	+ op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
	3569	+ break;
	3570	+ case HEM_TYPE_CQC_TIMER:
	3571	+ op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
	3572	+ break;
	3573	+ default:
	3574	+ dev_warn(hr_dev->dev,
	3575	+ "table %u not to be written by mailbox!\n", type);
	3576	+ return -EINVAL;
	3577	+ }
	3578	+
	3579	+ return op + step_idx;
	3580	+}
	3581	+
	3582	+static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba,
	3583	+ u32 hem_type, int step_idx)
	3584	+{
	3585	+ struct hns_roce_cmd_mailbox *mailbox;
	3586	+ int ret;
	3587	+ int op;
	3588	+
	3589	+ op = get_op_for_set_hem(hr_dev, hem_type, step_idx);
	3590	+ if (op < 0)
	3591	+ return 0;
	3592	+
	3593	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	3594	+ if (IS_ERR(mailbox))
	3595	+ return PTR_ERR(mailbox);
	3596	+
	3597	+ ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
	3598	+ 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
	3599	+
	3600	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	3601	+
	3602	+ return ret;
2323	3603	}
2324	3604
2325	3605	static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
2326	3606	struct hns_roce_hem_table *table, int obj,
2327	3607	int step_idx)
2328	3608	{
2329		- struct device *dev = hr_dev->dev;
2330		- struct hns_roce_cmd_mailbox *mailbox;
2331	3609	struct hns_roce_hem_iter iter;
2332	3610	struct hns_roce_hem_mhop mhop;
2333	3611	struct hns_roce_hem *hem;
..	..	@@ -2339,7 +3617,6 @@
2339	3617	u64 bt_ba = 0;
2340	3618	u32 chunk_ba_num;
2341	3619	u32 hop_num;
2342		- u16 op = 0xff;
2343	3620
2344	3621	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2345	3622	return 0;
..	..	@@ -2361,40 +3638,16 @@
2361	3638	hem_idx = i;
2362	3639	}
2363	3640
2364		- switch (table->type) {
2365		- case HEM_TYPE_QPC:
2366		- op = HNS_ROCE_CMD_WRITE_QPC_BT0;
2367		- break;
2368		- case HEM_TYPE_MTPT:
2369		- op = HNS_ROCE_CMD_WRITE_MPT_BT0;
2370		- break;
2371		- case HEM_TYPE_CQC:
2372		- op = HNS_ROCE_CMD_WRITE_CQC_BT0;
2373		- break;
2374		- case HEM_TYPE_SRQC:
2375		- op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
2376		- break;
2377		- default:
2378		- dev_warn(dev, "Table %d not to be written by mailbox!\n",
2379		- table->type);
2380		- return 0;
2381		- }
2382		- op += step_idx;
2383		-
2384		- mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2385		- if (IS_ERR(mailbox))
2386		- return PTR_ERR(mailbox);
	3641	+ if (table->type == HEM_TYPE_SCCC)
	3642	+ obj = mhop.l0_idx;
2387	3643
2388	3644	if (check_whether_last_step(hop_num, step_idx)) {
2389	3645	hem = table->hem[hem_idx];
2390	3646	for (hns_roce_hem_first(hem, &iter);
2391	3647	!hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
2392	3648	bt_ba = hns_roce_hem_addr(&iter);
2393		-
2394		- /* configure the ba, tag, and op */
2395		- ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
2396		- obj, 0, op,
2397		- HNS_ROCE_CMD_TIMEOUT_MSECS);
	3649	+ ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type,
	3650	+ step_idx);
2398	3651	}
2399	3652	} else {
2400	3653	if (step_idx == 0)
..	..	@@ -2402,12 +3655,9 @@
2402	3655	else if (step_idx == 1 && hop_num == 2)
2403	3656	bt_ba = table->bt_l1_dma_addr[l1_idx];
2404	3657
2405		- /* configure the ba, tag, and op */
2406		- ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
2407		- 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
	3658	+ ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);
2408	3659	}
2409	3660
2410		- hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2411	3661	return ret;
2412	3662	}
2413	3663
..	..	@@ -2417,7 +3667,7 @@
2417	3667	{
2418	3668	struct device *dev = hr_dev->dev;
2419	3669	struct hns_roce_cmd_mailbox *mailbox;
2420		- int ret = 0;
	3670	+ int ret;
2421	3671	u16 op = 0xff;
2422	3672
2423	3673	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
..	..	@@ -2433,14 +3683,24 @@
2433	3683	case HEM_TYPE_CQC:
2434	3684	op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
2435	3685	break;
	3686	+ case HEM_TYPE_SCCC:
	3687	+ case HEM_TYPE_QPC_TIMER:
	3688	+ case HEM_TYPE_CQC_TIMER:
	3689	+ break;
2436	3690	case HEM_TYPE_SRQC:
2437	3691	op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
2438	3692	break;
2439	3693	default:
2440		- dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
	3694	+ dev_warn(dev, "table %u not to be destroyed by mailbox!\n",
2441	3695	table->type);
2442	3696	return 0;
2443	3697	}
	3698	+
	3699	+ if (table->type == HEM_TYPE_SCCC \|\|
	3700	+ table->type == HEM_TYPE_QPC_TIMER \|\|
	3701	+ table->type == HEM_TYPE_CQC_TIMER)
	3702	+ return 0;
	3703	+
2444	3704	op += step_idx;
2445	3705
2446	3706	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
..	..	@@ -2456,20 +3716,22 @@
2456	3716	}
2457	3717
2458	3718	static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
2459		- struct hns_roce_mtt *mtt,
2460		- enum ib_qp_state cur_state,
2461		- enum ib_qp_state new_state,
2462	3719	struct hns_roce_v2_qp_context *context,
	3720	+ struct hns_roce_v2_qp_context *qpc_mask,
2463	3721	struct hns_roce_qp *hr_qp)
2464	3722	{
2465	3723	struct hns_roce_cmd_mailbox *mailbox;
	3724	+ int qpc_size;
2466	3725	int ret;
2467	3726
2468	3727	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2469	3728	if (IS_ERR(mailbox))
2470	3729	return PTR_ERR(mailbox);
2471	3730
2472		- memcpy(mailbox->buf, context, sizeof(context) 2);
	3731	+ /* The qpc size of HIP08 is only 256B, which is half of HIP09 */
	3732	+ qpc_size = hr_dev->caps.qpc_sz;
	3733	+ memcpy(mailbox->buf, context, qpc_size);
	3734	+ memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size);
2473	3735
2474	3736	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
2475	3737	HNS_ROCE_CMD_MODIFY_QPC,
..	..	@@ -2508,6 +3770,27 @@
2508	3770	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2509	3771	!!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
2510	3772	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
	3773	+ roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S,
	3774	+ !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
	3775	+ roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, 0);
	3776	+}
	3777	+
	3778	+static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
	3779	+ struct hns_roce_v2_qp_context *context,
	3780	+ struct hns_roce_v2_qp_context *qpc_mask)
	3781	+{
	3782	+ roce_set_field(context->byte_4_sqpn_tst,
	3783	+ V2_QPC_BYTE_4_SGE_SHIFT_M, V2_QPC_BYTE_4_SGE_SHIFT_S,
	3784	+ to_hr_hem_entries_shift(hr_qp->sge.sge_cnt,
	3785	+ hr_qp->sge.sge_shift));
	3786	+
	3787	+ roce_set_field(context->byte_20_smac_sgid_idx,
	3788	+ V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
	3789	+ ilog2(hr_qp->sq.wqe_cnt));
	3790	+
	3791	+ roce_set_field(context->byte_20_smac_sgid_idx,
	3792	+ V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
	3793	+ ilog2(hr_qp->rq.wqe_cnt));
2511	3794	}
2512	3795
2513	3796	static void modify_qp_reset_to_init(struct ib_qp *ibqp,
..	..	@@ -2527,310 +3810,50 @@
2527	3810	*/
2528	3811	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2529	3812	V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
2530		- roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2531		- V2_QPC_BYTE_4_TST_S, 0);
2532		-
2533		- if (ibqp->qp_type == IB_QPT_GSI)
2534		- roce_set_field(context->byte_4_sqpn_tst,
2535		- V2_QPC_BYTE_4_SGE_SHIFT_M,
2536		- V2_QPC_BYTE_4_SGE_SHIFT_S,
2537		- ilog2((unsigned int)hr_qp->sge.sge_cnt));
2538		- else
2539		- roce_set_field(context->byte_4_sqpn_tst,
2540		- V2_QPC_BYTE_4_SGE_SHIFT_M,
2541		- V2_QPC_BYTE_4_SGE_SHIFT_S,
2542		- hr_qp->sq.max_gs > 2 ?
2543		- ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2544		-
2545		- roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2546		- V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2547	3813
2548	3814	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2549	3815	V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2550		- roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2551		- V2_QPC_BYTE_4_SQPN_S, 0);
2552	3816
2553	3817	roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2554	3818	V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
2555		- roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2556		- V2_QPC_BYTE_16_PD_S, 0);
2557	3819
2558	3820	roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2559	3821	V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
2560		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2561		- V2_QPC_BYTE_20_RQWS_S, 0);
2562	3822
2563		- roce_set_field(context->byte_20_smac_sgid_idx,
2564		- V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2565		- ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2566		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2567		- V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2568		-
2569		- roce_set_field(context->byte_20_smac_sgid_idx,
2570		- V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2571		- ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2572		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2573		- V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
	3823	+ set_qpc_wqe_cnt(hr_qp, context, qpc_mask);
2574	3824
2575	3825	/* No VLAN need to set 0xFFF */
2576	3826	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
2577	3827	V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
2578		- roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
2579		- V2_QPC_BYTE_24_VLAN_ID_S, 0);
2580	3828
2581		- /*
2582		- * Set some fields in context to zero, Because the default values
2583		- * of all fields in context are zero, we need not set them to 0 again.
2584		- * but we should set the relevant fields of context mask to 0.
2585		- */
2586		- roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
2587		- roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
2588		- roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
2589		- roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
2590		-
2591		- roce_set_field(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_MAPID_M,
2592		- V2_QPC_BYTE_60_MAPID_S, 0);
2593		-
2594		- roce_set_bit(qpc_mask->byte_60_qpst_mapid,
2595		- V2_QPC_BYTE_60_INNER_MAP_IND_S, 0);
2596		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_MAP_IND_S,
2597		- 0);
2598		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_RQ_MAP_IND_S,
2599		- 0);
2600		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_EXT_MAP_IND_S,
2601		- 0);
2602		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_RLS_IND_S,
2603		- 0);
2604		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_EXT_IND_S,
2605		- 0);
2606		- roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
2607		- roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
2608		-
2609		- if (attr_mask & IB_QP_QKEY) {
2610		- context->qkey_xrcd = attr->qkey;
2611		- qpc_mask->qkey_xrcd = 0;
2612		- hr_qp->qkey = attr->qkey;
2613		- }
2614		-
2615		- if (hr_qp->rdb_en) {
	3829	+ if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
2616	3830	roce_set_bit(context->byte_68_rq_db,
2617	3831	V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
2618		- roce_set_bit(qpc_mask->byte_68_rq_db,
2619		- V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
2620		- }
2621	3832
2622	3833	roce_set_field(context->byte_68_rq_db,
2623	3834	V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
2624	3835	V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
2625	3836	((u32)hr_qp->rdb.dma) >> 1);
2626		- roce_set_field(qpc_mask->byte_68_rq_db,
2627		- V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
2628		- V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
2629		- context->rq_db_record_addr = hr_qp->rdb.dma >> 32;
2630		- qpc_mask->rq_db_record_addr = 0;
	3837	+ context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32);
2631	3838
2632	3839	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
2633	3840	(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
2634		- roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
2635	3841
2636	3842	roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2637	3843	V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
2638		- roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2639		- V2_QPC_BYTE_80_RX_CQN_S, 0);
2640	3844	if (ibqp->srq) {
2641	3845	roce_set_field(context->byte_76_srqn_op_en,
2642	3846	V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
2643	3847	to_hr_srq(ibqp->srq)->srqn);
2644		- roce_set_field(qpc_mask->byte_76_srqn_op_en,
2645		- V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2646	3848	roce_set_bit(context->byte_76_srqn_op_en,
2647	3849	V2_QPC_BYTE_76_SRQ_EN_S, 1);
2648		- roce_set_bit(qpc_mask->byte_76_srqn_op_en,
2649		- V2_QPC_BYTE_76_SRQ_EN_S, 0);
2650	3850	}
2651	3851
2652		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2653		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2654		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
2655		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2656		- V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
2657		- V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
2658		-
2659		- roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
2660		- V2_QPC_BYTE_92_SRQ_INFO_S, 0);
2661		-
2662		- roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
2663		- V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
2664		-
2665		- roce_set_field(qpc_mask->byte_104_rq_sge,
2666		- V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
2667		- V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
2668		-
2669		- roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2670		- V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
2671		- roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2672		- V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
2673		- V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
2674		- roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2675		- V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
2676		-
2677		- qpc_mask->rq_rnr_timer = 0;
2678		- qpc_mask->rx_msg_len = 0;
2679		- qpc_mask->rx_rkey_pkt_info = 0;
2680		- qpc_mask->rx_va = 0;
2681		-
2682		- roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
2683		- V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
2684		- roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
2685		- V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
2686		-
2687		- roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RSVD_RAQ_MAP_S, 0);
2688		- roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
2689		- V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
2690		- roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
2691		- V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
2692		-
2693		- roce_set_field(qpc_mask->byte_144_raq,
2694		- V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
2695		- V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
2696		- roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_RTY_INI_IND_S,
2697		- 0);
2698		- roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
2699		- V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
2700		- roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
2701		-
2702		- roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
2703		- V2_QPC_BYTE_148_RQ_MSN_S, 0);
2704		- roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
2705		- V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
2706		-
2707		- roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2708		- V2_QPC_BYTE_152_RAQ_PSN_S, 0);
2709		- roce_set_field(qpc_mask->byte_152_raq,
2710		- V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
2711		- V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
2712		-
2713		- roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
2714		- V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
2715		-
2716		- roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2717		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
2718		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
2719		- roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2720		- V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
2721		- V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
2722		-
2723		- roce_set_field(context->byte_168_irrl_idx,
2724		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2725		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_S,
2726		- ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2727		- roce_set_field(qpc_mask->byte_168_irrl_idx,
2728		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2729		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0);
2730		-
2731		- roce_set_bit(qpc_mask->byte_168_irrl_idx,
2732		- V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
2733		- roce_set_bit(qpc_mask->byte_168_irrl_idx,
2734		- V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
2735		- roce_set_field(qpc_mask->byte_168_irrl_idx,
2736		- V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
2737		- V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
2738		-
2739		- roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2740		- V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
2741		- roce_set_field(qpc_mask->byte_172_sq_psn,
2742		- V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2743		- V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
2744		-
2745		- roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
2746		- 0);
2747		-
2748		- roce_set_field(qpc_mask->byte_176_msg_pktn,
2749		- V2_QPC_BYTE_176_MSG_USE_PKTN_M,
2750		- V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
2751		- roce_set_field(qpc_mask->byte_176_msg_pktn,
2752		- V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
2753		- V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
2754		-
2755		- roce_set_field(qpc_mask->byte_184_irrl_idx,
2756		- V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
2757		- V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
2758		-
2759		- qpc_mask->cur_sge_offset = 0;
2760		-
2761		- roce_set_field(qpc_mask->byte_192_ext_sge,
2762		- V2_QPC_BYTE_192_CUR_SGE_IDX_M,
2763		- V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
2764		- roce_set_field(qpc_mask->byte_192_ext_sge,
2765		- V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
2766		- V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
2767		-
2768		- roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
2769		- V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
2770		-
2771		- roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
2772		- V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
2773		- roce_set_field(qpc_mask->byte_200_sq_max,
2774		- V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
2775		- V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
2776		-
2777		- roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
2778		- roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
2779		-
2780		- roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
2781		- V2_QPC_BYTE_212_CHECK_FLG_S, 0);
2782		-
2783		- qpc_mask->sq_timer = 0;
2784		-
2785		- roce_set_field(qpc_mask->byte_220_retry_psn_msn,
2786		- V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
2787		- V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
2788		- roce_set_field(qpc_mask->byte_232_irrl_sge,
2789		- V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
2790		- V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
2791		-
2792		- qpc_mask->irrl_cur_sge_offset = 0;
2793		-
2794		- roce_set_field(qpc_mask->byte_240_irrl_tail,
2795		- V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
2796		- V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
2797		- roce_set_field(qpc_mask->byte_240_irrl_tail,
2798		- V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
2799		- V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
2800		- roce_set_field(qpc_mask->byte_240_irrl_tail,
2801		- V2_QPC_BYTE_240_RX_ACK_MSN_M,
2802		- V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
2803		-
2804		- roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
2805		- V2_QPC_BYTE_248_IRRL_PSN_S, 0);
2806		- roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
2807		- 0);
2808		- roce_set_field(qpc_mask->byte_248_ack_psn,
2809		- V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
2810		- V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
2811		- roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
2812		- 0);
2813		- roce_set_bit(qpc_mask->byte_248_ack_psn,
2814		- V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
2815		- roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
2816		- 0);
	3852	+ roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
2817	3853
2818	3854	hr_qp->access_flags = attr->qp_access_flags;
2819		- hr_qp->pkey_index = attr->pkey_index;
2820	3855	roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2821	3856	V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
2822		- roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2823		- V2_QPC_BYTE_252_TX_CQN_S, 0);
2824		-
2825		- roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
2826		- V2_QPC_BYTE_252_ERR_TYPE_S, 0);
2827		-
2828		- roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
2829		- V2_QPC_BYTE_256_RQ_CQE_IDX_M,
2830		- V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
2831		- roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
2832		- V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
2833		- V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
2834	3857	}
2835	3858
2836	3859	static void modify_qp_init_to_init(struct ib_qp *ibqp,
..	..	@@ -2851,20 +3874,6 @@
2851	3874	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2852	3875	V2_QPC_BYTE_4_TST_S, 0);
2853	3876
2854		- if (ibqp->qp_type == IB_QPT_GSI)
2855		- roce_set_field(context->byte_4_sqpn_tst,
2856		- V2_QPC_BYTE_4_SGE_SHIFT_M,
2857		- V2_QPC_BYTE_4_SGE_SHIFT_S,
2858		- ilog2((unsigned int)hr_qp->sge.sge_cnt));
2859		- else
2860		- roce_set_field(context->byte_4_sqpn_tst,
2861		- V2_QPC_BYTE_4_SGE_SHIFT_M,
2862		- V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
2863		- ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2864		-
2865		- roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2866		- V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2867		-
2868	3877	if (attr_mask & IB_QP_ACCESS_FLAGS) {
2869	3878	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2870	3879	!!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
..	..	@@ -2882,6 +3891,12 @@
2882	3891	IB_ACCESS_REMOTE_ATOMIC));
2883	3892	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2884	3893	0);
	3894	+ roce_set_bit(context->byte_76_srqn_op_en,
	3895	+ V2_QPC_BYTE_76_EXT_ATE_S,
	3896	+ !!(attr->qp_access_flags &
	3897	+ IB_ACCESS_REMOTE_ATOMIC));
	3898	+ roce_set_bit(qpc_mask->byte_76_srqn_op_en,
	3899	+ V2_QPC_BYTE_76_EXT_ATE_S, 0);
2885	3900	} else {
2886	3901	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2887	3902	!!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
..	..	@@ -2897,19 +3912,12 @@
2897	3912	!!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
2898	3913	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2899	3914	0);
	3915	+ roce_set_bit(context->byte_76_srqn_op_en,
	3916	+ V2_QPC_BYTE_76_EXT_ATE_S,
	3917	+ !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
	3918	+ roce_set_bit(qpc_mask->byte_76_srqn_op_en,
	3919	+ V2_QPC_BYTE_76_EXT_ATE_S, 0);
2900	3920	}
2901		-
2902		- roce_set_field(context->byte_20_smac_sgid_idx,
2903		- V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2904		- ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2905		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2906		- V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2907		-
2908		- roce_set_field(context->byte_20_smac_sgid_idx,
2909		- V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2910		- ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2911		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2912		- V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
2913	3921
2914	3922	roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2915	3923	V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
..	..	@@ -2938,11 +3946,6 @@
2938	3946	V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2939	3947	}
2940	3948
2941		- if (attr_mask & IB_QP_QKEY) {
2942		- context->qkey_xrcd = attr->qkey;
2943		- qpc_mask->qkey_xrcd = 0;
2944		- }
2945		-
2946	3949	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2947	3950	V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2948	3951	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
..	..	@@ -2954,13 +3957,194 @@
2954	3957	roce_set_field(qpc_mask->byte_56_dqpn_err,
2955	3958	V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
2956	3959	}
	3960	+}
	3961	+
	3962	+static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
	3963	+ struct hns_roce_qp *hr_qp,
	3964	+ struct hns_roce_v2_qp_context *context,
	3965	+ struct hns_roce_v2_qp_context *qpc_mask)
	3966	+{
	3967	+ u64 mtts[MTT_MIN_COUNT] = { 0 };
	3968	+ u64 wqe_sge_ba;
	3969	+ int count;
	3970	+
	3971	+ /* Search qp buf's mtts */
	3972	+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
	3973	+ MTT_MIN_COUNT, &wqe_sge_ba);
	3974	+ if (hr_qp->rq.wqe_cnt && count < 1) {
	3975	+ ibdev_err(&hr_dev->ib_dev,
	3976	+ "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn);
	3977	+ return -EINVAL;
	3978	+ }
	3979	+
	3980	+ context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
	3981	+ qpc_mask->wqe_sge_ba = 0;
	3982	+
	3983	+ /*
	3984	+ * In v2 engine, software pass context and context mask to hardware
	3985	+ * when modifying qp. If software need modify some fields in context,
	3986	+ * we should set all bits of the relevant fields in context mask to
	3987	+ * 0 at the same time, else set them to 0x1.
	3988	+ */
	3989	+ roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
	3990	+ V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
	3991	+ roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
	3992	+ V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
	3993	+
	3994	+ roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
	3995	+ V2_QPC_BYTE_12_SQ_HOP_NUM_S,
	3996	+ to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num,
	3997	+ hr_qp->sq.wqe_cnt));
	3998	+ roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
	3999	+ V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
	4000	+
	4001	+ roce_set_field(context->byte_20_smac_sgid_idx,
	4002	+ V2_QPC_BYTE_20_SGE_HOP_NUM_M,
	4003	+ V2_QPC_BYTE_20_SGE_HOP_NUM_S,
	4004	+ to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num,
	4005	+ hr_qp->sge.sge_cnt));
	4006	+ roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
	4007	+ V2_QPC_BYTE_20_SGE_HOP_NUM_M,
	4008	+ V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
	4009	+
	4010	+ roce_set_field(context->byte_20_smac_sgid_idx,
	4011	+ V2_QPC_BYTE_20_RQ_HOP_NUM_M,
	4012	+ V2_QPC_BYTE_20_RQ_HOP_NUM_S,
	4013	+ to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num,
	4014	+ hr_qp->rq.wqe_cnt));
	4015	+
	4016	+ roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
	4017	+ V2_QPC_BYTE_20_RQ_HOP_NUM_M,
	4018	+ V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
	4019	+
	4020	+ roce_set_field(context->byte_16_buf_ba_pg_sz,
	4021	+ V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
	4022	+ V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
	4023	+ to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift));
	4024	+ roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
	4025	+ V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
	4026	+ V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
	4027	+
	4028	+ roce_set_field(context->byte_16_buf_ba_pg_sz,
	4029	+ V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
	4030	+ V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
	4031	+ to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift));
	4032	+ roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
	4033	+ V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
	4034	+ V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
	4035	+
	4036	+ context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
	4037	+ qpc_mask->rq_cur_blk_addr = 0;
	4038	+
	4039	+ roce_set_field(context->byte_92_srq_info,
	4040	+ V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
	4041	+ V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
	4042	+ upper_32_bits(to_hr_hw_page_addr(mtts[0])));
	4043	+ roce_set_field(qpc_mask->byte_92_srq_info,
	4044	+ V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
	4045	+ V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
	4046	+
	4047	+ context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
	4048	+ qpc_mask->rq_nxt_blk_addr = 0;
	4049	+
	4050	+ roce_set_field(context->byte_104_rq_sge,
	4051	+ V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
	4052	+ V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
	4053	+ upper_32_bits(to_hr_hw_page_addr(mtts[1])));
	4054	+ roce_set_field(qpc_mask->byte_104_rq_sge,
	4055	+ V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
	4056	+ V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
	4057	+
	4058	+ roce_set_field(context->byte_84_rq_ci_pi,
	4059	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
	4060	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
	4061	+ roce_set_field(qpc_mask->byte_84_rq_ci_pi,
	4062	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
	4063	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
	4064	+
	4065	+ roce_set_field(qpc_mask->byte_84_rq_ci_pi,
	4066	+ V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
	4067	+ V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
	4068	+
	4069	+ return 0;
	4070	+}
	4071	+
	4072	+static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,
	4073	+ struct hns_roce_qp *hr_qp,
	4074	+ struct hns_roce_v2_qp_context *context,
	4075	+ struct hns_roce_v2_qp_context *qpc_mask)
	4076	+{
	4077	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4078	+ u64 sge_cur_blk = 0;
	4079	+ u64 sq_cur_blk = 0;
	4080	+ int count;
	4081	+
	4082	+ /* search qp buf's mtts */
	4083	+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
	4084	+ if (count < 1) {
	4085	+ ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n",
	4086	+ hr_qp->qpn);
	4087	+ return -EINVAL;
	4088	+ }
	4089	+ if (hr_qp->sge.sge_cnt > 0) {
	4090	+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
	4091	+ hr_qp->sge.offset,
	4092	+ &sge_cur_blk, 1, NULL);
	4093	+ if (count < 1) {
	4094	+ ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",
	4095	+ hr_qp->qpn);
	4096	+ return -EINVAL;
	4097	+ }
	4098	+ }
	4099	+
	4100	+ /*
	4101	+ * In v2 engine, software pass context and context mask to hardware
	4102	+ * when modifying qp. If software need modify some fields in context,
	4103	+ * we should set all bits of the relevant fields in context mask to
	4104	+ * 0 at the same time, else set them to 0x1.
	4105	+ */
	4106	+ context->sq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk));
2957	4107	roce_set_field(context->byte_168_irrl_idx,
2958		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2959		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_S,
2960		- ilog2((unsigned int)hr_qp->sq.wqe_cnt));
	4108	+ V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
	4109	+ V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
	4110	+ upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
	4111	+ qpc_mask->sq_cur_blk_addr = 0;
2961	4112	roce_set_field(qpc_mask->byte_168_irrl_idx,
2962		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2963		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0);
	4113	+ V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
	4114	+ V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
	4115	+
	4116	+ context->sq_cur_sge_blk_addr =
	4117	+ cpu_to_le32(to_hr_hw_page_addr(sge_cur_blk));
	4118	+ roce_set_field(context->byte_184_irrl_idx,
	4119	+ V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
	4120	+ V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
	4121	+ upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
	4122	+ qpc_mask->sq_cur_sge_blk_addr = 0;
	4123	+ roce_set_field(qpc_mask->byte_184_irrl_idx,
	4124	+ V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
	4125	+ V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
	4126	+
	4127	+ context->rx_sq_cur_blk_addr =
	4128	+ cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk));
	4129	+ roce_set_field(context->byte_232_irrl_sge,
	4130	+ V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
	4131	+ V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
	4132	+ upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
	4133	+ qpc_mask->rx_sq_cur_blk_addr = 0;
	4134	+ roce_set_field(qpc_mask->byte_232_irrl_sge,
	4135	+ V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
	4136	+ V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
	4137	+
	4138	+ return 0;
	4139	+}
	4140	+
	4141	+static inline enum ib_mtu get_mtu(struct ib_qp *ibqp,
	4142	+ const struct ib_qp_attr *attr)
	4143	+{
	4144	+ if (ibqp->qp_type == IB_QPT_GSI \|\| ibqp->qp_type == IB_QPT_UD)
	4145	+ return IB_MTU_4096;
	4146	+
	4147	+ return attr->path_mtu;
2964	4148	}
2965	4149
2966	4150	static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
..	..	@@ -2971,162 +4155,63 @@
2971	4155	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2972	4156	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2973	4157	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2974		- struct device *dev = hr_dev->dev;
2975		- dma_addr_t dma_handle_3;
2976		- dma_addr_t dma_handle_2;
2977		- dma_addr_t dma_handle;
2978		- u32 page_size;
	4158	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4159	+ dma_addr_t trrl_ba;
	4160	+ dma_addr_t irrl_ba;
	4161	+ enum ib_mtu mtu;
	4162	+ u8 lp_pktn_ini;
2979	4163	u8 port_num;
2980		- u64 *mtts_3;
2981		- u64 *mtts_2;
2982	4164	u64 *mtts;
2983	4165	u8 *dmac;
2984	4166	u8 *smac;
2985	4167	int port;
	4168	+ int ret;
2986	4169
2987		- /* Search qp buf's mtts */
2988		- mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
2989		- hr_qp->mtt.first_seg, &dma_handle);
2990		- if (!mtts) {
2991		- dev_err(dev, "qp buf pa find failed\n");
2992		- return -EINVAL;
	4170	+ ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);
	4171	+ if (ret) {
	4172	+ ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);
	4173	+ return ret;
2993	4174	}
2994	4175
2995	4176	/* Search IRRL's mtts */
2996		- mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
2997		- hr_qp->qpn, &dma_handle_2);
2998		- if (!mtts_2) {
2999		- dev_err(dev, "qp irrl_table find failed\n");
	4177	+ mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
	4178	+ hr_qp->qpn, &irrl_ba);
	4179	+ if (!mtts) {
	4180	+ ibdev_err(ibdev, "failed to find qp irrl_table.\n");
3000	4181	return -EINVAL;
3001	4182	}
3002	4183
3003	4184	/* Search TRRL's mtts */
3004		- mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
3005		- hr_qp->qpn, &dma_handle_3);
3006		- if (!mtts_3) {
3007		- dev_err(dev, "qp trrl_table find failed\n");
	4185	+ mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
	4186	+ hr_qp->qpn, &trrl_ba);
	4187	+ if (!mtts) {
	4188	+ ibdev_err(ibdev, "failed to find qp trrl_table.\n");
3008	4189	return -EINVAL;
3009	4190	}
3010	4191
3011	4192	if (attr_mask & IB_QP_ALT_PATH) {
3012		- dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
	4193	+ ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",
	4194	+ attr_mask);
3013	4195	return -EINVAL;
3014	4196	}
3015	4197
3016		- dmac = (u8 *)attr->ah_attr.roce.dmac;
3017		- context->wqe_sge_ba = (u32)(dma_handle >> 3);
3018		- qpc_mask->wqe_sge_ba = 0;
3019		-
3020		- /*
3021		- * In v2 engine, software pass context and context mask to hardware
3022		- * when modifying qp. If software need modify some fields in context,
3023		- * we should set all bits of the relevant fields in context mask to
3024		- * 0 at the same time, else set them to 0x1.
3025		- */
3026		- roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3027		- V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3));
3028		- roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3029		- V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
3030		-
3031		- roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3032		- V2_QPC_BYTE_12_SQ_HOP_NUM_S,
3033		- hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
3034		- 0 : hr_dev->caps.mtt_hop_num);
3035		- roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3036		- V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
3037		-
3038		- roce_set_field(context->byte_20_smac_sgid_idx,
3039		- V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3040		- V2_QPC_BYTE_20_SGE_HOP_NUM_S,
3041		- ((ibqp->qp_type == IB_QPT_GSI) \|\| hr_qp->sq.max_gs > 2) ?
3042		- hr_dev->caps.mtt_hop_num : 0);
3043		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3044		- V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3045		- V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
3046		-
3047		- roce_set_field(context->byte_20_smac_sgid_idx,
3048		- V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3049		- V2_QPC_BYTE_20_RQ_HOP_NUM_S,
3050		- hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
3051		- 0 : hr_dev->caps.mtt_hop_num);
3052		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3053		- V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3054		- V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
3055		-
3056		- roce_set_field(context->byte_16_buf_ba_pg_sz,
3057		- V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3058		- V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
3059		- hr_dev->caps.mtt_ba_pg_sz + PG_SHIFT_OFFSET);
3060		- roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3061		- V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3062		- V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
3063		-
3064		- roce_set_field(context->byte_16_buf_ba_pg_sz,
3065		- V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3066		- V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
3067		- hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
3068		- roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3069		- V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3070		- V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
3071		-
3072		- roce_set_field(context->byte_80_rnr_rx_cqn,
3073		- V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3074		- V2_QPC_BYTE_80_MIN_RNR_TIME_S, attr->min_rnr_timer);
3075		- roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
3076		- V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3077		- V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
3078		-
3079		- page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3080		- context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size]
3081		- >> PAGE_ADDR_SHIFT);
3082		- qpc_mask->rq_cur_blk_addr = 0;
3083		-
3084		- roce_set_field(context->byte_92_srq_info,
3085		- V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3086		- V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
3087		- mtts[hr_qp->rq.offset / page_size]
3088		- >> (32 + PAGE_ADDR_SHIFT));
3089		- roce_set_field(qpc_mask->byte_92_srq_info,
3090		- V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3091		- V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
3092		-
3093		- context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1]
3094		- >> PAGE_ADDR_SHIFT);
3095		- qpc_mask->rq_nxt_blk_addr = 0;
3096		-
3097		- roce_set_field(context->byte_104_rq_sge,
3098		- V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3099		- V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
3100		- mtts[hr_qp->rq.offset / page_size + 1]
3101		- >> (32 + PAGE_ADDR_SHIFT));
3102		- roce_set_field(qpc_mask->byte_104_rq_sge,
3103		- V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3104		- V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
3105		-
3106		- roce_set_field(context->byte_108_rx_reqepsn,
3107		- V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3108		- V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
3109		- roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3110		- V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3111		- V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
3112		-
3113	4198	roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3114		- V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
	4199	+ V2_QPC_BYTE_132_TRRL_BA_S, trrl_ba >> 4);
3115	4200	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3116	4201	V2_QPC_BYTE_132_TRRL_BA_S, 0);
3117		- context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4));
	4202	+ context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4));
3118	4203	qpc_mask->trrl_ba = 0;
3119	4204	roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3120	4205	V2_QPC_BYTE_140_TRRL_BA_S,
3121		- (u32)(dma_handle_3 >> (32 + 16 + 4)));
	4206	+ (u32)(trrl_ba >> (32 + 16 + 4)));
3122	4207	roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3123	4208	V2_QPC_BYTE_140_TRRL_BA_S, 0);
3124	4209
3125		- context->irrl_ba = (u32)(dma_handle_2 >> 6);
	4210	+ context->irrl_ba = cpu_to_le32(irrl_ba >> 6);
3126	4211	qpc_mask->irrl_ba = 0;
3127	4212	roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3128	4213	V2_QPC_BYTE_208_IRRL_BA_S,
3129		- dma_handle_2 >> (32 + 6));
	4214	+ irrl_ba >> (32 + 6));
3130	4215	roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3131	4216	V2_QPC_BYTE_208_IRRL_BA_S, 0);
3132	4217
..	..	@@ -3141,20 +4226,12 @@
3141	4226	port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
3142	4227
3143	4228	smac = (u8 *)hr_dev->dev_addr[port];
	4229	+ dmac = (u8 *)attr->ah_attr.roce.dmac;
3144	4230	/* when dmac equals smac or loop_idc is 1, it should loopback */
3145	4231	if (ether_addr_equal_unaligned(dmac, smac) \|\|
3146	4232	hr_dev->loop_idc == 0x1) {
3147	4233	roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
3148	4234	roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
3149		- }
3150		-
3151		- if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
3152		- attr->max_dest_rd_atomic) {
3153		- roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
3154		- V2_QPC_BYTE_140_RR_MAX_S,
3155		- fls(attr->max_dest_rd_atomic - 1));
3156		- roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
3157		- V2_QPC_BYTE_140_RR_MAX_S, 0);
3158	4235	}
3159	4236
3160	4237	if (attr_mask & IB_QP_DEST_QPN) {
..	..	@@ -3167,45 +4244,45 @@
3167	4244	/* Configure GID index */
3168	4245	port_num = rdma_ah_get_port_num(&attr->ah_attr);
3169	4246	roce_set_field(context->byte_20_smac_sgid_idx,
3170		- V2_QPC_BYTE_20_SGID_IDX_M,
3171		- V2_QPC_BYTE_20_SGID_IDX_S,
	4247	+ V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
3172	4248	hns_get_gid_index(hr_dev, port_num - 1,
3173	4249	grh->sgid_index));
3174	4250	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3175		- V2_QPC_BYTE_20_SGID_IDX_M,
3176		- V2_QPC_BYTE_20_SGID_IDX_S, 0);
3177		- memcpy(&(context->dmac), dmac, 4);
	4251	+ V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
	4252	+
	4253	+ memcpy(&(context->dmac), dmac, sizeof(u32));
3178	4254	roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3179	4255	V2_QPC_BYTE_52_DMAC_S, ((u16 )(&dmac[4])));
3180	4256	qpc_mask->dmac = 0;
3181	4257	roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3182	4258	V2_QPC_BYTE_52_DMAC_S, 0);
3183	4259
	4260	+ mtu = get_mtu(ibqp, attr);
	4261	+ hr_qp->path_mtu = mtu;
	4262	+
	4263	+ if (attr_mask & IB_QP_PATH_MTU) {
	4264	+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
	4265	+ V2_QPC_BYTE_24_MTU_S, mtu);
	4266	+ roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
	4267	+ V2_QPC_BYTE_24_MTU_S, 0);
	4268	+ }
	4269	+
	4270	+#define MAX_LP_MSG_LEN 65536
	4271	+ /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 64KB */
	4272	+ lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / ib_mtu_enum_to_int(mtu));
	4273	+
3184	4274	roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3185		- V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
	4275	+ V2_QPC_BYTE_56_LP_PKTN_INI_S, lp_pktn_ini);
3186	4276	roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3187	4277	V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
3188	4278
3189		- if (ibqp->qp_type == IB_QPT_GSI \|\| ibqp->qp_type == IB_QPT_UD)
3190		- roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3191		- V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
3192		- else if (attr_mask & IB_QP_PATH_MTU)
3193		- roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3194		- V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
	4279	+ /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */
	4280	+ roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
	4281	+ V2_QPC_BYTE_172_ACK_REQ_FREQ_S, lp_pktn_ini);
	4282	+ roce_set_field(qpc_mask->byte_172_sq_psn,
	4283	+ V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
	4284	+ V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
3195	4285
3196		- roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3197		- V2_QPC_BYTE_24_MTU_S, 0);
3198		-
3199		- roce_set_field(context->byte_84_rq_ci_pi,
3200		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3201		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
3202		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3203		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3204		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3205		-
3206		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3207		- V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3208		- V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3209	4286	roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3210	4287	V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3211	4288	roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
..	..	@@ -3217,16 +4294,12 @@
3217	4294	context->rq_rnr_timer = 0;
3218	4295	qpc_mask->rq_rnr_timer = 0;
3219	4296
3220		- roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3221		- V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
3222		- roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3223		- V2_QPC_BYTE_152_RAQ_PSN_S, 0);
3224		-
3225	4297	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3226	4298	V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3227	4299	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3228	4300	V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3229	4301
	4302	+ /* rocee send 2^lp_sgen_ini segs every time */
3230	4303	roce_set_field(context->byte_168_irrl_idx,
3231	4304	V2_QPC_BYTE_168_LP_SGEN_INI_M,
3232	4305	V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
..	..	@@ -3244,74 +4317,20 @@
3244	4317	{
3245	4318	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3246	4319	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3247		- struct device *dev = hr_dev->dev;
3248		- dma_addr_t dma_handle;
3249		- u32 page_size;
3250		- u64 *mtts;
3251		-
3252		- /* Search qp buf's mtts */
3253		- mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
3254		- hr_qp->mtt.first_seg, &dma_handle);
3255		- if (!mtts) {
3256		- dev_err(dev, "qp buf pa find failed\n");
3257		- return -EINVAL;
3258		- }
	4320	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4321	+ int ret;
3259	4322
3260	4323	/* Not support alternate path and path migration */
3261		- if ((attr_mask & IB_QP_ALT_PATH) \|\|
3262		- (attr_mask & IB_QP_PATH_MIG_STATE)) {
3263		- dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
	4324	+ if (attr_mask & (IB_QP_ALT_PATH \| IB_QP_PATH_MIG_STATE)) {
	4325	+ ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
3264	4326	return -EINVAL;
3265	4327	}
3266	4328
3267		- /*
3268		- * In v2 engine, software pass context and context mask to hardware
3269		- * when modifying qp. If software need modify some fields in context,
3270		- * we should set all bits of the relevant fields in context mask to
3271		- * 0 at the same time, else set them to 0x1.
3272		- */
3273		- roce_set_field(context->byte_60_qpst_mapid,
3274		- V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M,
3275		- V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S, attr->retry_cnt);
3276		- roce_set_field(qpc_mask->byte_60_qpst_mapid,
3277		- V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M,
3278		- V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S, 0);
3279		-
3280		- context->sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
3281		- roce_set_field(context->byte_168_irrl_idx,
3282		- V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3283		- V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
3284		- mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3285		- qpc_mask->sq_cur_blk_addr = 0;
3286		- roce_set_field(qpc_mask->byte_168_irrl_idx,
3287		- V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3288		- V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
3289		-
3290		- page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3291		- context->sq_cur_sge_blk_addr =
3292		- ((ibqp->qp_type == IB_QPT_GSI) \|\| hr_qp->sq.max_gs > 2) ?
3293		- ((u32)(mtts[hr_qp->sge.offset / page_size]
3294		- >> PAGE_ADDR_SHIFT)) : 0;
3295		- roce_set_field(context->byte_184_irrl_idx,
3296		- V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3297		- V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
3298		- ((ibqp->qp_type == IB_QPT_GSI) \|\| hr_qp->sq.max_gs > 2) ?
3299		- (mtts[hr_qp->sge.offset / page_size] >>
3300		- (32 + PAGE_ADDR_SHIFT)) : 0);
3301		- qpc_mask->sq_cur_sge_blk_addr = 0;
3302		- roce_set_field(qpc_mask->byte_184_irrl_idx,
3303		- V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3304		- V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
3305		-
3306		- context->rx_sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
3307		- roce_set_field(context->byte_232_irrl_sge,
3308		- V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3309		- V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
3310		- mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3311		- qpc_mask->rx_sq_cur_blk_addr = 0;
3312		- roce_set_field(qpc_mask->byte_232_irrl_sge,
3313		- V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3314		- V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
	4329	+ ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);
	4330	+ if (ret) {
	4331	+ ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);
	4332	+ return ret;
	4333	+ }
3315	4334
3316	4335	/*
3317	4336	* Set some fields in context to zero, Because the default values
..	..	@@ -3326,13 +4345,6 @@
3326	4345	V2_QPC_BYTE_240_RX_ACK_MSN_M,
3327	4346	V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3328	4347
3329		- roce_set_field(context->byte_244_rnr_rxack,
3330		- V2_QPC_BYTE_244_RX_ACK_EPSN_M,
3331		- V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
3332		- roce_set_field(qpc_mask->byte_244_rnr_rxack,
3333		- V2_QPC_BYTE_244_RX_ACK_EPSN_M,
3334		- V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
3335		-
3336	4348	roce_set_field(qpc_mask->byte_248_ack_psn,
3337	4349	V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3338	4350	V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
..	..	@@ -3346,27 +4358,6 @@
3346	4358	V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3347	4359	V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3348	4360
3349		- roce_set_field(context->byte_220_retry_psn_msn,
3350		- V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
3351		- V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
3352		- roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3353		- V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
3354		- V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
3355		-
3356		- roce_set_field(context->byte_224_retry_msg,
3357		- V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
3358		- V2_QPC_BYTE_224_RETRY_MSG_PSN_S, attr->sq_psn >> 16);
3359		- roce_set_field(qpc_mask->byte_224_retry_msg,
3360		- V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
3361		- V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
3362		-
3363		- roce_set_field(context->byte_224_retry_msg,
3364		- V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
3365		- V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, attr->sq_psn);
3366		- roce_set_field(qpc_mask->byte_224_retry_msg,
3367		- V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
3368		- V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
3369		-
3370	4361	roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3371	4362	V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3372	4363	V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
..	..	@@ -3377,51 +4368,311 @@
3377	4368	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3378	4369	V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3379	4370
3380		- roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M,
3381		- V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt);
3382		- roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M,
3383		- V2_QPC_BYTE_212_RETRY_CNT_S, 0);
3384		-
3385		- roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
3386		- V2_QPC_BYTE_212_RETRY_NUM_INIT_S, attr->retry_cnt);
3387		- roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
3388		- V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
3389		-
3390		- roce_set_field(context->byte_244_rnr_rxack,
3391		- V2_QPC_BYTE_244_RNR_NUM_INIT_M,
3392		- V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
3393		- roce_set_field(qpc_mask->byte_244_rnr_rxack,
3394		- V2_QPC_BYTE_244_RNR_NUM_INIT_M,
3395		- V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
3396		-
3397		- roce_set_field(context->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M,
3398		- V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
3399		- roce_set_field(qpc_mask->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M,
3400		- V2_QPC_BYTE_244_RNR_CNT_S, 0);
3401		-
3402	4371	roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3403	4372	V2_QPC_BYTE_212_LSN_S, 0x100);
3404	4373	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3405	4374	V2_QPC_BYTE_212_LSN_S, 0);
3406	4375
3407		- if (attr_mask & IB_QP_TIMEOUT) {
3408		- roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_AT_M,
3409		- V2_QPC_BYTE_28_AT_S, attr->timeout);
3410		- roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_AT_M,
3411		- V2_QPC_BYTE_28_AT_S, 0);
3412		- }
3413		-
3414		- roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3415		- V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
3416		- roce_set_field(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3417		- V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
3418		-
3419	4376	roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3420	4377	V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3421		- roce_set_field(context->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
3422		- V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
3423		- roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
3424		- V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
	4378	+
	4379	+ return 0;
	4380	+}
	4381	+
	4382	+static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
	4383	+{
	4384	+ if (!fl)
	4385	+ fl = rdma_calc_flow_label(lqpn, rqpn);
	4386	+
	4387	+ return rdma_flow_label_to_udp_sport(fl);
	4388	+}
	4389	+
	4390	+static int hns_roce_v2_set_path(struct ib_qp *ibqp,
	4391	+ const struct ib_qp_attr *attr,
	4392	+ int attr_mask,
	4393	+ struct hns_roce_v2_qp_context *context,
	4394	+ struct hns_roce_v2_qp_context *qpc_mask)
	4395	+{
	4396	+ const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
	4397	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4398	+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
	4399	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4400	+ const struct ib_gid_attr *gid_attr = NULL;
	4401	+ int is_roce_protocol;
	4402	+ u16 vlan_id = 0xffff;
	4403	+ bool is_udp = false;
	4404	+ u8 ib_port;
	4405	+ u8 hr_port;
	4406	+ int ret;
	4407	+
	4408	+ ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
	4409	+ hr_port = ib_port - 1;
	4410	+ is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
	4411	+ rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
	4412	+
	4413	+ if (is_roce_protocol) {
	4414	+ gid_attr = attr->ah_attr.grh.sgid_attr;
	4415	+ ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
	4416	+ if (ret)
	4417	+ return ret;
	4418	+
	4419	+ if (gid_attr)
	4420	+ is_udp = (gid_attr->gid_type ==
	4421	+ IB_GID_TYPE_ROCE_UDP_ENCAP);
	4422	+ }
	4423	+
	4424	+ if (vlan_id < VLAN_N_VID) {
	4425	+ roce_set_bit(context->byte_76_srqn_op_en,
	4426	+ V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
	4427	+ roce_set_bit(qpc_mask->byte_76_srqn_op_en,
	4428	+ V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
	4429	+ roce_set_bit(context->byte_168_irrl_idx,
	4430	+ V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
	4431	+ roce_set_bit(qpc_mask->byte_168_irrl_idx,
	4432	+ V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
	4433	+ }
	4434	+
	4435	+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
	4436	+ V2_QPC_BYTE_24_VLAN_ID_S, vlan_id);
	4437	+ roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
	4438	+ V2_QPC_BYTE_24_VLAN_ID_S, 0);
	4439	+
	4440	+ if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
	4441	+ ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
	4442	+ grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
	4443	+ return -EINVAL;
	4444	+ }
	4445	+
	4446	+ if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
	4447	+ ibdev_err(ibdev, "ah attr is not RDMA roce type\n");
	4448	+ return -EINVAL;
	4449	+ }
	4450	+
	4451	+ roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
	4452	+ V2_QPC_BYTE_52_UDPSPN_S,
	4453	+ is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num,
	4454	+ attr->dest_qp_num) : 0);
	4455	+
	4456	+ roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
	4457	+ V2_QPC_BYTE_52_UDPSPN_S, 0);
	4458	+
	4459	+ roce_set_field(context->byte_20_smac_sgid_idx,
	4460	+ V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
	4461	+ grh->sgid_index);
	4462	+
	4463	+ roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
	4464	+ V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
	4465	+
	4466	+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
	4467	+ V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
	4468	+ roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
	4469	+ V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
	4470	+
	4471	+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
	4472	+ V2_QPC_BYTE_24_TC_S, get_tclass(&attr->ah_attr.grh));
	4473	+ roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
	4474	+ V2_QPC_BYTE_24_TC_S, 0);
	4475	+
	4476	+ roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
	4477	+ V2_QPC_BYTE_28_FL_S, grh->flow_label);
	4478	+ roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
	4479	+ V2_QPC_BYTE_28_FL_S, 0);
	4480	+ memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
	4481	+ memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
	4482	+
	4483	+ hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
	4484	+ if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) {
	4485	+ ibdev_err(ibdev,
	4486	+ "failed to fill QPC, sl (%d) shouldn't be larger than %d.\n",
	4487	+ hr_qp->sl, MAX_SERVICE_LEVEL);
	4488	+ return -EINVAL;
	4489	+ }
	4490	+
	4491	+ roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
	4492	+ V2_QPC_BYTE_28_SL_S, hr_qp->sl);
	4493	+ roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
	4494	+ V2_QPC_BYTE_28_SL_S, 0);
	4495	+
	4496	+ return 0;
	4497	+}
	4498	+
	4499	+static bool check_qp_state(enum ib_qp_state cur_state,
	4500	+ enum ib_qp_state new_state)
	4501	+{
	4502	+ static const bool sm[][IB_QPS_ERR + 1] = {
	4503	+ [IB_QPS_RESET] = { [IB_QPS_RESET] = true,
	4504	+ [IB_QPS_INIT] = true },
	4505	+ [IB_QPS_INIT] = { [IB_QPS_RESET] = true,
	4506	+ [IB_QPS_INIT] = true,
	4507	+ [IB_QPS_RTR] = true,
	4508	+ [IB_QPS_ERR] = true },
	4509	+ [IB_QPS_RTR] = { [IB_QPS_RESET] = true,
	4510	+ [IB_QPS_RTS] = true,
	4511	+ [IB_QPS_ERR] = true },
	4512	+ [IB_QPS_RTS] = { [IB_QPS_RESET] = true,
	4513	+ [IB_QPS_RTS] = true,
	4514	+ [IB_QPS_ERR] = true },
	4515	+ [IB_QPS_SQD] = {},
	4516	+ [IB_QPS_SQE] = {},
	4517	+ [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }
	4518	+ };
	4519	+
	4520	+ return sm[cur_state][new_state];
	4521	+}
	4522	+
	4523	+static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
	4524	+ const struct ib_qp_attr *attr,
	4525	+ int attr_mask,
	4526	+ enum ib_qp_state cur_state,
	4527	+ enum ib_qp_state new_state,
	4528	+ struct hns_roce_v2_qp_context *context,
	4529	+ struct hns_roce_v2_qp_context *qpc_mask)
	4530	+{
	4531	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4532	+ int ret = 0;
	4533	+
	4534	+ if (!check_qp_state(cur_state, new_state)) {
	4535	+ ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n");
	4536	+ return -EINVAL;
	4537	+ }
	4538	+
	4539	+ if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
	4540	+ memset(qpc_mask, 0, hr_dev->caps.qpc_sz);
	4541	+ modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
	4542	+ qpc_mask);
	4543	+ } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
	4544	+ modify_qp_init_to_init(ibqp, attr, attr_mask, context,
	4545	+ qpc_mask);
	4546	+ } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
	4547	+ ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
	4548	+ qpc_mask);
	4549	+ } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
	4550	+ ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
	4551	+ qpc_mask);
	4552	+ }
	4553	+
	4554	+ return ret;
	4555	+}
	4556	+
	4557	+static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
	4558	+ const struct ib_qp_attr *attr,
	4559	+ int attr_mask,
	4560	+ struct hns_roce_v2_qp_context *context,
	4561	+ struct hns_roce_v2_qp_context *qpc_mask)
	4562	+{
	4563	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4564	+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
	4565	+ int ret = 0;
	4566	+
	4567	+ if (attr_mask & IB_QP_AV) {
	4568	+ ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
	4569	+ qpc_mask);
	4570	+ if (ret)
	4571	+ return ret;
	4572	+ }
	4573	+
	4574	+ if (attr_mask & IB_QP_TIMEOUT) {
	4575	+ if (attr->timeout < 31) {
	4576	+ roce_set_field(context->byte_28_at_fl,
	4577	+ V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
	4578	+ attr->timeout);
	4579	+ roce_set_field(qpc_mask->byte_28_at_fl,
	4580	+ V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
	4581	+ 0);
	4582	+ } else {
	4583	+ ibdev_warn(&hr_dev->ib_dev,
	4584	+ "Local ACK timeout shall be 0 to 30.\n");
	4585	+ }
	4586	+ }
	4587	+
	4588	+ if (attr_mask & IB_QP_RETRY_CNT) {
	4589	+ roce_set_field(context->byte_212_lsn,
	4590	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
	4591	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
	4592	+ attr->retry_cnt);
	4593	+ roce_set_field(qpc_mask->byte_212_lsn,
	4594	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
	4595	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
	4596	+
	4597	+ roce_set_field(context->byte_212_lsn,
	4598	+ V2_QPC_BYTE_212_RETRY_CNT_M,
	4599	+ V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt);
	4600	+ roce_set_field(qpc_mask->byte_212_lsn,
	4601	+ V2_QPC_BYTE_212_RETRY_CNT_M,
	4602	+ V2_QPC_BYTE_212_RETRY_CNT_S, 0);
	4603	+ }
	4604	+
	4605	+ if (attr_mask & IB_QP_RNR_RETRY) {
	4606	+ roce_set_field(context->byte_244_rnr_rxack,
	4607	+ V2_QPC_BYTE_244_RNR_NUM_INIT_M,
	4608	+ V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
	4609	+ roce_set_field(qpc_mask->byte_244_rnr_rxack,
	4610	+ V2_QPC_BYTE_244_RNR_NUM_INIT_M,
	4611	+ V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
	4612	+
	4613	+ roce_set_field(context->byte_244_rnr_rxack,
	4614	+ V2_QPC_BYTE_244_RNR_CNT_M,
	4615	+ V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
	4616	+ roce_set_field(qpc_mask->byte_244_rnr_rxack,
	4617	+ V2_QPC_BYTE_244_RNR_CNT_M,
	4618	+ V2_QPC_BYTE_244_RNR_CNT_S, 0);
	4619	+ }
	4620	+
	4621	+ /* RC&UC&UD required attr */
	4622	+ if (attr_mask & IB_QP_SQ_PSN) {
	4623	+ roce_set_field(context->byte_172_sq_psn,
	4624	+ V2_QPC_BYTE_172_SQ_CUR_PSN_M,
	4625	+ V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
	4626	+ roce_set_field(qpc_mask->byte_172_sq_psn,
	4627	+ V2_QPC_BYTE_172_SQ_CUR_PSN_M,
	4628	+ V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
	4629	+
	4630	+ roce_set_field(context->byte_196_sq_psn,
	4631	+ V2_QPC_BYTE_196_SQ_MAX_PSN_M,
	4632	+ V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
	4633	+ roce_set_field(qpc_mask->byte_196_sq_psn,
	4634	+ V2_QPC_BYTE_196_SQ_MAX_PSN_M,
	4635	+ V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
	4636	+
	4637	+ roce_set_field(context->byte_220_retry_psn_msn,
	4638	+ V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
	4639	+ V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
	4640	+ roce_set_field(qpc_mask->byte_220_retry_psn_msn,
	4641	+ V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
	4642	+ V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
	4643	+
	4644	+ roce_set_field(context->byte_224_retry_msg,
	4645	+ V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
	4646	+ V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
	4647	+ attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
	4648	+ roce_set_field(qpc_mask->byte_224_retry_msg,
	4649	+ V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
	4650	+ V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
	4651	+
	4652	+ roce_set_field(context->byte_224_retry_msg,
	4653	+ V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
	4654	+ V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
	4655	+ attr->sq_psn);
	4656	+ roce_set_field(qpc_mask->byte_224_retry_msg,
	4657	+ V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
	4658	+ V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
	4659	+
	4660	+ roce_set_field(context->byte_244_rnr_rxack,
	4661	+ V2_QPC_BYTE_244_RX_ACK_EPSN_M,
	4662	+ V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
	4663	+ roce_set_field(qpc_mask->byte_244_rnr_rxack,
	4664	+ V2_QPC_BYTE_244_RX_ACK_EPSN_M,
	4665	+ V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
	4666	+ }
	4667	+
	4668	+ if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
	4669	+ attr->max_dest_rd_atomic) {
	4670	+ roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
	4671	+ V2_QPC_BYTE_140_RR_MAX_S,
	4672	+ fls(attr->max_dest_rd_atomic - 1));
	4673	+ roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
	4674	+ V2_QPC_BYTE_140_RR_MAX_S, 0);
	4675	+ }
3425	4676
3426	4677	if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
3427	4678	roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
..	..	@@ -3431,197 +4682,51 @@
3431	4682	V2_QPC_BYTE_208_SR_MAX_M,
3432	4683	V2_QPC_BYTE_208_SR_MAX_S, 0);
3433	4684	}
3434		- return 0;
3435		-}
3436		-
3437		-static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
3438		- const struct ib_qp_attr *attr,
3439		- int attr_mask, enum ib_qp_state cur_state,
3440		- enum ib_qp_state new_state)
3441		-{
3442		- struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3443		- struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3444		- struct hns_roce_v2_qp_context *context;
3445		- struct hns_roce_v2_qp_context *qpc_mask;
3446		- struct device *dev = hr_dev->dev;
3447		- int ret = -EINVAL;
3448		-
3449		- context = kcalloc(2, sizeof(*context), GFP_ATOMIC);
3450		- if (!context)
3451		- return -ENOMEM;
3452		-
3453		- qpc_mask = context + 1;
3454		- /*
3455		- * In v2 engine, software pass context and context mask to hardware
3456		- * when modifying qp. If software need modify some fields in context,
3457		- * we should set all bits of the relevant fields in context mask to
3458		- * 0 at the same time, else set them to 0x1.
3459		- */
3460		- memset(qpc_mask, 0xff, sizeof(*qpc_mask));
3461		- if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3462		- modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
3463		- qpc_mask);
3464		- } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
3465		- modify_qp_init_to_init(ibqp, attr, attr_mask, context,
3466		- qpc_mask);
3467		- } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3468		- ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
3469		- qpc_mask);
3470		- if (ret)
3471		- goto out;
3472		- } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
3473		- ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
3474		- qpc_mask);
3475		- if (ret)
3476		- goto out;
3477		- } else if ((cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) \|\|
3478		- (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS) \|\|
3479		- (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD) \|\|
3480		- (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD) \|\|
3481		- (cur_state == IB_QPS_SQD && new_state == IB_QPS_RTS) \|\|
3482		- (cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) \|\|
3483		- (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) \|\|
3484		- (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) \|\|
3485		- (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) \|\|
3486		- (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) \|\|
3487		- (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) \|\|
3488		- (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) \|\|
3489		- (cur_state == IB_QPS_SQD && new_state == IB_QPS_ERR) \|\|
3490		- (cur_state == IB_QPS_SQE && new_state == IB_QPS_ERR) \|\|
3491		- (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR)) {
3492		- /* Nothing */
3493		- ;
3494		- } else {
3495		- dev_err(dev, "Illegal state for QP!\n");
3496		- ret = -EINVAL;
3497		- goto out;
3498		- }
3499		-
3500		- /* When QP state is err, SQ and RQ WQE should be flushed */
3501		- if (new_state == IB_QPS_ERR) {
3502		- roce_set_field(context->byte_160_sq_ci_pi,
3503		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3504		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
3505		- hr_qp->sq.head);
3506		- roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3507		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3508		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
3509		-
3510		- if (!ibqp->srq) {
3511		- roce_set_field(context->byte_84_rq_ci_pi,
3512		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3513		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
3514		- hr_qp->rq.head);
3515		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3516		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3517		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3518		- }
3519		- }
3520		-
3521		- if (attr_mask & IB_QP_AV) {
3522		- const struct ib_global_route *grh =
3523		- rdma_ah_read_grh(&attr->ah_attr);
3524		- const struct ib_gid_attr *gid_attr = NULL;
3525		- u8 src_mac[ETH_ALEN];
3526		- int is_roce_protocol;
3527		- u16 vlan = 0xffff;
3528		- u8 ib_port;
3529		- u8 hr_port;
3530		-
3531		- ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num :
3532		- hr_qp->port + 1;
3533		- hr_port = ib_port - 1;
3534		- is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
3535		- rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
3536		-
3537		- if (is_roce_protocol) {
3538		- gid_attr = attr->ah_attr.grh.sgid_attr;
3539		- vlan = rdma_vlan_dev_vlan_id(gid_attr->ndev);
3540		- memcpy(src_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
3541		- }
3542		-
3543		- roce_set_field(context->byte_24_mtu_tc,
3544		- V2_QPC_BYTE_24_VLAN_ID_M,
3545		- V2_QPC_BYTE_24_VLAN_ID_S, vlan);
3546		- roce_set_field(qpc_mask->byte_24_mtu_tc,
3547		- V2_QPC_BYTE_24_VLAN_ID_M,
3548		- V2_QPC_BYTE_24_VLAN_ID_S, 0);
3549		-
3550		- if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
3551		- dev_err(hr_dev->dev,
3552		- "sgid_index(%u) too large. max is %d\n",
3553		- grh->sgid_index,
3554		- hr_dev->caps.gid_table_len[hr_port]);
3555		- ret = -EINVAL;
3556		- goto out;
3557		- }
3558		-
3559		- if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
3560		- dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n");
3561		- ret = -EINVAL;
3562		- goto out;
3563		- }
3564		-
3565		- roce_set_field(context->byte_52_udpspn_dmac,
3566		- V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S,
3567		- (gid_attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) ?
3568		- 0 : 0x12b7);
3569		-
3570		- roce_set_field(qpc_mask->byte_52_udpspn_dmac,
3571		- V2_QPC_BYTE_52_UDPSPN_M,
3572		- V2_QPC_BYTE_52_UDPSPN_S, 0);
3573		-
3574		- roce_set_field(context->byte_20_smac_sgid_idx,
3575		- V2_QPC_BYTE_20_SGID_IDX_M,
3576		- V2_QPC_BYTE_20_SGID_IDX_S, grh->sgid_index);
3577		-
3578		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3579		- V2_QPC_BYTE_20_SGID_IDX_M,
3580		- V2_QPC_BYTE_20_SGID_IDX_S, 0);
3581		-
3582		- roce_set_field(context->byte_24_mtu_tc,
3583		- V2_QPC_BYTE_24_HOP_LIMIT_M,
3584		- V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
3585		- roce_set_field(qpc_mask->byte_24_mtu_tc,
3586		- V2_QPC_BYTE_24_HOP_LIMIT_M,
3587		- V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
3588		-
3589		- roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
3590		- V2_QPC_BYTE_24_TC_S, grh->traffic_class);
3591		- roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
3592		- V2_QPC_BYTE_24_TC_S, 0);
3593		- roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
3594		- V2_QPC_BYTE_28_FL_S, grh->flow_label);
3595		- roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
3596		- V2_QPC_BYTE_28_FL_S, 0);
3597		- memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
3598		- memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
3599		- roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
3600		- V2_QPC_BYTE_28_SL_S,
3601		- rdma_ah_get_sl(&attr->ah_attr));
3602		- roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
3603		- V2_QPC_BYTE_28_SL_S, 0);
3604		- hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
3605		- }
3606	4685
3607	4686	if (attr_mask & (IB_QP_ACCESS_FLAGS \| IB_QP_MAX_DEST_RD_ATOMIC))
3608	4687	set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
3609	4688
3610		- /* Every status migrate must change state */
3611		- roce_set_field(context->byte_60_qpst_mapid, V2_QPC_BYTE_60_QP_ST_M,
3612		- V2_QPC_BYTE_60_QP_ST_S, new_state);
3613		- roce_set_field(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_QP_ST_M,
3614		- V2_QPC_BYTE_60_QP_ST_S, 0);
3615		-
3616		- /* SW pass context to HW */
3617		- ret = hns_roce_v2_qp_modify(hr_dev, &hr_qp->mtt, cur_state, new_state,
3618		- context, hr_qp);
3619		- if (ret) {
3620		- dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
3621		- goto out;
	4689	+ if (attr_mask & IB_QP_MIN_RNR_TIMER) {
	4690	+ roce_set_field(context->byte_80_rnr_rx_cqn,
	4691	+ V2_QPC_BYTE_80_MIN_RNR_TIME_M,
	4692	+ V2_QPC_BYTE_80_MIN_RNR_TIME_S,
	4693	+ attr->min_rnr_timer);
	4694	+ roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
	4695	+ V2_QPC_BYTE_80_MIN_RNR_TIME_M,
	4696	+ V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
3622	4697	}
3623	4698
3624		- hr_qp->state = new_state;
	4699	+ /* RC&UC required attr */
	4700	+ if (attr_mask & IB_QP_RQ_PSN) {
	4701	+ roce_set_field(context->byte_108_rx_reqepsn,
	4702	+ V2_QPC_BYTE_108_RX_REQ_EPSN_M,
	4703	+ V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
	4704	+ roce_set_field(qpc_mask->byte_108_rx_reqepsn,
	4705	+ V2_QPC_BYTE_108_RX_REQ_EPSN_M,
	4706	+ V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
	4707	+
	4708	+ roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
	4709	+ V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
	4710	+ roce_set_field(qpc_mask->byte_152_raq,
	4711	+ V2_QPC_BYTE_152_RAQ_PSN_M,
	4712	+ V2_QPC_BYTE_152_RAQ_PSN_S, 0);
	4713	+ }
	4714	+
	4715	+ if (attr_mask & IB_QP_QKEY) {
	4716	+ context->qkey_xrcd = cpu_to_le32(attr->qkey);
	4717	+ qpc_mask->qkey_xrcd = 0;
	4718	+ hr_qp->qkey = attr->qkey;
	4719	+ }
	4720	+
	4721	+ return ret;
	4722	+}
	4723	+
	4724	+static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
	4725	+ const struct ib_qp_attr *attr,
	4726	+ int attr_mask)
	4727	+{
	4728	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4729	+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3625	4730
3626	4731	if (attr_mask & IB_QP_ACCESS_FLAGS)
3627	4732	hr_qp->atomic_rd_en = attr->qp_access_flags;
..	..	@@ -3632,6 +4737,90 @@
3632	4737	hr_qp->port = attr->port_num - 1;
3633	4738	hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
3634	4739	}
	4740	+}
	4741	+
	4742	+static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
	4743	+ const struct ib_qp_attr *attr,
	4744	+ int attr_mask, enum ib_qp_state cur_state,
	4745	+ enum ib_qp_state new_state)
	4746	+{
	4747	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4748	+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
	4749	+ struct hns_roce_v2_qp_context ctx[2];
	4750	+ struct hns_roce_v2_qp_context *context = ctx;
	4751	+ struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
	4752	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4753	+ unsigned long sq_flag = 0;
	4754	+ unsigned long rq_flag = 0;
	4755	+ int ret;
	4756	+
	4757	+ /*
	4758	+ * In v2 engine, software pass context and context mask to hardware
	4759	+ * when modifying qp. If software need modify some fields in context,
	4760	+ * we should set all bits of the relevant fields in context mask to
	4761	+ * 0 at the same time, else set them to 0x1.
	4762	+ */
	4763	+ memset(context, 0, hr_dev->caps.qpc_sz);
	4764	+ memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
	4765	+
	4766	+ ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
	4767	+ new_state, context, qpc_mask);
	4768	+ if (ret)
	4769	+ goto out;
	4770	+
	4771	+ /* When QP state is err, SQ and RQ WQE should be flushed */
	4772	+ if (new_state == IB_QPS_ERR) {
	4773	+ spin_lock_irqsave(&hr_qp->sq.lock, sq_flag);
	4774	+ hr_qp->state = IB_QPS_ERR;
	4775	+ roce_set_field(context->byte_160_sq_ci_pi,
	4776	+ V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
	4777	+ V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
	4778	+ hr_qp->sq.head);
	4779	+ roce_set_field(qpc_mask->byte_160_sq_ci_pi,
	4780	+ V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
	4781	+ V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
	4782	+ spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
	4783	+
	4784	+ if (!ibqp->srq) {
	4785	+ spin_lock_irqsave(&hr_qp->rq.lock, rq_flag);
	4786	+ roce_set_field(context->byte_84_rq_ci_pi,
	4787	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
	4788	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
	4789	+ hr_qp->rq.head);
	4790	+ roce_set_field(qpc_mask->byte_84_rq_ci_pi,
	4791	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
	4792	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
	4793	+ spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag);
	4794	+ }
	4795	+ }
	4796	+
	4797	+ /* Configure the optional fields */
	4798	+ ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
	4799	+ qpc_mask);
	4800	+ if (ret)
	4801	+ goto out;
	4802	+
	4803	+ roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
	4804	+ ibqp->srq ? 1 : 0);
	4805	+ roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
	4806	+ V2_QPC_BYTE_108_INV_CREDIT_S, 0);
	4807	+
	4808	+ /* Every status migrate must change state */
	4809	+ roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
	4810	+ V2_QPC_BYTE_60_QP_ST_S, new_state);
	4811	+ roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
	4812	+ V2_QPC_BYTE_60_QP_ST_S, 0);
	4813	+
	4814	+ /* SW pass context to HW */
	4815	+ ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);
	4816	+ if (ret) {
	4817	+ ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret);
	4818	+ goto out;
	4819	+ }
	4820	+
	4821	+ hr_qp->state = new_state;
	4822	+
	4823	+ hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
3635	4824
3636	4825	if (new_state == IB_QPS_RESET && !ibqp->uobject) {
3637	4826	hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
..	..	@@ -3644,30 +4833,29 @@
3644	4833	hr_qp->rq.tail = 0;
3645	4834	hr_qp->sq.head = 0;
3646	4835	hr_qp->sq.tail = 0;
3647		- hr_qp->sq_next_wqe = 0;
3648	4836	hr_qp->next_sge = 0;
3649	4837	if (hr_qp->rq.wqe_cnt)
3650	4838	*hr_qp->rdb.db_record = 0;
3651	4839	}
3652	4840
3653	4841	out:
3654		- kfree(context);
3655	4842	return ret;
3656	4843	}
3657	4844
3658		-static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
	4845	+static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
3659	4846	{
3660		- switch (state) {
3661		- case HNS_ROCE_QP_ST_RST: return IB_QPS_RESET;
3662		- case HNS_ROCE_QP_ST_INIT: return IB_QPS_INIT;
3663		- case HNS_ROCE_QP_ST_RTR: return IB_QPS_RTR;
3664		- case HNS_ROCE_QP_ST_RTS: return IB_QPS_RTS;
3665		- case HNS_ROCE_QP_ST_SQ_DRAINING:
3666		- case HNS_ROCE_QP_ST_SQD: return IB_QPS_SQD;
3667		- case HNS_ROCE_QP_ST_SQER: return IB_QPS_SQE;
3668		- case HNS_ROCE_QP_ST_ERR: return IB_QPS_ERR;
3669		- default: return -1;
3670		- }
	4847	+ static const enum ib_qp_state map[] = {
	4848	+ [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,
	4849	+ [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,
	4850	+ [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,
	4851	+ [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,
	4852	+ [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,
	4853	+ [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,
	4854	+ [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,
	4855	+ [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD
	4856	+ };
	4857	+
	4858	+ return (state < ARRAY_SIZE(map)) ? map[state] : -1;
3671	4859	}
3672	4860
3673	4861	static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
..	..	@@ -3684,12 +4872,10 @@
3684	4872	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
3685	4873	HNS_ROCE_CMD_QUERY_QPC,
3686	4874	HNS_ROCE_CMD_TIMEOUT_MSECS);
3687		- if (ret) {
3688		- dev_err(hr_dev->dev, "QUERY QP cmd process error\n");
	4875	+ if (ret)
3689	4876	goto out;
3690		- }
3691	4877
3692		- memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
	4878	+ memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz);
3693	4879
3694	4880	out:
3695	4881	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
..	..	@@ -3702,15 +4888,11 @@
3702	4888	{
3703	4889	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3704	4890	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3705		- struct hns_roce_v2_qp_context *context;
3706		- struct device *dev = hr_dev->dev;
	4891	+ struct hns_roce_v2_qp_context context = {};
	4892	+ struct ib_device *ibdev = &hr_dev->ib_dev;
3707	4893	int tmp_qp_state;
3708	4894	int state;
3709	4895	int ret;
3710		-
3711		- context = kzalloc(sizeof(*context), GFP_KERNEL);
3712		- if (!context)
3713		- return -ENOMEM;
3714	4896
3715	4897	memset(qp_attr, 0, sizeof(*qp_attr));
3716	4898	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
..	..	@@ -3723,89 +4905,92 @@
3723	4905	goto done;
3724	4906	}
3725	4907
3726		- ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, context);
	4908	+ ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
3727	4909	if (ret) {
3728		- dev_err(dev, "query qpc error\n");
	4910	+ ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);
3729	4911	ret = -EINVAL;
3730	4912	goto out;
3731	4913	}
3732	4914
3733		- state = roce_get_field(context->byte_60_qpst_mapid,
	4915	+ state = roce_get_field(context.byte_60_qpst_tempid,
3734	4916	V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
3735	4917	tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
3736	4918	if (tmp_qp_state == -1) {
3737		- dev_err(dev, "Illegal ib_qp_state\n");
	4919	+ ibdev_err(ibdev, "Illegal ib_qp_state\n");
3738	4920	ret = -EINVAL;
3739	4921	goto out;
3740	4922	}
3741	4923	hr_qp->state = (u8)tmp_qp_state;
3742	4924	qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
3743		- qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->byte_24_mtu_tc,
	4925	+ qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context.byte_24_mtu_tc,
3744	4926	V2_QPC_BYTE_24_MTU_M,
3745	4927	V2_QPC_BYTE_24_MTU_S);
3746	4928	qp_attr->path_mig_state = IB_MIG_ARMED;
3747	4929	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
3748	4930	if (hr_qp->ibqp.qp_type == IB_QPT_UD)
3749		- qp_attr->qkey = V2_QKEY_VAL;
	4931	+ qp_attr->qkey = le32_to_cpu(context.qkey_xrcd);
3750	4932
3751		- qp_attr->rq_psn = roce_get_field(context->byte_108_rx_reqepsn,
	4933	+ qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn,
3752	4934	V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3753	4935	V2_QPC_BYTE_108_RX_REQ_EPSN_S);
3754		- qp_attr->sq_psn = (u32)roce_get_field(context->byte_172_sq_psn,
	4936	+ qp_attr->sq_psn = (u32)roce_get_field(context.byte_172_sq_psn,
3755	4937	V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3756	4938	V2_QPC_BYTE_172_SQ_CUR_PSN_S);
3757		- qp_attr->dest_qp_num = (u8)roce_get_field(context->byte_56_dqpn_err,
	4939	+ qp_attr->dest_qp_num = (u8)roce_get_field(context.byte_56_dqpn_err,
3758	4940	V2_QPC_BYTE_56_DQPN_M,
3759	4941	V2_QPC_BYTE_56_DQPN_S);
3760		- qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en,
3761		- V2_QPC_BYTE_76_RRE_S)) << 2) \|
3762		- ((roce_get_bit(context->byte_76_srqn_op_en,
3763		- V2_QPC_BYTE_76_RWE_S)) << 1) \|
3764		- ((roce_get_bit(context->byte_76_srqn_op_en,
3765		- V2_QPC_BYTE_76_ATE_S)) << 3);
	4942	+ qp_attr->qp_access_flags = ((roce_get_bit(context.byte_76_srqn_op_en,
	4943	+ V2_QPC_BYTE_76_RRE_S)) << V2_QP_RRE_S) \|
	4944	+ ((roce_get_bit(context.byte_76_srqn_op_en,
	4945	+ V2_QPC_BYTE_76_RWE_S)) << V2_QP_RWE_S) \|
	4946	+ ((roce_get_bit(context.byte_76_srqn_op_en,
	4947	+ V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
	4948	+
3766	4949	if (hr_qp->ibqp.qp_type == IB_QPT_RC \|\|
3767	4950	hr_qp->ibqp.qp_type == IB_QPT_UC) {
3768	4951	struct ib_global_route *grh =
3769	4952	rdma_ah_retrieve_grh(&qp_attr->ah_attr);
3770	4953
3771	4954	rdma_ah_set_sl(&qp_attr->ah_attr,
3772		- roce_get_field(context->byte_28_at_fl,
	4955	+ roce_get_field(context.byte_28_at_fl,
3773	4956	V2_QPC_BYTE_28_SL_M,
3774	4957	V2_QPC_BYTE_28_SL_S));
3775		- grh->flow_label = roce_get_field(context->byte_28_at_fl,
	4958	+ grh->flow_label = roce_get_field(context.byte_28_at_fl,
3776	4959	V2_QPC_BYTE_28_FL_M,
3777	4960	V2_QPC_BYTE_28_FL_S);
3778		- grh->sgid_index = roce_get_field(context->byte_20_smac_sgid_idx,
	4961	+ grh->sgid_index = roce_get_field(context.byte_20_smac_sgid_idx,
3779	4962	V2_QPC_BYTE_20_SGID_IDX_M,
3780	4963	V2_QPC_BYTE_20_SGID_IDX_S);
3781		- grh->hop_limit = roce_get_field(context->byte_24_mtu_tc,
	4964	+ grh->hop_limit = roce_get_field(context.byte_24_mtu_tc,
3782	4965	V2_QPC_BYTE_24_HOP_LIMIT_M,
3783	4966	V2_QPC_BYTE_24_HOP_LIMIT_S);
3784		- grh->traffic_class = roce_get_field(context->byte_24_mtu_tc,
	4967	+ grh->traffic_class = roce_get_field(context.byte_24_mtu_tc,
3785	4968	V2_QPC_BYTE_24_TC_M,
3786	4969	V2_QPC_BYTE_24_TC_S);
3787	4970
3788		- memcpy(grh->dgid.raw, context->dgid, sizeof(grh->dgid.raw));
	4971	+ memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
3789	4972	}
3790	4973
3791	4974	qp_attr->port_num = hr_qp->port + 1;
3792	4975	qp_attr->sq_draining = 0;
3793		- qp_attr->max_rd_atomic = 1 << roce_get_field(context->byte_208_irrl,
	4976	+ qp_attr->max_rd_atomic = 1 << roce_get_field(context.byte_208_irrl,
3794	4977	V2_QPC_BYTE_208_SR_MAX_M,
3795	4978	V2_QPC_BYTE_208_SR_MAX_S);
3796		- qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->byte_140_raq,
	4979	+ qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq,
3797	4980	V2_QPC_BYTE_140_RR_MAX_M,
3798	4981	V2_QPC_BYTE_140_RR_MAX_S);
3799		- qp_attr->min_rnr_timer = (u8)roce_get_field(context->byte_80_rnr_rx_cqn,
	4982	+ qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn,
3800	4983	V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3801	4984	V2_QPC_BYTE_80_MIN_RNR_TIME_S);
3802		- qp_attr->timeout = (u8)roce_get_field(context->byte_28_at_fl,
	4985	+ qp_attr->timeout = (u8)roce_get_field(context.byte_28_at_fl,
3803	4986	V2_QPC_BYTE_28_AT_M,
3804	4987	V2_QPC_BYTE_28_AT_S);
3805		- qp_attr->retry_cnt = roce_get_field(context->byte_212_lsn,
3806		- V2_QPC_BYTE_212_RETRY_CNT_M,
3807		- V2_QPC_BYTE_212_RETRY_CNT_S);
3808		- qp_attr->rnr_retry = context->rq_rnr_timer;
	4988	+ qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn,
	4989	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
	4990	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_S);
	4991	+ qp_attr->rnr_retry = roce_get_field(context.byte_244_rnr_rxack,
	4992	+ V2_QPC_BYTE_244_RNR_NUM_INIT_M,
	4993	+ V2_QPC_BYTE_244_RNR_NUM_INIT_S);
3809	4994
3810	4995	done:
3811	4996	qp_attr->cur_qp_state = qp_attr->qp_state;
..	..	@@ -3825,100 +5010,306 @@
3825	5010
3826	5011	out:
3827	5012	mutex_unlock(&hr_qp->mutex);
3828		- kfree(context);
3829	5013	return ret;
3830	5014	}
3831	5015
3832	5016	static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
3833	5017	struct hns_roce_qp *hr_qp,
3834		- int is_user)
	5018	+ struct ib_udata *udata)
3835	5019	{
	5020	+ struct ib_device *ibdev = &hr_dev->ib_dev;
3836	5021	struct hns_roce_cq send_cq, recv_cq;
3837		- struct device *dev = hr_dev->dev;
3838		- int ret;
	5022	+ unsigned long flags;
	5023	+ int ret = 0;
3839	5024
3840	5025	if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
3841	5026	/* Modify qp to reset before destroying qp */
3842	5027	ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
3843	5028	hr_qp->state, IB_QPS_RESET);
3844		- if (ret) {
3845		- dev_err(dev, "modify QP %06lx to ERR failed.\n",
3846		- hr_qp->qpn);
3847		- return ret;
3848		- }
	5029	+ if (ret)
	5030	+ ibdev_err(ibdev,
	5031	+ "failed to modify QP to RST, ret = %d.\n",
	5032	+ ret);
3849	5033	}
3850	5034
3851		- send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
3852		- recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
	5035	+ send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;
	5036	+ recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;
3853	5037
	5038	+ spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
3854	5039	hns_roce_lock_cqs(send_cq, recv_cq);
3855	5040
3856		- if (!is_user) {
3857		- __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
3858		- to_hr_srq(hr_qp->ibqp.srq) : NULL);
3859		- if (send_cq != recv_cq)
	5041	+ if (!udata) {
	5042	+ if (recv_cq)
	5043	+ __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn,
	5044	+ (hr_qp->ibqp.srq ?
	5045	+ to_hr_srq(hr_qp->ibqp.srq) :
	5046	+ NULL));
	5047	+
	5048	+ if (send_cq && send_cq != recv_cq)
3860	5049	__hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
	5050	+
3861	5051	}
3862	5052
3863	5053	hns_roce_qp_remove(hr_dev, hr_qp);
3864	5054
3865	5055	hns_roce_unlock_cqs(send_cq, recv_cq);
	5056	+ spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
3866	5057
3867		- hns_roce_qp_free(hr_dev, hr_qp);
3868		-
3869		- /* Not special_QP, free their QPN */
3870		- if ((hr_qp->ibqp.qp_type == IB_QPT_RC) \|\|
3871		- (hr_qp->ibqp.qp_type == IB_QPT_UC) \|\|
3872		- (hr_qp->ibqp.qp_type == IB_QPT_UD))
3873		- hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
3874		-
3875		- hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
3876		-
3877		- if (is_user) {
3878		- if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
3879		- hns_roce_db_unmap_user(
3880		- to_hr_ucontext(hr_qp->ibqp.uobject->context),
3881		- &hr_qp->sdb);
3882		-
3883		- if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
3884		- hns_roce_db_unmap_user(
3885		- to_hr_ucontext(hr_qp->ibqp.uobject->context),
3886		- &hr_qp->rdb);
3887		- ib_umem_release(hr_qp->umem);
3888		- } else {
3889		- kfree(hr_qp->sq.wrid);
3890		- kfree(hr_qp->rq.wrid);
3891		- hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
3892		- if (hr_qp->rq.wqe_cnt)
3893		- hns_roce_free_db(hr_dev, &hr_qp->rdb);
3894		- }
3895		-
3896		- if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
3897		- kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
3898		- kfree(hr_qp->rq_inl_buf.wqe_list);
3899		- }
3900		-
3901		- return 0;
	5058	+ return ret;
3902	5059	}
3903	5060
3904		-static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp)
	5061	+static int hns_roce_v2_destroy_qp(struct ib_qp ibqp, struct ib_udata udata)
3905	5062	{
3906	5063	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3907	5064	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3908	5065	int ret;
3909	5066
3910		- ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject);
3911		- if (ret) {
3912		- dev_err(hr_dev->dev, "Destroy qp failed(%d)\n", ret);
3913		- return ret;
3914		- }
	5067	+ ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
	5068	+ if (ret)
	5069	+ ibdev_err(&hr_dev->ib_dev,
	5070	+ "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",
	5071	+ hr_qp->qpn, ret);
3915	5072
3916		- if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
3917		- kfree(hr_to_hr_sqp(hr_qp));
3918		- else
3919		- kfree(hr_qp);
	5073	+ hns_roce_qp_destroy(hr_dev, hr_qp, udata);
3920	5074
3921	5075	return 0;
	5076	+}
	5077	+
	5078	+static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
	5079	+ struct hns_roce_qp *hr_qp)
	5080	+{
	5081	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	5082	+ struct hns_roce_sccc_clr_done *resp;
	5083	+ struct hns_roce_sccc_clr *clr;
	5084	+ struct hns_roce_cmq_desc desc;
	5085	+ int ret, i;
	5086	+
	5087	+ mutex_lock(&hr_dev->qp_table.scc_mutex);
	5088	+
	5089	+ /* set scc ctx clear done flag */
	5090	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
	5091	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	5092	+ if (ret) {
	5093	+ ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);
	5094	+ goto out;
	5095	+ }
	5096	+
	5097	+ /* clear scc context */
	5098	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
	5099	+ clr = (struct hns_roce_sccc_clr *)desc.data;
	5100	+ clr->qpn = cpu_to_le32(hr_qp->qpn);
	5101	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	5102	+ if (ret) {
	5103	+ ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);
	5104	+ goto out;
	5105	+ }
	5106	+
	5107	+ /* query scc context clear is done or not */
	5108	+ resp = (struct hns_roce_sccc_clr_done *)desc.data;
	5109	+ for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
	5110	+ hns_roce_cmq_setup_basic_desc(&desc,
	5111	+ HNS_ROCE_OPC_QUERY_SCCC, true);
	5112	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	5113	+ if (ret) {
	5114	+ ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n",
	5115	+ ret);
	5116	+ goto out;
	5117	+ }
	5118	+
	5119	+ if (resp->clr_done)
	5120	+ goto out;
	5121	+
	5122	+ msleep(20);
	5123	+ }
	5124	+
	5125	+ ibdev_err(ibdev, "Query SCC clr done flag overtime.\n");
	5126	+ ret = -ETIMEDOUT;
	5127	+
	5128	+out:
	5129	+ mutex_unlock(&hr_dev->qp_table.scc_mutex);
	5130	+ return ret;
	5131	+}
	5132	+
	5133	+static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
	5134	+ struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
	5135	+ u32 cqn, void mb_buf, u64 mtts_wqe,
	5136	+ u64 *mtts_idx, dma_addr_t dma_handle_wqe,
	5137	+ dma_addr_t dma_handle_idx)
	5138	+{
	5139	+ struct hns_roce_srq_context *srq_context;
	5140	+
	5141	+ srq_context = mb_buf;
	5142	+ memset(srq_context, 0, sizeof(*srq_context));
	5143	+
	5144	+ roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
	5145	+ SRQC_BYTE_4_SRQ_ST_S, 1);
	5146	+
	5147	+ roce_set_field(srq_context->byte_4_srqn_srqst,
	5148	+ SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
	5149	+ SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
	5150	+ to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num,
	5151	+ srq->wqe_cnt));
	5152	+ roce_set_field(srq_context->byte_4_srqn_srqst,
	5153	+ SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
	5154	+ ilog2(srq->wqe_cnt));
	5155	+
	5156	+ roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
	5157	+ SRQC_BYTE_4_SRQN_S, srq->srqn);
	5158	+
	5159	+ roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5160	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
	5161	+
	5162	+ roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
	5163	+ SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
	5164	+
	5165	+ srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
	5166	+
	5167	+ roce_set_field(srq_context->byte_24_wqe_bt_ba,
	5168	+ SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
	5169	+ SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
	5170	+ dma_handle_wqe >> 35);
	5171	+
	5172	+ roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
	5173	+ SRQC_BYTE_28_PD_S, pdn);
	5174	+ roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
	5175	+ SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
	5176	+ fls(srq->max_gs - 1));
	5177	+
	5178	+ srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3);
	5179	+ roce_set_field(srq_context->rsv_idx_bt_ba,
	5180	+ SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
	5181	+ SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
	5182	+ dma_handle_idx >> 35);
	5183	+
	5184	+ srq_context->idx_cur_blk_addr =
	5185	+ cpu_to_le32(to_hr_hw_page_addr(mtts_idx[0]));
	5186	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5187	+ SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
	5188	+ SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
	5189	+ upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));
	5190	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5191	+ SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
	5192	+ SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
	5193	+ to_hr_hem_hopnum(hr_dev->caps.idx_hop_num,
	5194	+ srq->wqe_cnt));
	5195	+
	5196	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5197	+ SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
	5198	+ SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
	5199	+ to_hr_hw_page_shift(srq->idx_que.mtr.hem_cfg.ba_pg_shift));
	5200	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5201	+ SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
	5202	+ SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
	5203	+ to_hr_hw_page_shift(srq->idx_que.mtr.hem_cfg.buf_pg_shift));
	5204	+
	5205	+ srq_context->idx_nxt_blk_addr =
	5206	+ cpu_to_le32(to_hr_hw_page_addr(mtts_idx[1]));
	5207	+ roce_set_field(srq_context->rsv_idxnxtblkaddr,
	5208	+ SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
	5209	+ SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
	5210	+ upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));
	5211	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5212	+ SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
	5213	+ cqn);
	5214	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5215	+ SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
	5216	+ SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
	5217	+ to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift));
	5218	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5219	+ SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
	5220	+ SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
	5221	+ to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift));
	5222	+
	5223	+ roce_set_bit(srq_context->db_record_addr_record_en,
	5224	+ SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
	5225	+}
	5226	+
	5227	+static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
	5228	+ struct ib_srq_attr *srq_attr,
	5229	+ enum ib_srq_attr_mask srq_attr_mask,
	5230	+ struct ib_udata *udata)
	5231	+{
	5232	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	5233	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	5234	+ struct hns_roce_srq_context *srq_context;
	5235	+ struct hns_roce_srq_context *srqc_mask;
	5236	+ struct hns_roce_cmd_mailbox *mailbox;
	5237	+ int ret;
	5238	+
	5239	+ /* Resizing SRQs is not supported yet */
	5240	+ if (srq_attr_mask & IB_SRQ_MAX_WR)
	5241	+ return -EINVAL;
	5242	+
	5243	+ if (srq_attr_mask & IB_SRQ_LIMIT) {
	5244	+ if (srq_attr->srq_limit >= srq->wqe_cnt)
	5245	+ return -EINVAL;
	5246	+
	5247	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	5248	+ if (IS_ERR(mailbox))
	5249	+ return PTR_ERR(mailbox);
	5250	+
	5251	+ srq_context = mailbox->buf;
	5252	+ srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
	5253	+
	5254	+ memset(srqc_mask, 0xff, sizeof(*srqc_mask));
	5255	+
	5256	+ roce_set_field(srq_context->byte_8_limit_wl,
	5257	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5258	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
	5259	+ roce_set_field(srqc_mask->byte_8_limit_wl,
	5260	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5261	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
	5262	+
	5263	+ ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
	5264	+ HNS_ROCE_CMD_MODIFY_SRQC,
	5265	+ HNS_ROCE_CMD_TIMEOUT_MSECS);
	5266	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	5267	+ if (ret) {
	5268	+ ibdev_err(&hr_dev->ib_dev,
	5269	+ "failed to handle cmd of modifying SRQ, ret = %d.\n",
	5270	+ ret);
	5271	+ return ret;
	5272	+ }
	5273	+ }
	5274	+
	5275	+ return 0;
	5276	+}
	5277	+
	5278	+static int hns_roce_v2_query_srq(struct ib_srq ibsrq, struct ib_srq_attr attr)
	5279	+{
	5280	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	5281	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	5282	+ struct hns_roce_srq_context *srq_context;
	5283	+ struct hns_roce_cmd_mailbox *mailbox;
	5284	+ int limit_wl;
	5285	+ int ret;
	5286	+
	5287	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	5288	+ if (IS_ERR(mailbox))
	5289	+ return PTR_ERR(mailbox);
	5290	+
	5291	+ srq_context = mailbox->buf;
	5292	+ ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
	5293	+ HNS_ROCE_CMD_QUERY_SRQC,
	5294	+ HNS_ROCE_CMD_TIMEOUT_MSECS);
	5295	+ if (ret) {
	5296	+ ibdev_err(&hr_dev->ib_dev,
	5297	+ "failed to process cmd of querying SRQ, ret = %d.\n",
	5298	+ ret);
	5299	+ goto out;
	5300	+ }
	5301	+
	5302	+ limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
	5303	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5304	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S);
	5305	+
	5306	+ attr->srq_limit = limit_wl;
	5307	+ attr->max_wr = srq->wqe_cnt - 1;
	5308	+ attr->max_sge = srq->max_gs;
	5309	+
	5310	+out:
	5311	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	5312	+ return ret;
3922	5313	}
3923	5314
3924	5315	static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
..	..	@@ -3957,55 +5348,65 @@
3957	5348	HNS_ROCE_CMD_TIMEOUT_MSECS);
3958	5349	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3959	5350	if (ret)
3960		- dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n");
	5351	+ ibdev_err(&hr_dev->ib_dev,
	5352	+ "failed to process cmd when modifying CQ, ret = %d.\n",
	5353	+ ret);
3961	5354
3962	5355	return ret;
3963		-}
3964		-
3965		-static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)
3966		-{
3967		- struct hns_roce_qp *hr_qp;
3968		- struct ib_qp_attr attr;
3969		- int attr_mask;
3970		- int ret;
3971		-
3972		- hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
3973		- if (!hr_qp) {
3974		- dev_warn(hr_dev->dev, "no hr_qp can be found!\n");
3975		- return;
3976		- }
3977		-
3978		- if (hr_qp->ibqp.uobject) {
3979		- if (hr_qp->sdb_en == 1) {
3980		- hr_qp->sq.head = (int )(hr_qp->sdb.virt_addr);
3981		- if (hr_qp->rdb_en == 1)
3982		- hr_qp->rq.head = (int )(hr_qp->rdb.virt_addr);
3983		- } else {
3984		- dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n");
3985		- return;
3986		- }
3987		- }
3988		-
3989		- attr_mask = IB_QP_STATE;
3990		- attr.qp_state = IB_QPS_ERR;
3991		- ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask,
3992		- hr_qp->state, IB_QPS_ERR);
3993		- if (ret)
3994		- dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n",
3995		- qpn);
3996	5356	}
3997	5357
3998	5358	static void hns_roce_irq_work_handle(struct work_struct *work)
3999	5359	{
4000	5360	struct hns_roce_work *irq_work =
4001	5361	container_of(work, struct hns_roce_work, work);
	5362	+ struct ib_device *ibdev = &irq_work->hr_dev->ib_dev;
4002	5363	u32 qpn = irq_work->qpn;
	5364	+ u32 cqn = irq_work->cqn;
4003	5365
4004	5366	switch (irq_work->event_type) {
	5367	+ case HNS_ROCE_EVENT_TYPE_PATH_MIG:
	5368	+ ibdev_info(ibdev, "Path migrated succeeded.\n");
	5369	+ break;
	5370	+ case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
	5371	+ ibdev_warn(ibdev, "Path migration failed.\n");
	5372	+ break;
	5373	+ case HNS_ROCE_EVENT_TYPE_COMM_EST:
	5374	+ break;
	5375	+ case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
	5376	+ ibdev_warn(ibdev, "Send queue drained.\n");
	5377	+ break;
4005	5378	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
	5379	+ ibdev_err(ibdev, "Local work queue 0x%x catast error, sub_event type is: %d\n",
	5380	+ qpn, irq_work->sub_type);
	5381	+ break;
4006	5382	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
	5383	+ ibdev_err(ibdev, "Invalid request local work queue 0x%x error.\n",
	5384	+ qpn);
	5385	+ break;
4007	5386	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4008		- hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
	5387	+ ibdev_err(ibdev, "Local access violation work queue 0x%x error, sub_event type is: %d\n",
	5388	+ qpn, irq_work->sub_type);
	5389	+ break;
	5390	+ case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
	5391	+ ibdev_warn(ibdev, "SRQ limit reach.\n");
	5392	+ break;
	5393	+ case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
	5394	+ ibdev_warn(ibdev, "SRQ last wqe reach.\n");
	5395	+ break;
	5396	+ case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
	5397	+ ibdev_err(ibdev, "SRQ catas error.\n");
	5398	+ break;
	5399	+ case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
	5400	+ ibdev_err(ibdev, "CQ 0x%x access err.\n", cqn);
	5401	+ break;
	5402	+ case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
	5403	+ ibdev_warn(ibdev, "CQ 0x%x overflow\n", cqn);
	5404	+ break;
	5405	+ case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
	5406	+ ibdev_warn(ibdev, "DB overflow.\n");
	5407	+ break;
	5408	+ case HNS_ROCE_EVENT_TYPE_FLR:
	5409	+ ibdev_warn(ibdev, "Function level reset.\n");
4009	5410	break;
4010	5411	default:
4011	5412	break;
..	..	@@ -4015,7 +5416,8 @@
4015	5416	}
4016	5417
4017	5418	static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
4018		- struct hns_roce_eq *eq, u32 qpn)
	5419	+ struct hns_roce_eq *eq,
	5420	+ u32 qpn, u32 cqn)
4019	5421	{
4020	5422	struct hns_roce_work *irq_work;
4021	5423
..	..	@@ -4026,6 +5428,7 @@
4026	5428	INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
4027	5429	irq_work->hr_dev = hr_dev;
4028	5430	irq_work->qpn = qpn;
	5431	+ irq_work->cqn = cqn;
4029	5432	irq_work->event_type = eq->event_type;
4030	5433	irq_work->sub_type = eq->sub_type;
4031	5434	queue_work(hr_dev->irq_workq, &(irq_work->work));
..	..	@@ -4033,10 +5436,8 @@
4033	5436
4034	5437	static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
4035	5438	{
4036		- u32 doorbell[2];
4037		-
4038		- doorbell[0] = 0;
4039		- doorbell[1] = 0;
	5439	+ struct hns_roce_dev *hr_dev = eq->hr_dev;
	5440	+ __le32 doorbell[2] = {};
4040	5441
4041	5442	if (eq->type_flag == HNS_ROCE_AEQ) {
4042	5443	roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
..	..	@@ -4059,164 +5460,16 @@
4059	5460	HNS_ROCE_V2_EQ_DB_PARA_S,
4060	5461	(eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
4061	5462
4062		- hns_roce_write64_k(doorbell, eq->doorbell);
4063		-}
4064		-
4065		-static void hns_roce_v2_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
4066		- struct hns_roce_aeqe *aeqe,
4067		- u32 qpn)
4068		-{
4069		- struct device *dev = hr_dev->dev;
4070		- int sub_type;
4071		-
4072		- dev_warn(dev, "Local work queue catastrophic error.\n");
4073		- sub_type = roce_get_field(aeqe->asyn, HNS_ROCE_V2_AEQE_SUB_TYPE_M,
4074		- HNS_ROCE_V2_AEQE_SUB_TYPE_S);
4075		- switch (sub_type) {
4076		- case HNS_ROCE_LWQCE_QPC_ERROR:
4077		- dev_warn(dev, "QP %d, QPC error.\n", qpn);
4078		- break;
4079		- case HNS_ROCE_LWQCE_MTU_ERROR:
4080		- dev_warn(dev, "QP %d, MTU error.\n", qpn);
4081		- break;
4082		- case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
4083		- dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
4084		- break;
4085		- case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
4086		- dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
4087		- break;
4088		- case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
4089		- dev_warn(dev, "QP %d, WQE shift error.\n", qpn);
4090		- break;
4091		- default:
4092		- dev_err(dev, "Unhandled sub_event type %d.\n", sub_type);
4093		- break;
4094		- }
4095		-}
4096		-
4097		-static void hns_roce_v2_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
4098		- struct hns_roce_aeqe *aeqe, u32 qpn)
4099		-{
4100		- struct device *dev = hr_dev->dev;
4101		- int sub_type;
4102		-
4103		- dev_warn(dev, "Local access violation work queue error.\n");
4104		- sub_type = roce_get_field(aeqe->asyn, HNS_ROCE_V2_AEQE_SUB_TYPE_M,
4105		- HNS_ROCE_V2_AEQE_SUB_TYPE_S);
4106		- switch (sub_type) {
4107		- case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
4108		- dev_warn(dev, "QP %d, R_key violation.\n", qpn);
4109		- break;
4110		- case HNS_ROCE_LAVWQE_LENGTH_ERROR:
4111		- dev_warn(dev, "QP %d, length error.\n", qpn);
4112		- break;
4113		- case HNS_ROCE_LAVWQE_VA_ERROR:
4114		- dev_warn(dev, "QP %d, VA error.\n", qpn);
4115		- break;
4116		- case HNS_ROCE_LAVWQE_PD_ERROR:
4117		- dev_err(dev, "QP %d, PD error.\n", qpn);
4118		- break;
4119		- case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
4120		- dev_warn(dev, "QP %d, rw acc error.\n", qpn);
4121		- break;
4122		- case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
4123		- dev_warn(dev, "QP %d, key state error.\n", qpn);
4124		- break;
4125		- case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
4126		- dev_warn(dev, "QP %d, MR operation error.\n", qpn);
4127		- break;
4128		- default:
4129		- dev_err(dev, "Unhandled sub_event type %d.\n", sub_type);
4130		- break;
4131		- }
4132		-}
4133		-
4134		-static void hns_roce_v2_qp_err_handle(struct hns_roce_dev *hr_dev,
4135		- struct hns_roce_aeqe *aeqe,
4136		- int event_type, u32 qpn)
4137		-{
4138		- struct device *dev = hr_dev->dev;
4139		-
4140		- switch (event_type) {
4141		- case HNS_ROCE_EVENT_TYPE_COMM_EST:
4142		- dev_warn(dev, "Communication established.\n");
4143		- break;
4144		- case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4145		- dev_warn(dev, "Send queue drained.\n");
4146		- break;
4147		- case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
4148		- hns_roce_v2_wq_catas_err_handle(hr_dev, aeqe, qpn);
4149		- break;
4150		- case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4151		- dev_warn(dev, "Invalid request local work queue error.\n");
4152		- break;
4153		- case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4154		- hns_roce_v2_local_wq_access_err_handle(hr_dev, aeqe, qpn);
4155		- break;
4156		- default:
4157		- break;
4158		- }
4159		-
4160		- hns_roce_qp_event(hr_dev, qpn, event_type);
4161		-}
4162		-
4163		-static void hns_roce_v2_cq_err_handle(struct hns_roce_dev *hr_dev,
4164		- struct hns_roce_aeqe *aeqe,
4165		- int event_type, u32 cqn)
4166		-{
4167		- struct device *dev = hr_dev->dev;
4168		-
4169		- switch (event_type) {
4170		- case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4171		- dev_warn(dev, "CQ 0x%x access err.\n", cqn);
4172		- break;
4173		- case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4174		- dev_warn(dev, "CQ 0x%x overflow\n", cqn);
4175		- break;
4176		- default:
4177		- break;
4178		- }
4179		-
4180		- hns_roce_cq_event(hr_dev, cqn, event_type);
4181		-}
4182		-
4183		-static struct hns_roce_aeqe get_aeqe_v2(struct hns_roce_eq eq, u32 entry)
4184		-{
4185		- u32 buf_chk_sz;
4186		- unsigned long off;
4187		-
4188		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4189		- off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4190		-
4191		- return (struct hns_roce_aeqe )((char )(eq->buf_list->buf) +
4192		- off % buf_chk_sz);
4193		-}
4194		-
4195		-static struct hns_roce_aeqe mhop_get_aeqe(struct hns_roce_eq eq, u32 entry)
4196		-{
4197		- u32 buf_chk_sz;
4198		- unsigned long off;
4199		-
4200		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4201		-
4202		- off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4203		-
4204		- if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4205		- return (struct hns_roce_aeqe )((u8 )(eq->bt_l0) +
4206		- off % buf_chk_sz);
4207		- else
4208		- return (struct hns_roce_aeqe )((u8 )
4209		- (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
	5463	+ hns_roce_write64(hr_dev, doorbell, eq->doorbell);
4210	5464	}
4211	5465
4212	5466	static struct hns_roce_aeqe next_aeqe_sw_v2(struct hns_roce_eq eq)
4213	5467	{
4214	5468	struct hns_roce_aeqe *aeqe;
4215	5469
4216		- if (!eq->hop_num)
4217		- aeqe = get_aeqe_v2(eq, eq->cons_index);
4218		- else
4219		- aeqe = mhop_get_aeqe(eq, eq->cons_index);
	5470	+ aeqe = hns_roce_buf_offset(eq->mtr.kmem,
	5471	+ (eq->cons_index & (eq->entries - 1)) *
	5472	+ eq->eqe_size);
4220	5473
4221	5474	return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
4222	5475	!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
..	..	@@ -4226,15 +5479,15 @@
4226	5479	struct hns_roce_eq *eq)
4227	5480	{
4228	5481	struct device *dev = hr_dev->dev;
4229		- struct hns_roce_aeqe *aeqe;
	5482	+ struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
4230	5483	int aeqe_found = 0;
4231	5484	int event_type;
4232	5485	int sub_type;
	5486	+ u32 srqn;
4233	5487	u32 qpn;
4234	5488	u32 cqn;
4235	5489
4236		- while ((aeqe = next_aeqe_sw_v2(eq))) {
4237		-
	5490	+ while (aeqe) {
4238	5491	/* Make sure we read AEQ entry after we have checked the
4239	5492	* ownership bit
4240	5493	*/
..	..	@@ -4252,34 +5505,30 @@
4252	5505	cqn = roce_get_field(aeqe->event.cq_event.cq,
4253	5506	HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4254	5507	HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
	5508	+ srqn = roce_get_field(aeqe->event.srq_event.srq,
	5509	+ HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
	5510	+ HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
4255	5511
4256	5512	switch (event_type) {
4257	5513	case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4258		- dev_warn(dev, "Path migrated succeeded.\n");
4259		- break;
4260	5514	case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4261		- dev_warn(dev, "Path migration failed.\n");
4262		- break;
4263	5515	case HNS_ROCE_EVENT_TYPE_COMM_EST:
4264	5516	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4265	5517	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
	5518	+ case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4266	5519	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4267	5520	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4268		- hns_roce_v2_qp_err_handle(hr_dev, aeqe, event_type,
4269		- qpn);
	5521	+ hns_roce_qp_event(hr_dev, qpn, event_type);
4270	5522	break;
4271	5523	case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4272		- case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4273	5524	case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4274		- dev_warn(dev, "SRQ not support.\n");
	5525	+ hns_roce_srq_event(hr_dev, srqn, event_type);
4275	5526	break;
4276	5527	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4277	5528	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4278		- hns_roce_v2_cq_err_handle(hr_dev, aeqe, event_type,
4279		- cqn);
	5529	+ hns_roce_cq_event(hr_dev, cqn, event_type);
4280	5530	break;
4281	5531	case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4282		- dev_warn(dev, "DB overflow.\n");
4283	5532	break;
4284	5533	case HNS_ROCE_EVENT_TYPE_MB:
4285	5534	hns_roce_cmd_event(hr_dev,
..	..	@@ -4288,70 +5537,39 @@
4288	5537	le64_to_cpu(aeqe->event.cmd.out_param));
4289	5538	break;
4290	5539	case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
4291		- dev_warn(dev, "CEQ overflow.\n");
4292	5540	break;
4293	5541	case HNS_ROCE_EVENT_TYPE_FLR:
4294		- dev_warn(dev, "Function level reset.\n");
4295	5542	break;
4296	5543	default:
4297	5544	dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
4298	5545	event_type, eq->eqn, eq->cons_index);
4299	5546	break;
4300		- };
	5547	+ }
4301	5548
4302	5549	eq->event_type = event_type;
4303	5550	eq->sub_type = sub_type;
4304	5551	++eq->cons_index;
4305	5552	aeqe_found = 1;
4306	5553
4307		- if (eq->cons_index > (2 * eq->entries - 1)) {
4308		- dev_warn(dev, "cons_index overflow, set back to 0.\n");
	5554	+ if (eq->cons_index > (2 * eq->entries - 1))
4309	5555	eq->cons_index = 0;
4310		- }
4311		- hns_roce_v2_init_irq_work(hr_dev, eq, qpn);
	5556	+
	5557	+ hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
	5558	+
	5559	+ aeqe = next_aeqe_sw_v2(eq);
4312	5560	}
4313	5561
4314	5562	set_eq_cons_index_v2(eq);
4315	5563	return aeqe_found;
4316	5564	}
4317	5565
4318		-static struct hns_roce_ceqe get_ceqe_v2(struct hns_roce_eq eq, u32 entry)
4319		-{
4320		- u32 buf_chk_sz;
4321		- unsigned long off;
4322		-
4323		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4324		- off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
4325		-
4326		- return (struct hns_roce_ceqe )((char )(eq->buf_list->buf) +
4327		- off % buf_chk_sz);
4328		-}
4329		-
4330		-static struct hns_roce_ceqe mhop_get_ceqe(struct hns_roce_eq eq, u32 entry)
4331		-{
4332		- u32 buf_chk_sz;
4333		- unsigned long off;
4334		-
4335		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4336		-
4337		- off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
4338		-
4339		- if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4340		- return (struct hns_roce_ceqe )((u8 )(eq->bt_l0) +
4341		- off % buf_chk_sz);
4342		- else
4343		- return (struct hns_roce_ceqe )((u8 )(eq->buf[off /
4344		- buf_chk_sz]) + off % buf_chk_sz);
4345		-}
4346		-
4347	5566	static struct hns_roce_ceqe next_ceqe_sw_v2(struct hns_roce_eq eq)
4348	5567	{
4349	5568	struct hns_roce_ceqe *ceqe;
4350	5569
4351		- if (!eq->hop_num)
4352		- ceqe = get_ceqe_v2(eq, eq->cons_index);
4353		- else
4354		- ceqe = mhop_get_ceqe(eq, eq->cons_index);
	5570	+ ceqe = hns_roce_buf_offset(eq->mtr.kmem,
	5571	+ (eq->cons_index & (eq->entries - 1)) *
	5572	+ eq->eqe_size);
4355	5573
4356	5574	return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
4357	5575	(!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
..	..	@@ -4360,20 +5578,17 @@
4360	5578	static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
4361	5579	struct hns_roce_eq *eq)
4362	5580	{
4363		- struct device *dev = hr_dev->dev;
4364		- struct hns_roce_ceqe *ceqe;
	5581	+ struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
4365	5582	int ceqe_found = 0;
4366	5583	u32 cqn;
4367	5584
4368		- while ((ceqe = next_ceqe_sw_v2(eq))) {
4369		-
	5585	+ while (ceqe) {
4370	5586	/* Make sure we read CEQ entry after we have checked the
4371	5587	* ownership bit
4372	5588	*/
4373	5589	dma_rmb();
4374	5590
4375		- cqn = roce_get_field(ceqe->comp,
4376		- HNS_ROCE_V2_CEQE_COMP_CQN_M,
	5591	+ cqn = roce_get_field(ceqe->comp, HNS_ROCE_V2_CEQE_COMP_CQN_M,
4377	5592	HNS_ROCE_V2_CEQE_COMP_CQN_S);
4378	5593
4379	5594	hns_roce_cq_completion(hr_dev, cqn);
..	..	@@ -4381,10 +5596,10 @@
4381	5596	++eq->cons_index;
4382	5597	ceqe_found = 1;
4383	5598
4384		- if (eq->cons_index > (2 * eq->entries - 1)) {
4385		- dev_warn(dev, "cons_index overflow, set back to 0.\n");
	5599	+ if (eq->cons_index > (EQ_DEPTH_COEFF * eq->entries - 1))
4386	5600	eq->cons_index = 0;
4387		- }
	5601	+
	5602	+ ceqe = next_ceqe_sw_v2(eq);
4388	5603	}
4389	5604
4390	5605	set_eq_cons_index_v2(eq);
..	..	@@ -4396,7 +5611,7 @@
4396	5611	{
4397	5612	struct hns_roce_eq *eq = eq_ptr;
4398	5613	struct hns_roce_dev *hr_dev = eq->hr_dev;
4399		- int int_work = 0;
	5614	+ int int_work;
4400	5615
4401	5616	if (eq->type_flag == HNS_ROCE_CEQ)
4402	5617	/* Completion event interrupt */
..	..	@@ -4420,33 +5635,44 @@
4420	5635	int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
4421	5636	int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
4422	5637
4423		- if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
	5638	+ if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
	5639	+ struct pci_dev *pdev = hr_dev->pci_dev;
	5640	+ struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
	5641	+ const struct hnae3_ae_ops *ops = ae_dev->ops;
	5642	+
4424	5643	dev_err(dev, "AEQ overflow!\n");
4425	5644
4426		- roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1);
4427		- roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
	5645	+ roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG,
	5646	+ 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S);
4428	5647
4429		- roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
	5648	+ /* Set reset level for reset_event() */
	5649	+ if (ops->set_default_reset_request)
	5650	+ ops->set_default_reset_request(ae_dev,
	5651	+ HNAE3_FUNC_RESET);
	5652	+ if (ops->reset_event)
	5653	+ ops->reset_event(pdev, NULL);
	5654	+
	5655	+ int_en \|= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
4430	5656	roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4431	5657
4432	5658	int_work = 1;
4433		- } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
	5659	+ } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
4434	5660	dev_err(dev, "BUS ERR!\n");
4435	5661
4436		- roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1);
	5662	+ int_st \|= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S;
4437	5663	roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
4438	5664
4439		- roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
	5665	+ int_en \|= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
4440	5666	roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4441	5667
4442	5668	int_work = 1;
4443		- } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
	5669	+ } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
4444	5670	dev_err(dev, "OTHER ERR!\n");
4445	5671
4446		- roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1);
	5672	+ int_st \|= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S;
4447	5673	roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
4448	5674
4449		- roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
	5675	+ int_en \|= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
4450	5676	roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4451	5677
4452	5678	int_work = 1;
..	..	@@ -4501,502 +5727,187 @@
4501	5727	dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
4502	5728	}
4503	5729
4504		-static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
4505		- struct hns_roce_eq *eq)
	5730	+static void free_eq_buf(struct hns_roce_dev hr_dev, struct hns_roce_eq eq)
4506	5731	{
4507		- struct device *dev = hr_dev->dev;
4508		- u64 idx;
4509		- u64 size;
4510		- u32 buf_chk_sz;
4511		- u32 bt_chk_sz;
4512		- u32 mhop_num;
4513		- int eqe_alloc;
4514		- int i = 0;
4515		- int j = 0;
4516		-
4517		- mhop_num = hr_dev->caps.eqe_hop_num;
4518		- buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
4519		- bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
4520		-
4521		- /* hop_num = 0 */
4522		- if (mhop_num == HNS_ROCE_HOP_NUM_0) {
4523		- dma_free_coherent(dev, (unsigned int)(eq->entries *
4524		- eq->eqe_size), eq->bt_l0, eq->l0_dma);
4525		- return;
4526		- }
4527		-
4528		- /* hop_num = 1 or hop = 2 */
4529		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
4530		- if (mhop_num == 1) {
4531		- for (i = 0; i < eq->l0_last_num; i++) {
4532		- if (i == eq->l0_last_num - 1) {
4533		- eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
4534		- size = (eq->entries - eqe_alloc) * eq->eqe_size;
4535		- dma_free_coherent(dev, size, eq->buf[i],
4536		- eq->buf_dma[i]);
4537		- break;
4538		- }
4539		- dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
4540		- eq->buf_dma[i]);
4541		- }
4542		- } else if (mhop_num == 2) {
4543		- for (i = 0; i < eq->l0_last_num; i++) {
4544		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
4545		- eq->l1_dma[i]);
4546		-
4547		- for (j = 0; j < bt_chk_sz / 8; j++) {
4548		- idx = i * (bt_chk_sz / 8) + j;
4549		- if ((i == eq->l0_last_num - 1)
4550		- && j == eq->l1_last_num - 1) {
4551		- eqe_alloc = (buf_chk_sz / eq->eqe_size)
4552		- * idx;
4553		- size = (eq->entries - eqe_alloc)
4554		- * eq->eqe_size;
4555		- dma_free_coherent(dev, size,
4556		- eq->buf[idx],
4557		- eq->buf_dma[idx]);
4558		- break;
4559		- }
4560		- dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
4561		- eq->buf_dma[idx]);
4562		- }
4563		- }
4564		- }
4565		- kfree(eq->buf_dma);
4566		- kfree(eq->buf);
4567		- kfree(eq->l1_dma);
4568		- kfree(eq->bt_l1);
4569		- eq->buf_dma = NULL;
4570		- eq->buf = NULL;
4571		- eq->l1_dma = NULL;
4572		- eq->bt_l1 = NULL;
	5732	+ hns_roce_mtr_destroy(hr_dev, &eq->mtr);
4573	5733	}
4574	5734
4575		-static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
4576		- struct hns_roce_eq *eq)
	5735	+static int config_eqc(struct hns_roce_dev hr_dev, struct hns_roce_eq eq,
	5736	+ void *mb_buf)
4577	5737	{
4578		- u32 buf_chk_sz;
4579		-
4580		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4581		-
4582		- if (hr_dev->caps.eqe_hop_num) {
4583		- hns_roce_mhop_free_eq(hr_dev, eq);
4584		- return;
4585		- }
4586		-
4587		- dma_free_coherent(hr_dev->dev, buf_chk_sz, eq->buf_list->buf,
4588		- eq->buf_list->map);
4589		- kfree(eq->buf_list);
4590		-}
4591		-
4592		-static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
4593		- struct hns_roce_eq *eq,
4594		- void *mb_buf)
4595		-{
	5738	+ u64 eqe_ba[MTT_MIN_COUNT] = { 0 };
4596	5739	struct hns_roce_eq_context *eqc;
	5740	+ u64 bt_ba = 0;
	5741	+ int count;
4597	5742
4598	5743	eqc = mb_buf;
4599	5744	memset(eqc, 0, sizeof(struct hns_roce_eq_context));
4600	5745
4601	5746	/* init eqc */
4602	5747	eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
4603		- eq->hop_num = hr_dev->caps.eqe_hop_num;
4604	5748	eq->cons_index = 0;
4605	5749	eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
4606	5750	eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
4607	5751	eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
4608		- eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
4609		- eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
4610	5752	eq->shift = ilog2((unsigned int)eq->entries);
4611	5753
4612		- if (!eq->hop_num)
4613		- eq->eqe_ba = eq->buf_list->map;
4614		- else
4615		- eq->eqe_ba = eq->l0_dma;
	5754	+ /* if not multi-hop, eqe buffer only use one trunk */
	5755	+ count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT,
	5756	+ &bt_ba);
	5757	+ if (count < 1) {
	5758	+ dev_err(hr_dev->dev, "failed to find EQE mtr\n");
	5759	+ return -ENOBUFS;
	5760	+ }
4616	5761
4617	5762	/* set eqc state */
4618		- roce_set_field(eqc->byte_4,
4619		- HNS_ROCE_EQC_EQ_ST_M,
4620		- HNS_ROCE_EQC_EQ_ST_S,
	5763	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQ_ST_M, HNS_ROCE_EQC_EQ_ST_S,
4621	5764	HNS_ROCE_V2_EQ_STATE_VALID);
4622	5765
4623	5766	/* set eqe hop num */
4624		- roce_set_field(eqc->byte_4,
4625		- HNS_ROCE_EQC_HOP_NUM_M,
	5767	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_HOP_NUM_M,
4626	5768	HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
4627	5769
4628	5770	/* set eqc over_ignore */
4629		- roce_set_field(eqc->byte_4,
4630		- HNS_ROCE_EQC_OVER_IGNORE_M,
	5771	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_OVER_IGNORE_M,
4631	5772	HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
4632	5773
4633	5774	/* set eqc coalesce */
4634		- roce_set_field(eqc->byte_4,
4635		- HNS_ROCE_EQC_COALESCE_M,
	5775	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_COALESCE_M,
4636	5776	HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
4637	5777
4638	5778	/* set eqc arm_state */
4639		- roce_set_field(eqc->byte_4,
4640		- HNS_ROCE_EQC_ARM_ST_M,
	5779	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_ARM_ST_M,
4641	5780	HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
4642	5781
4643	5782	/* set eqn */
4644		- roce_set_field(eqc->byte_4,
4645		- HNS_ROCE_EQC_EQN_M,
4646		- HNS_ROCE_EQC_EQN_S, eq->eqn);
	5783	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQN_M, HNS_ROCE_EQC_EQN_S,
	5784	+ eq->eqn);
4647	5785
4648	5786	/* set eqe_cnt */
4649		- roce_set_field(eqc->byte_4,
4650		- HNS_ROCE_EQC_EQE_CNT_M,
4651		- HNS_ROCE_EQC_EQE_CNT_S,
4652		- HNS_ROCE_EQ_INIT_EQE_CNT);
	5787	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQE_CNT_M,
	5788	+ HNS_ROCE_EQC_EQE_CNT_S, HNS_ROCE_EQ_INIT_EQE_CNT);
4653	5789
4654	5790	/* set eqe_ba_pg_sz */
4655		- roce_set_field(eqc->byte_8,
4656		- HNS_ROCE_EQC_BA_PG_SZ_M,
	5791	+ roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M,
4657	5792	HNS_ROCE_EQC_BA_PG_SZ_S,
4658		- eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
	5793	+ to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift));
4659	5794
4660	5795	/* set eqe_buf_pg_sz */
4661		- roce_set_field(eqc->byte_8,
4662		- HNS_ROCE_EQC_BUF_PG_SZ_M,
	5796	+ roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M,
4663	5797	HNS_ROCE_EQC_BUF_PG_SZ_S,
4664		- eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
	5798	+ to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift));
4665	5799
4666	5800	/* set eq_producer_idx */
4667		- roce_set_field(eqc->byte_8,
4668		- HNS_ROCE_EQC_PROD_INDX_M,
4669		- HNS_ROCE_EQC_PROD_INDX_S,
4670		- HNS_ROCE_EQ_INIT_PROD_IDX);
	5801	+ roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M,
	5802	+ HNS_ROCE_EQC_PROD_INDX_S, HNS_ROCE_EQ_INIT_PROD_IDX);
4671	5803
4672	5804	/* set eq_max_cnt */
4673		- roce_set_field(eqc->byte_12,
4674		- HNS_ROCE_EQC_MAX_CNT_M,
	5805	+ roce_set_field(eqc->byte_12, HNS_ROCE_EQC_MAX_CNT_M,
4675	5806	HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
4676	5807
4677	5808	/* set eq_period */
4678		- roce_set_field(eqc->byte_12,
4679		- HNS_ROCE_EQC_PERIOD_M,
	5809	+ roce_set_field(eqc->byte_12, HNS_ROCE_EQC_PERIOD_M,
4680	5810	HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
4681	5811
4682	5812	/* set eqe_report_timer */
4683		- roce_set_field(eqc->eqe_report_timer,
4684		- HNS_ROCE_EQC_REPORT_TIMER_M,
	5813	+ roce_set_field(eqc->eqe_report_timer, HNS_ROCE_EQC_REPORT_TIMER_M,
4685	5814	HNS_ROCE_EQC_REPORT_TIMER_S,
4686	5815	HNS_ROCE_EQ_INIT_REPORT_TIMER);
4687	5816
4688		- /* set eqe_ba [34:3] */
4689		- roce_set_field(eqc->eqe_ba0,
4690		- HNS_ROCE_EQC_EQE_BA_L_M,
4691		- HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
	5817	+ /* set bt_ba [34:3] */
	5818	+ roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M,
	5819	+ HNS_ROCE_EQC_EQE_BA_L_S, bt_ba >> 3);
4692	5820
4693		- /* set eqe_ba [64:35] */
4694		- roce_set_field(eqc->eqe_ba1,
4695		- HNS_ROCE_EQC_EQE_BA_H_M,
4696		- HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
	5821	+ /* set bt_ba [64:35] */
	5822	+ roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M,
	5823	+ HNS_ROCE_EQC_EQE_BA_H_S, bt_ba >> 35);
4697	5824
4698	5825	/* set eq shift */
4699		- roce_set_field(eqc->byte_28,
4700		- HNS_ROCE_EQC_SHIFT_M,
4701		- HNS_ROCE_EQC_SHIFT_S, eq->shift);
	5826	+ roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S,
	5827	+ eq->shift);
4702	5828
4703	5829	/* set eq MSI_IDX */
4704		- roce_set_field(eqc->byte_28,
4705		- HNS_ROCE_EQC_MSI_INDX_M,
4706		- HNS_ROCE_EQC_MSI_INDX_S,
4707		- HNS_ROCE_EQ_INIT_MSI_IDX);
	5830	+ roce_set_field(eqc->byte_28, HNS_ROCE_EQC_MSI_INDX_M,
	5831	+ HNS_ROCE_EQC_MSI_INDX_S, HNS_ROCE_EQ_INIT_MSI_IDX);
4708	5832
4709	5833	/* set cur_eqe_ba [27:12] */
4710		- roce_set_field(eqc->byte_28,
4711		- HNS_ROCE_EQC_CUR_EQE_BA_L_M,
4712		- HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
	5834	+ roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M,
	5835	+ HNS_ROCE_EQC_CUR_EQE_BA_L_S, eqe_ba[0] >> 12);
4713	5836
4714	5837	/* set cur_eqe_ba [59:28] */
4715		- roce_set_field(eqc->byte_32,
4716		- HNS_ROCE_EQC_CUR_EQE_BA_M_M,
4717		- HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
	5838	+ roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M,
	5839	+ HNS_ROCE_EQC_CUR_EQE_BA_M_S, eqe_ba[0] >> 28);
4718	5840
4719	5841	/* set cur_eqe_ba [63:60] */
4720		- roce_set_field(eqc->byte_36,
4721		- HNS_ROCE_EQC_CUR_EQE_BA_H_M,
4722		- HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
	5842	+ roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M,
	5843	+ HNS_ROCE_EQC_CUR_EQE_BA_H_S, eqe_ba[0] >> 60);
4723	5844
4724	5845	/* set eq consumer idx */
4725		- roce_set_field(eqc->byte_36,
4726		- HNS_ROCE_EQC_CONS_INDX_M,
4727		- HNS_ROCE_EQC_CONS_INDX_S,
4728		- HNS_ROCE_EQ_INIT_CONS_IDX);
	5846	+ roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M,
	5847	+ HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX);
4729	5848
4730		- /* set nex_eqe_ba[43:12] */
4731		- roce_set_field(eqc->nxt_eqe_ba0,
4732		- HNS_ROCE_EQC_NXT_EQE_BA_L_M,
4733		- HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
	5849	+ roce_set_field(eqc->byte_40, HNS_ROCE_EQC_NXT_EQE_BA_L_M,
	5850	+ HNS_ROCE_EQC_NXT_EQE_BA_L_S, eqe_ba[1] >> 12);
4734	5851
4735		- /* set nex_eqe_ba[63:44] */
4736		- roce_set_field(eqc->nxt_eqe_ba1,
4737		- HNS_ROCE_EQC_NXT_EQE_BA_H_M,
4738		- HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
4739		-}
	5852	+ roce_set_field(eqc->byte_44, HNS_ROCE_EQC_NXT_EQE_BA_H_M,
	5853	+ HNS_ROCE_EQC_NXT_EQE_BA_H_S, eqe_ba[1] >> 44);
4740	5854
4741		-static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
4742		- struct hns_roce_eq *eq)
4743		-{
4744		- struct device *dev = hr_dev->dev;
4745		- int eq_alloc_done = 0;
4746		- int eq_buf_cnt = 0;
4747		- int eqe_alloc;
4748		- u32 buf_chk_sz;
4749		- u32 bt_chk_sz;
4750		- u32 mhop_num;
4751		- u64 size;
4752		- u64 idx;
4753		- int ba_num;
4754		- int bt_num;
4755		- int record_i;
4756		- int record_j;
4757		- int i = 0;
4758		- int j = 0;
4759		-
4760		- mhop_num = hr_dev->caps.eqe_hop_num;
4761		- buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
4762		- bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
4763		-
4764		- ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1)
4765		- / buf_chk_sz;
4766		- bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8);
4767		-
4768		- /* hop_num = 0 */
4769		- if (mhop_num == HNS_ROCE_HOP_NUM_0) {
4770		- if (eq->entries > buf_chk_sz / eq->eqe_size) {
4771		- dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
4772		- eq->entries);
4773		- return -EINVAL;
4774		- }
4775		- eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
4776		- &(eq->l0_dma), GFP_KERNEL);
4777		- if (!eq->bt_l0)
4778		- return -ENOMEM;
4779		-
4780		- eq->cur_eqe_ba = eq->l0_dma;
4781		- eq->nxt_eqe_ba = 0;
4782		-
4783		- memset(eq->bt_l0, 0, eq->entries * eq->eqe_size);
4784		-
4785		- return 0;
4786		- }
4787		-
4788		- eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
4789		- if (!eq->buf_dma)
4790		- return -ENOMEM;
4791		- eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
4792		- if (!eq->buf)
4793		- goto err_kcalloc_buf;
4794		-
4795		- if (mhop_num == 2) {
4796		- eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
4797		- if (!eq->l1_dma)
4798		- goto err_kcalloc_l1_dma;
4799		-
4800		- eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
4801		- if (!eq->bt_l1)
4802		- goto err_kcalloc_bt_l1;
4803		- }
4804		-
4805		- /* alloc L0 BT */
4806		- eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
4807		- if (!eq->bt_l0)
4808		- goto err_dma_alloc_l0;
4809		-
4810		- if (mhop_num == 1) {
4811		- if (ba_num > (bt_chk_sz / 8))
4812		- dev_err(dev, "ba_num %d is too large for 1 hop\n",
4813		- ba_num);
4814		-
4815		- /* alloc buf */
4816		- for (i = 0; i < bt_chk_sz / 8; i++) {
4817		- if (eq_buf_cnt + 1 < ba_num) {
4818		- size = buf_chk_sz;
4819		- } else {
4820		- eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
4821		- size = (eq->entries - eqe_alloc) * eq->eqe_size;
4822		- }
4823		- eq->buf[i] = dma_alloc_coherent(dev, size,
4824		- &(eq->buf_dma[i]),
4825		- GFP_KERNEL);
4826		- if (!eq->buf[i])
4827		- goto err_dma_alloc_buf;
4828		-
4829		- memset(eq->buf[i], 0, size);
4830		- *(eq->bt_l0 + i) = eq->buf_dma[i];
4831		-
4832		- eq_buf_cnt++;
4833		- if (eq_buf_cnt >= ba_num)
4834		- break;
4835		- }
4836		- eq->cur_eqe_ba = eq->buf_dma[0];
4837		- if (ba_num > 1)
4838		- eq->nxt_eqe_ba = eq->buf_dma[1];
4839		-
4840		- } else if (mhop_num == 2) {
4841		- /* alloc L1 BT and buf */
4842		- for (i = 0; i < bt_chk_sz / 8; i++) {
4843		- eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
4844		- &(eq->l1_dma[i]),
4845		- GFP_KERNEL);
4846		- if (!eq->bt_l1[i])
4847		- goto err_dma_alloc_l1;
4848		- *(eq->bt_l0 + i) = eq->l1_dma[i];
4849		-
4850		- for (j = 0; j < bt_chk_sz / 8; j++) {
4851		- idx = i * bt_chk_sz / 8 + j;
4852		- if (eq_buf_cnt + 1 < ba_num) {
4853		- size = buf_chk_sz;
4854		- } else {
4855		- eqe_alloc = (buf_chk_sz / eq->eqe_size)
4856		- * idx;
4857		- size = (eq->entries - eqe_alloc)
4858		- * eq->eqe_size;
4859		- }
4860		- eq->buf[idx] = dma_alloc_coherent(dev, size,
4861		- &(eq->buf_dma[idx]),
4862		- GFP_KERNEL);
4863		- if (!eq->buf[idx])
4864		- goto err_dma_alloc_buf;
4865		-
4866		- memset(eq->buf[idx], 0, size);
4867		- *(eq->bt_l1[i] + j) = eq->buf_dma[idx];
4868		-
4869		- eq_buf_cnt++;
4870		- if (eq_buf_cnt >= ba_num) {
4871		- eq_alloc_done = 1;
4872		- break;
4873		- }
4874		- }
4875		-
4876		- if (eq_alloc_done)
4877		- break;
4878		- }
4879		- eq->cur_eqe_ba = eq->buf_dma[0];
4880		- if (ba_num > 1)
4881		- eq->nxt_eqe_ba = eq->buf_dma[1];
4882		- }
4883		-
4884		- eq->l0_last_num = i + 1;
4885		- if (mhop_num == 2)
4886		- eq->l1_last_num = j + 1;
	5855	+ roce_set_field(eqc->byte_44, HNS_ROCE_EQC_EQE_SIZE_M,
	5856	+ HNS_ROCE_EQC_EQE_SIZE_S,
	5857	+ eq->eqe_size == HNS_ROCE_V3_EQE_SIZE ? 1 : 0);
4887	5858
4888	5859	return 0;
	5860	+}
4889	5861
4890		-err_dma_alloc_l1:
4891		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
4892		- eq->bt_l0 = NULL;
4893		- eq->l0_dma = 0;
4894		- for (i -= 1; i >= 0; i--) {
4895		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
4896		- eq->l1_dma[i]);
	5862	+static int alloc_eq_buf(struct hns_roce_dev hr_dev, struct hns_roce_eq eq)
	5863	+{
	5864	+ struct hns_roce_buf_attr buf_attr = {};
	5865	+ int err;
4897	5866
4898		- for (j = 0; j < bt_chk_sz / 8; j++) {
4899		- idx = i * bt_chk_sz / 8 + j;
4900		- dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
4901		- eq->buf_dma[idx]);
4902		- }
4903		- }
4904		- goto err_dma_alloc_l0;
	5867	+ if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)
	5868	+ eq->hop_num = 0;
	5869	+ else
	5870	+ eq->hop_num = hr_dev->caps.eqe_hop_num;
4905	5871
4906		-err_dma_alloc_buf:
4907		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
4908		- eq->bt_l0 = NULL;
4909		- eq->l0_dma = 0;
	5872	+ buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + HNS_HW_PAGE_SHIFT;
	5873	+ buf_attr.region[0].size = eq->entries * eq->eqe_size;
	5874	+ buf_attr.region[0].hopnum = eq->hop_num;
	5875	+ buf_attr.region_count = 1;
	5876	+ buf_attr.fixed_page = true;
4910	5877
4911		- if (mhop_num == 1)
4912		- for (i -= 1; i >= 0; i--)
4913		- dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
4914		- eq->buf_dma[i]);
4915		- else if (mhop_num == 2) {
4916		- record_i = i;
4917		- record_j = j;
4918		- for (; i >= 0; i--) {
4919		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
4920		- eq->l1_dma[i]);
	5878	+ err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
	5879	+ hr_dev->caps.eqe_ba_pg_sz +
	5880	+ HNS_HW_PAGE_SHIFT, NULL, 0);
	5881	+ if (err)
	5882	+ dev_err(hr_dev->dev, "Failed to alloc EQE mtr, err %d\n", err);
4921	5883
4922		- for (j = 0; j < bt_chk_sz / 8; j++) {
4923		- if (i == record_i && j >= record_j)
4924		- break;
4925		-
4926		- idx = i * bt_chk_sz / 8 + j;
4927		- dma_free_coherent(dev, buf_chk_sz,
4928		- eq->buf[idx],
4929		- eq->buf_dma[idx]);
4930		- }
4931		- }
4932		- }
4933		-
4934		-err_dma_alloc_l0:
4935		- kfree(eq->bt_l1);
4936		- eq->bt_l1 = NULL;
4937		-
4938		-err_kcalloc_bt_l1:
4939		- kfree(eq->l1_dma);
4940		- eq->l1_dma = NULL;
4941		-
4942		-err_kcalloc_l1_dma:
4943		- kfree(eq->buf);
4944		- eq->buf = NULL;
4945		-
4946		-err_kcalloc_buf:
4947		- kfree(eq->buf_dma);
4948		- eq->buf_dma = NULL;
4949		-
4950		- return -ENOMEM;
	5884	+ return err;
4951	5885	}
4952	5886
4953	5887	static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
4954	5888	struct hns_roce_eq *eq,
4955	5889	unsigned int eq_cmd)
4956	5890	{
4957		- struct device *dev = hr_dev->dev;
4958	5891	struct hns_roce_cmd_mailbox *mailbox;
4959		- u32 buf_chk_sz = 0;
4960	5892	int ret;
4961	5893
4962	5894	/* Allocate mailbox memory */
4963	5895	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4964		- if (IS_ERR(mailbox))
4965		- return PTR_ERR(mailbox);
	5896	+ if (IS_ERR_OR_NULL(mailbox))
	5897	+ return -ENOMEM;
4966	5898
4967		- if (!hr_dev->caps.eqe_hop_num) {
4968		- buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
	5899	+ ret = alloc_eq_buf(hr_dev, eq);
	5900	+ if (ret)
	5901	+ goto free_cmd_mbox;
4969	5902
4970		- eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
4971		- GFP_KERNEL);
4972		- if (!eq->buf_list) {
4973		- ret = -ENOMEM;
4974		- goto free_cmd_mbox;
4975		- }
4976		-
4977		- eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
4978		- &(eq->buf_list->map),
4979		- GFP_KERNEL);
4980		- if (!eq->buf_list->buf) {
4981		- ret = -ENOMEM;
4982		- goto err_alloc_buf;
4983		- }
4984		-
4985		- memset(eq->buf_list->buf, 0, buf_chk_sz);
4986		- } else {
4987		- ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
4988		- if (ret) {
4989		- ret = -ENOMEM;
4990		- goto free_cmd_mbox;
4991		- }
4992		- }
4993		-
4994		- hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
	5903	+ ret = config_eqc(hr_dev, eq, mailbox->buf);
	5904	+ if (ret)
	5905	+ goto err_cmd_mbox;
4995	5906
4996	5907	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
4997	5908	eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
4998	5909	if (ret) {
4999		- dev_err(dev, "[mailbox cmd] create eqc failed.\n");
	5910	+ dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n");
5000	5911	goto err_cmd_mbox;
5001	5912	}
5002	5913
..	..	@@ -5005,21 +5916,99 @@
5005	5916	return 0;
5006	5917
5007	5918	err_cmd_mbox:
5008		- if (!hr_dev->caps.eqe_hop_num)
5009		- dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
5010		- eq->buf_list->map);
5011		- else {
5012		- hns_roce_mhop_free_eq(hr_dev, eq);
5013		- goto free_cmd_mbox;
5014		- }
5015		-
5016		-err_alloc_buf:
5017		- kfree(eq->buf_list);
	5919	+ free_eq_buf(hr_dev, eq);
5018	5920
5019	5921	free_cmd_mbox:
5020	5922	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5021	5923
5022	5924	return ret;
	5925	+}
	5926	+
	5927	+static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
	5928	+ int comp_num, int aeq_num, int other_num)
	5929	+{
	5930	+ struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
	5931	+ int i, j;
	5932	+ int ret;
	5933	+
	5934	+ for (i = 0; i < irq_num; i++) {
	5935	+ hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
	5936	+ GFP_KERNEL);
	5937	+ if (!hr_dev->irq_names[i]) {
	5938	+ ret = -ENOMEM;
	5939	+ goto err_kzalloc_failed;
	5940	+ }
	5941	+ }
	5942	+
	5943	+ /* irq contains: abnormal + AEQ + CEQ */
	5944	+ for (j = 0; j < other_num; j++)
	5945	+ snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
	5946	+ "hns-abn-%d", j);
	5947	+
	5948	+ for (j = other_num; j < (other_num + aeq_num); j++)
	5949	+ snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
	5950	+ "hns-aeq-%d", j - other_num);
	5951	+
	5952	+ for (j = (other_num + aeq_num); j < irq_num; j++)
	5953	+ snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
	5954	+ "hns-ceq-%d", j - other_num - aeq_num);
	5955	+
	5956	+ for (j = 0; j < irq_num; j++) {
	5957	+ if (j < other_num)
	5958	+ ret = request_irq(hr_dev->irq[j],
	5959	+ hns_roce_v2_msix_interrupt_abn,
	5960	+ 0, hr_dev->irq_names[j], hr_dev);
	5961	+
	5962	+ else if (j < (other_num + comp_num))
	5963	+ ret = request_irq(eq_table->eq[j - other_num].irq,
	5964	+ hns_roce_v2_msix_interrupt_eq,
	5965	+ 0, hr_dev->irq_names[j + aeq_num],
	5966	+ &eq_table->eq[j - other_num]);
	5967	+ else
	5968	+ ret = request_irq(eq_table->eq[j - other_num].irq,
	5969	+ hns_roce_v2_msix_interrupt_eq,
	5970	+ 0, hr_dev->irq_names[j - comp_num],
	5971	+ &eq_table->eq[j - other_num]);
	5972	+ if (ret) {
	5973	+ dev_err(hr_dev->dev, "Request irq error!\n");
	5974	+ goto err_request_failed;
	5975	+ }
	5976	+ }
	5977	+
	5978	+ return 0;
	5979	+
	5980	+err_request_failed:
	5981	+ for (j -= 1; j >= 0; j--)
	5982	+ if (j < other_num)
	5983	+ free_irq(hr_dev->irq[j], hr_dev);
	5984	+ else
	5985	+ free_irq(eq_table->eq[j - other_num].irq,
	5986	+ &eq_table->eq[j - other_num]);
	5987	+
	5988	+err_kzalloc_failed:
	5989	+ for (i -= 1; i >= 0; i--)
	5990	+ kfree(hr_dev->irq_names[i]);
	5991	+
	5992	+ return ret;
	5993	+}
	5994	+
	5995	+static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
	5996	+{
	5997	+ int irq_num;
	5998	+ int eq_num;
	5999	+ int i;
	6000	+
	6001	+ eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
	6002	+ irq_num = eq_num + hr_dev->caps.num_other_vectors;
	6003	+
	6004	+ for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
	6005	+ free_irq(hr_dev->irq[i], hr_dev);
	6006	+
	6007	+ for (i = 0; i < eq_num; i++)
	6008	+ free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
	6009	+
	6010	+ for (i = 0; i < irq_num; i++)
	6011	+ kfree(hr_dev->irq_names[i]);
5023	6012	}
5024	6013
5025	6014	static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
..	..	@@ -5033,7 +6022,7 @@
5033	6022	int other_num;
5034	6023	int comp_num;
5035	6024	int aeq_num;
5036		- int i, j, k;
	6025	+ int i;
5037	6026	int ret;
5038	6027
5039	6028	other_num = hr_dev->caps.num_other_vectors;
..	..	@@ -5047,27 +6036,18 @@
5047	6036	if (!eq_table->eq)
5048	6037	return -ENOMEM;
5049	6038
5050		- for (i = 0; i < irq_num; i++) {
5051		- hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5052		- GFP_KERNEL);
5053		- if (!hr_dev->irq_names[i]) {
5054		- ret = -ENOMEM;
5055		- goto err_failed_kzalloc;
5056		- }
5057		- }
5058		-
5059	6039	/* create eq */
5060		- for (j = 0; j < eq_num; j++) {
5061		- eq = &eq_table->eq[j];
	6040	+ for (i = 0; i < eq_num; i++) {
	6041	+ eq = &eq_table->eq[i];
5062	6042	eq->hr_dev = hr_dev;
5063		- eq->eqn = j;
5064		- if (j < comp_num) {
	6043	+ eq->eqn = i;
	6044	+ if (i < comp_num) {
5065	6045	/* CEQ */
5066	6046	eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
5067	6047	eq->type_flag = HNS_ROCE_CEQ;
5068	6048	eq->entries = hr_dev->caps.ceqe_depth;
5069		- eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
5070		- eq->irq = hr_dev->irq[j + other_num + aeq_num];
	6049	+ eq->eqe_size = hr_dev->caps.ceqe_size;
	6050	+ eq->irq = hr_dev->irq[i + other_num + aeq_num];
5071	6051	eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
5072	6052	eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
5073	6053	} else {
..	..	@@ -5075,8 +6055,8 @@
5075	6055	eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
5076	6056	eq->type_flag = HNS_ROCE_AEQ;
5077	6057	eq->entries = hr_dev->caps.aeqe_depth;
5078		- eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
5079		- eq->irq = hr_dev->irq[j - comp_num + other_num];
	6058	+ eq->eqe_size = hr_dev->caps.aeqe_size;
	6059	+ eq->irq = hr_dev->irq[i - comp_num + other_num];
5080	6060	eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
5081	6061	eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
5082	6062	}
..	..	@@ -5091,67 +6071,31 @@
5091	6071	/* enable irq */
5092	6072	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
5093	6073
5094		- /* irq contains: abnormal + AEQ + CEQ*/
5095		- for (k = 0; k < irq_num; k++)
5096		- if (k < other_num)
5097		- snprintf((char *)hr_dev->irq_names[k],
5098		- HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k);
5099		- else if (k < (other_num + aeq_num))
5100		- snprintf((char *)hr_dev->irq_names[k],
5101		- HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",
5102		- k - other_num);
5103		- else
5104		- snprintf((char *)hr_dev->irq_names[k],
5105		- HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",
5106		- k - other_num - aeq_num);
5107		-
5108		- for (k = 0; k < irq_num; k++) {
5109		- if (k < other_num)
5110		- ret = request_irq(hr_dev->irq[k],
5111		- hns_roce_v2_msix_interrupt_abn,
5112		- 0, hr_dev->irq_names[k], hr_dev);
5113		-
5114		- else if (k < (other_num + comp_num))
5115		- ret = request_irq(eq_table->eq[k - other_num].irq,
5116		- hns_roce_v2_msix_interrupt_eq,
5117		- 0, hr_dev->irq_names[k + aeq_num],
5118		- &eq_table->eq[k - other_num]);
5119		- else
5120		- ret = request_irq(eq_table->eq[k - other_num].irq,
5121		- hns_roce_v2_msix_interrupt_eq,
5122		- 0, hr_dev->irq_names[k - comp_num],
5123		- &eq_table->eq[k - other_num]);
5124		- if (ret) {
5125		- dev_err(dev, "Request irq error!\n");
5126		- goto err_request_irq_fail;
5127		- }
	6074	+ ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num,
	6075	+ aeq_num, other_num);
	6076	+ if (ret) {
	6077	+ dev_err(dev, "Request irq failed.\n");
	6078	+ goto err_request_irq_fail;
5128	6079	}
5129	6080
5130		- hr_dev->irq_workq =
5131		- create_singlethread_workqueue("hns_roce_irq_workqueue");
	6081	+ hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0);
5132	6082	if (!hr_dev->irq_workq) {
5133	6083	dev_err(dev, "Create irq workqueue failed!\n");
5134	6084	ret = -ENOMEM;
5135		- goto err_request_irq_fail;
	6085	+ goto err_create_wq_fail;
5136	6086	}
5137	6087
5138	6088	return 0;
5139	6089
	6090	+err_create_wq_fail:
	6091	+ __hns_roce_free_irq(hr_dev);
	6092	+
5140	6093	err_request_irq_fail:
5141		- for (k -= 1; k >= 0; k--)
5142		- if (k < other_num)
5143		- free_irq(hr_dev->irq[k], hr_dev);
5144		- else
5145		- free_irq(eq_table->eq[k - other_num].irq,
5146		- &eq_table->eq[k - other_num]);
	6094	+ hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5147	6095
5148	6096	err_create_eq_fail:
5149		- for (j -= 1; j >= 0; j--)
5150		- hns_roce_v2_free_eq(hr_dev, &eq_table->eq[j]);
5151		-
5152		-err_failed_kzalloc:
5153	6097	for (i -= 1; i >= 0; i--)
5154		- kfree(hr_dev->irq_names[i]);
	6098	+ free_eq_buf(hr_dev, &eq_table->eq[i]);
5155	6099	kfree(eq_table->eq);
5156	6100
5157	6101	return ret;
..	..	@@ -5160,35 +6104,47 @@
5160	6104	static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
5161	6105	{
5162	6106	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5163		- int irq_num;
5164	6107	int eq_num;
5165	6108	int i;
5166	6109
5167	6110	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
5168		- irq_num = eq_num + hr_dev->caps.num_other_vectors;
5169	6111
5170	6112	/* Disable irq */
5171	6113	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5172	6114
5173		- for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
5174		- free_irq(hr_dev->irq[i], hr_dev);
	6115	+ __hns_roce_free_irq(hr_dev);
5175	6116
5176	6117	for (i = 0; i < eq_num; i++) {
5177	6118	hns_roce_v2_destroy_eqc(hr_dev, i);
5178	6119
5179		- free_irq(eq_table->eq[i].irq, &eq_table->eq[i]);
5180		-
5181		- hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
	6120	+ free_eq_buf(hr_dev, &eq_table->eq[i]);
5182	6121	}
5183		-
5184		- for (i = 0; i < irq_num; i++)
5185		- kfree(hr_dev->irq_names[i]);
5186	6122
5187	6123	kfree(eq_table->eq);
5188	6124
5189	6125	flush_workqueue(hr_dev->irq_workq);
5190	6126	destroy_workqueue(hr_dev->irq_workq);
5191	6127	}
	6128	+
	6129	+static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
	6130	+ .query_cqc_info = hns_roce_v2_query_cqc_info,
	6131	+};
	6132	+
	6133	+static const struct ib_device_ops hns_roce_v2_dev_ops = {
	6134	+ .destroy_qp = hns_roce_v2_destroy_qp,
	6135	+ .modify_cq = hns_roce_v2_modify_cq,
	6136	+ .poll_cq = hns_roce_v2_poll_cq,
	6137	+ .post_recv = hns_roce_v2_post_recv,
	6138	+ .post_send = hns_roce_v2_post_send,
	6139	+ .query_qp = hns_roce_v2_query_qp,
	6140	+ .req_notify_cq = hns_roce_v2_req_notify_cq,
	6141	+};
	6142	+
	6143	+static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
	6144	+ .modify_srq = hns_roce_v2_modify_srq,
	6145	+ .post_srq_recv = hns_roce_v2_post_srq_recv,
	6146	+ .query_srq = hns_roce_v2_query_srq,
	6147	+};
5192	6148
5193	6149	static const struct hns_roce_hw hns_roce_hw_v2 = {
5194	6150	.cmq_init = hns_roce_v2_cmq_init,
..	..	@@ -5198,16 +6154,20 @@
5198	6154	.hw_exit = hns_roce_v2_exit,
5199	6155	.post_mbox = hns_roce_v2_post_mbox,
5200	6156	.chk_mbox = hns_roce_v2_chk_mbox,
	6157	+ .rst_prc_mbox = hns_roce_v2_rst_process_cmd,
5201	6158	.set_gid = hns_roce_v2_set_gid,
5202	6159	.set_mac = hns_roce_v2_set_mac,
5203	6160	.write_mtpt = hns_roce_v2_write_mtpt,
5204	6161	.rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
	6162	+ .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
	6163	+ .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
5205	6164	.write_cqc = hns_roce_v2_write_cqc,
5206	6165	.set_hem = hns_roce_v2_set_hem,
5207	6166	.clear_hem = hns_roce_v2_clear_hem,
5208	6167	.modify_qp = hns_roce_v2_modify_qp,
5209	6168	.query_qp = hns_roce_v2_query_qp,
5210	6169	.destroy_qp = hns_roce_v2_destroy_qp,
	6170	+ .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
5211	6171	.modify_cq = hns_roce_v2_modify_cq,
5212	6172	.post_send = hns_roce_v2_post_send,
5213	6173	.post_recv = hns_roce_v2_post_recv,
..	..	@@ -5215,6 +6175,12 @@
5215	6175	.poll_cq = hns_roce_v2_poll_cq,
5216	6176	.init_eq = hns_roce_v2_init_eq_table,
5217	6177	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
	6178	+ .write_srqc = hns_roce_v2_write_srqc,
	6179	+ .modify_srq = hns_roce_v2_modify_srq,
	6180	+ .query_srq = hns_roce_v2_query_srq,
	6181	+ .post_srq_recv = hns_roce_v2_post_srq_recv,
	6182	+ .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
	6183	+ .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
5218	6184	};
5219	6185
5220	6186	static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
..	..	@@ -5229,19 +6195,16 @@
5229	6195
5230	6196	MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
5231	6197
5232		-static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
	6198	+static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
5233	6199	struct hnae3_handle *handle)
5234	6200	{
5235		- const struct pci_device_id *id;
	6201	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
5236	6202	int i;
5237	6203
5238		- id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);
5239		- if (!id) {
5240		- dev_err(hr_dev->dev, "device is not compatible!\n");
5241		- return -ENXIO;
5242		- }
5243		-
	6204	+ hr_dev->pci_dev = handle->pdev;
	6205	+ hr_dev->dev = &handle->pdev->dev;
5244	6206	hr_dev->hw = &hns_roce_hw_v2;
	6207	+ hr_dev->dfx = &hns_roce_dfx_hw_v2;
5245	6208	hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
5246	6209	hr_dev->odb_offset = hr_dev->sdb_offset;
5247	6210
..	..	@@ -5262,15 +6225,16 @@
5262	6225	hr_dev->cmd_mod = 1;
5263	6226	hr_dev->loop_idc = 0;
5264	6227
5265		- return 0;
	6228	+ hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
	6229	+ priv->handle = handle;
5266	6230	}
5267	6231
5268		-static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
	6232	+static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
5269	6233	{
5270	6234	struct hns_roce_dev *hr_dev;
5271	6235	int ret;
5272	6236
5273		- hr_dev = (struct hns_roce_dev )ib_alloc_device(sizeof(hr_dev));
	6237	+ hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
5274	6238	if (!hr_dev)
5275	6239	return -ENOMEM;
5276	6240
..	..	@@ -5280,21 +6244,15 @@
5280	6244	goto error_failed_kzalloc;
5281	6245	}
5282	6246
5283		- hr_dev->pci_dev = handle->pdev;
5284		- hr_dev->dev = &handle->pdev->dev;
5285		- handle->priv = hr_dev;
5286		-
5287		- ret = hns_roce_hw_v2_get_cfg(hr_dev, handle);
5288		- if (ret) {
5289		- dev_err(hr_dev->dev, "Get Configuration failed!\n");
5290		- goto error_failed_get_cfg;
5291		- }
	6247	+ hns_roce_hw_v2_get_cfg(hr_dev, handle);
5292	6248
5293	6249	ret = hns_roce_init(hr_dev);
5294	6250	if (ret) {
5295	6251	dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
5296	6252	goto error_failed_get_cfg;
5297	6253	}
	6254	+
	6255	+ handle->priv = hr_dev;
5298	6256
5299	6257	return 0;
5300	6258
..	..	@@ -5307,54 +6265,125 @@
5307	6265	return ret;
5308	6266	}
5309	6267
5310		-static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
	6268	+static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
5311	6269	bool reset)
5312	6270	{
5313		- struct hns_roce_dev hr_dev = (struct hns_roce_dev )handle->priv;
	6271	+ struct hns_roce_dev *hr_dev = handle->priv;
5314	6272
5315	6273	if (!hr_dev)
5316	6274	return;
	6275	+
	6276	+ handle->priv = NULL;
	6277	+
	6278	+ hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;
	6279	+ hns_roce_handle_device_err(hr_dev);
5317	6280
5318	6281	hns_roce_exit(hr_dev);
5319	6282	kfree(hr_dev->priv);
5320	6283	ib_dealloc_device(&hr_dev->ib_dev);
5321	6284	}
5322	6285
5323		-static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
	6286	+static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
5324	6287	{
5325		- struct hns_roce_dev hr_dev = (struct hns_roce_dev )handle->priv;
5326		- struct ib_event event;
	6288	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	6289	+ const struct pci_device_id *id;
	6290	+ struct device *dev = &handle->pdev->dev;
	6291	+ int ret;
5327	6292
5328		- if (!hr_dev) {
5329		- dev_err(&handle->pdev->dev,
5330		- "Input parameter handle->priv is NULL!\n");
5331		- return -EINVAL;
	6293	+ handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
	6294	+
	6295	+ if (ops->ae_dev_resetting(handle) \|\| ops->get_hw_reset_stat(handle)) {
	6296	+ handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
	6297	+ goto reset_chk_err;
5332	6298	}
5333	6299
5334		- hr_dev->active = false;
5335		- hr_dev->is_reset = true;
	6300	+ id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
	6301	+ if (!id)
	6302	+ return 0;
5336	6303
5337		- event.event = IB_EVENT_DEVICE_FATAL;
5338		- event.device = &hr_dev->ib_dev;
5339		- event.element.port_num = 1;
5340		- ib_dispatch_event(&event);
	6304	+ ret = __hns_roce_hw_v2_init_instance(handle);
	6305	+ if (ret) {
	6306	+ handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
	6307	+ dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
	6308	+ if (ops->ae_dev_resetting(handle) \|\|
	6309	+ ops->get_hw_reset_stat(handle))
	6310	+ goto reset_chk_err;
	6311	+ else
	6312	+ return ret;
	6313	+ }
	6314	+
	6315	+ handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
	6316	+
	6317	+
	6318	+ return 0;
	6319	+
	6320	+reset_chk_err:
	6321	+ dev_err(dev, "Device is busy in resetting state.\n"
	6322	+ "please retry later.\n");
	6323	+
	6324	+ return -EBUSY;
	6325	+}
	6326	+
	6327	+static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
	6328	+ bool reset)
	6329	+{
	6330	+ if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
	6331	+ return;
	6332	+
	6333	+ handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
	6334	+
	6335	+ __hns_roce_hw_v2_uninit_instance(handle, reset);
	6336	+
	6337	+ handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
	6338	+}
	6339	+static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
	6340	+{
	6341	+ struct hns_roce_dev *hr_dev;
	6342	+
	6343	+ if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
	6344	+ set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
	6345	+ return 0;
	6346	+ }
	6347	+
	6348	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
	6349	+ clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
	6350	+
	6351	+ hr_dev = handle->priv;
	6352	+ if (!hr_dev)
	6353	+ return 0;
	6354	+
	6355	+ hr_dev->active = false;
	6356	+ hr_dev->dis_db = true;
	6357	+ hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
5341	6358
5342	6359	return 0;
5343	6360	}
5344	6361
5345	6362	static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
5346	6363	{
	6364	+ struct device *dev = &handle->pdev->dev;
5347	6365	int ret;
5348	6366
5349		- ret = hns_roce_hw_v2_init_instance(handle);
	6367	+ if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
	6368	+ &handle->rinfo.state)) {
	6369	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
	6370	+ return 0;
	6371	+ }
	6372	+
	6373	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
	6374	+
	6375	+ dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
	6376	+ ret = __hns_roce_hw_v2_init_instance(handle);
5350	6377	if (ret) {
5351	6378	/* when reset notify type is HNAE3_INIT_CLIENT In reset notify
5352	6379	* callback function, RoCE Engine reinitialize. If RoCE reinit
5353	6380	* failed, we should inform NIC driver.
5354	6381	*/
5355	6382	handle->priv = NULL;
5356		- dev_err(&handle->pdev->dev,
5357		- "In reset process RoCE reinit failed %d.\n", ret);
	6383	+ dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
	6384	+ } else {
	6385	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
	6386	+ dev_info(dev, "Reset done, RoCE client reinit finished.\n");
5358	6387	}
5359	6388
5360	6389	return ret;
..	..	@@ -5362,8 +6391,14 @@
5362	6391
5363	6392	static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
5364	6393	{
5365		- msleep(100);
5366		- hns_roce_hw_v2_uninit_instance(handle, false);
	6394	+ if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
	6395	+ return 0;
	6396	+
	6397	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
	6398	+ dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
	6399	+ msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
	6400	+ __hns_roce_hw_v2_uninit_instance(handle, false);
	6401	+
5367	6402	return 0;
5368	6403	}
5369	6404