~hc/RK356X_SDK_RELEASE.git

..	..	@@ -33,11 +33,14 @@
33	33	#include <linux/acpi.h>
34	34	#include <linux/etherdevice.h>
35	35	#include <linux/interrupt.h>
	36	+#include <linux/iopoll.h>
36	37	#include <linux/kernel.h>
37	38	#include <linux/types.h>
38	39	#include <net/addrconf.h>
39	40	#include <rdma/ib_addr.h>
	41	+#include <rdma/ib_cache.h>
40	42	#include <rdma/ib_umem.h>
	43	+#include <rdma/uverbs_ioctl.h>
41	44
42	45	#include "hnae3.h"
43	46	#include "hns_roce_common.h"
..	..	@@ -54,166 +57,612 @@
54	57	dseg->len = cpu_to_le32(sg->length);
55	58	}
56	59
57		-static void set_extend_sge(struct hns_roce_qp qp, const struct ib_send_wr wr,
58		- unsigned int *sge_ind)
	60	+/*
	61	+ * mapped-value = 1 + real-value
	62	+ * The hns wr opcode real value is start from 0, In order to distinguish between
	63	+ * initialized and uninitialized map values, we plus 1 to the actual value when
	64	+ * defining the mapping, so that the validity can be identified by checking the
	65	+ * mapped value is greater than 0.
	66	+ */
	67	+#define HR_OPC_MAP(ib_key, hr_key) \
	68	+ [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key
	69	+
	70	+static const u32 hns_roce_op_code[] = {
	71	+ HR_OPC_MAP(RDMA_WRITE, RDMA_WRITE),
	72	+ HR_OPC_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE_WITH_IMM),
	73	+ HR_OPC_MAP(SEND, SEND),
	74	+ HR_OPC_MAP(SEND_WITH_IMM, SEND_WITH_IMM),
	75	+ HR_OPC_MAP(RDMA_READ, RDMA_READ),
	76	+ HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP),
	77	+ HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD),
	78	+ HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV),
	79	+ HR_OPC_MAP(LOCAL_INV, LOCAL_INV),
	80	+ HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP),
	81	+ HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD),
	82	+ HR_OPC_MAP(REG_MR, FAST_REG_PMR),
	83	+};
	84	+
	85	+static u32 to_hr_opcode(u32 ib_opcode)
59	86	{
60		- struct hns_roce_v2_wqe_data_seg *dseg;
61		- struct ib_sge *sg;
62		- int num_in_wqe = 0;
63		- int extend_sge_num;
64		- int fi_sge_num;
65		- int se_sge_num;
66		- int shift;
67		- int i;
	87	+ if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code))
	88	+ return HNS_ROCE_V2_WQE_OP_MASK;
68	89
69		- if (qp->ibqp.qp_type == IB_QPT_RC \|\| qp->ibqp.qp_type == IB_QPT_UC)
70		- num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
71		- extend_sge_num = wr->num_sge - num_in_wqe;
72		- sg = wr->sg_list + num_in_wqe;
73		- shift = qp->hr_buf.page_shift;
	90	+ return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :
	91	+ HNS_ROCE_V2_WQE_OP_MASK;
	92	+}
74	93
75		- /*
76		- * Check whether wr->num_sge sges are in the same page. If not, we
77		- * should calculate how many sges in the first page and the second
78		- * page.
79		- */
80		- dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
81		- fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
82		- (uintptr_t)dseg) /
83		- sizeof(struct hns_roce_v2_wqe_data_seg);
84		- if (extend_sge_num > fi_sge_num) {
85		- se_sge_num = extend_sge_num - fi_sge_num;
86		- for (i = 0; i < fi_sge_num; i++) {
87		- set_data_seg_v2(dseg++, sg + i);
88		- (*sge_ind)++;
89		- }
90		- dseg = get_send_extend_sge(qp,
91		- (*sge_ind) & (qp->sge.sge_cnt - 1));
92		- for (i = 0; i < se_sge_num; i++) {
93		- set_data_seg_v2(dseg++, sg + fi_sge_num + i);
94		- (*sge_ind)++;
95		- }
	94	+static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
	95	+ const struct ib_reg_wr *wr)
	96	+{
	97	+ struct hns_roce_wqe_frmr_seg *fseg =
	98	+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
	99	+ struct hns_roce_mr *mr = to_hr_mr(wr->mr);
	100	+ u64 pbl_ba;
	101	+
	102	+ /* use ib_access_flags */
	103	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
	104	+ wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
	105	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
	106	+ wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
	107	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RR_S,
	108	+ wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
	109	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RW_S,
	110	+ wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
	111	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_LW_S,
	112	+ wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
	113	+
	114	+ /* Data structure reuse may lead to confusion */
	115	+ pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
	116	+ rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));
	117	+ rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));
	118	+
	119	+ rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
	120	+ rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
	121	+ rc_sq_wqe->rkey = cpu_to_le32(wr->key);
	122	+ rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
	123	+
	124	+ fseg->pbl_size = cpu_to_le32(mr->npages);
	125	+ roce_set_field(fseg->mode_buf_pg_sz,
	126	+ V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
	127	+ V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
	128	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
	129	+ roce_set_bit(fseg->mode_buf_pg_sz,
	130	+ V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
	131	+}
	132	+
	133	+static void set_atomic_seg(const struct ib_send_wr *wr,
	134	+ struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
	135	+ unsigned int valid_num_sge)
	136	+{
	137	+ struct hns_roce_v2_wqe_data_seg *dseg =
	138	+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
	139	+ struct hns_roce_wqe_atomic_seg *aseg =
	140	+ (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);
	141	+
	142	+ set_data_seg_v2(dseg, wr->sg_list);
	143	+
	144	+ if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
	145	+ aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
	146	+ aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add);
96	147	} else {
97		- for (i = 0; i < extend_sge_num; i++) {
98		- set_data_seg_v2(dseg++, sg + i);
99		- (*sge_ind)++;
	148	+ aseg->fetchadd_swap_data =
	149	+ cpu_to_le64(atomic_wr(wr)->compare_add);
	150	+ aseg->cmp_data = 0;
	151	+ }
	152	+
	153	+ roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
	154	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
	155	+}
	156	+
	157	+static unsigned int get_std_sge_num(struct hns_roce_qp *qp)
	158	+{
	159	+ if (qp->ibqp.qp_type == IB_QPT_GSI \|\| qp->ibqp.qp_type == IB_QPT_UD)
	160	+ return 0;
	161	+
	162	+ return HNS_ROCE_SGE_IN_WQE;
	163	+}
	164	+
	165	+static int fill_ext_sge_inl_data(struct hns_roce_qp *qp,
	166	+ const struct ib_send_wr *wr,
	167	+ unsigned int *sge_idx, u32 msg_len)
	168	+{
	169	+ struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;
	170	+ unsigned int left_len_in_pg;
	171	+ unsigned int idx = *sge_idx;
	172	+ unsigned int std_sge_num;
	173	+ unsigned int i = 0;
	174	+ unsigned int len;
	175	+ void *addr;
	176	+ void *dseg;
	177	+
	178	+ std_sge_num = get_std_sge_num(qp);
	179	+ if (msg_len > (qp->sq.max_gs - std_sge_num) * HNS_ROCE_SGE_SIZE) {
	180	+ ibdev_err(ibdev,
	181	+ "no enough extended sge space for inline data.\n");
	182	+ return -EINVAL;
	183	+ }
	184	+
	185	+ dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
	186	+ left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg;
	187	+ len = wr->sg_list[0].length;
	188	+ addr = (void *)(unsigned long)(wr->sg_list[0].addr);
	189	+
	190	+ /* When copying data to extended sge space, the left length in page may
	191	+ * not long enough for current user's sge. So the data should be
	192	+ * splited into several parts, one in the first page, and the others in
	193	+ * the subsequent pages.
	194	+ */
	195	+ while (1) {
	196	+ if (len <= left_len_in_pg) {
	197	+ memcpy(dseg, addr, len);
	198	+
	199	+ idx += len / HNS_ROCE_SGE_SIZE;
	200	+
	201	+ i++;
	202	+ if (i >= wr->num_sge)
	203	+ break;
	204	+
	205	+ left_len_in_pg -= len;
	206	+ len = wr->sg_list[i].length;
	207	+ addr = (void *)(unsigned long)(wr->sg_list[i].addr);
	208	+ dseg += len;
	209	+ } else {
	210	+ memcpy(dseg, addr, left_len_in_pg);
	211	+
	212	+ len -= left_len_in_pg;
	213	+ addr += left_len_in_pg;
	214	+ idx += left_len_in_pg / HNS_ROCE_SGE_SIZE;
	215	+ dseg = hns_roce_get_extend_sge(qp,
	216	+ idx & (qp->sge.sge_cnt - 1));
	217	+ left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT;
100	218	}
101	219	}
	220	+
	221	+ *sge_idx = idx;
	222	+
	223	+ return 0;
	224	+}
	225	+
	226	+static void set_extend_sge(struct hns_roce_qp qp, struct ib_sge sge,
	227	+ unsigned int *sge_ind, unsigned int cnt)
	228	+{
	229	+ struct hns_roce_v2_wqe_data_seg *dseg;
	230	+ unsigned int idx = *sge_ind;
	231	+
	232	+ while (cnt > 0) {
	233	+ dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
	234	+ if (likely(sge->length)) {
	235	+ set_data_seg_v2(dseg, sge);
	236	+ idx++;
	237	+ cnt--;
	238	+ }
	239	+ sge++;
	240	+ }
	241	+
	242	+ *sge_ind = idx;
	243	+}
	244	+
	245	+static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len)
	246	+{
	247	+ struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
	248	+ int mtu = ib_mtu_enum_to_int(qp->path_mtu);
	249	+
	250	+ if (len > qp->max_inline_data \|\| len > mtu) {
	251	+ ibdev_err(&hr_dev->ib_dev,
	252	+ "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n",
	253	+ len, qp->max_inline_data, mtu);
	254	+ return false;
	255	+ }
	256	+
	257	+ return true;
	258	+}
	259	+
	260	+static int set_rc_inl(struct hns_roce_qp qp, const struct ib_send_wr wr,
	261	+ struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
	262	+ unsigned int *sge_idx)
	263	+{
	264	+ struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
	265	+ u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len);
	266	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	267	+ unsigned int curr_idx = *sge_idx;
	268	+ void *dseg = rc_sq_wqe;
	269	+ unsigned int i;
	270	+ int ret;
	271	+
	272	+ if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {
	273	+ ibdev_err(ibdev, "invalid inline parameters!\n");
	274	+ return -EINVAL;
	275	+ }
	276	+
	277	+ if (!check_inl_data_len(qp, msg_len))
	278	+ return -EINVAL;
	279	+
	280	+ dseg += sizeof(struct hns_roce_v2_rc_send_wqe);
	281	+
	282	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, 1);
	283	+
	284	+ if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) {
	285	+ roce_set_bit(rc_sq_wqe->byte_20,
	286	+ V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 0);
	287	+
	288	+ for (i = 0; i < wr->num_sge; i++) {
	289	+ memcpy(dseg, ((void *)wr->sg_list[i].addr),
	290	+ wr->sg_list[i].length);
	291	+ dseg += wr->sg_list[i].length;
	292	+ }
	293	+ } else {
	294	+ roce_set_bit(rc_sq_wqe->byte_20,
	295	+ V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 1);
	296	+
	297	+ ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len);
	298	+ if (ret)
	299	+ return ret;
	300	+
	301	+ roce_set_field(rc_sq_wqe->byte_16,
	302	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
	303	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
	304	+ curr_idx - *sge_idx);
	305	+ }
	306	+
	307	+ *sge_idx = curr_idx;
	308	+
	309	+ return 0;
102	310	}
103	311
104	312	static int set_rwqe_data_seg(struct ib_qp ibqp, const struct ib_send_wr wr,
105	313	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
106		- void wqe, unsigned int sge_ind,
107		- const struct ib_send_wr **bad_wr)
	314	+ unsigned int *sge_ind,
	315	+ unsigned int valid_num_sge)
108	316	{
109		- struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
110		- struct hns_roce_v2_wqe_data_seg *dseg = wqe;
	317	+ struct hns_roce_v2_wqe_data_seg *dseg =
	318	+ (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
111	319	struct hns_roce_qp *qp = to_hr_qp(ibqp);
	320	+ int j = 0;
112	321	int i;
113	322
114		- if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
115		- if (le32_to_cpu(rc_sq_wqe->msg_len) >
116		- hr_dev->caps.max_sq_inline) {
117		- *bad_wr = wr;
118		- dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
119		- rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
120		- return -EINVAL;
121		- }
	323	+ roce_set_field(rc_sq_wqe->byte_20,
	324	+ V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
	325	+ V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
	326	+ (*sge_ind) & (qp->sge.sge_cnt - 1));
122	327
123		- if (wr->opcode == IB_WR_RDMA_READ) {
124		- *bad_wr = wr;
125		- dev_err(hr_dev->dev, "Not support inline data!\n");
126		- return -EINVAL;
127		- }
	328	+ if (wr->send_flags & IB_SEND_INLINE)
	329	+ return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind);
128	330
	331	+ if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {
129	332	for (i = 0; i < wr->num_sge; i++) {
130		- memcpy(wqe, ((void *)wr->sg_list[i].addr),
131		- wr->sg_list[i].length);
132		- wqe += wr->sg_list[i].length;
	333	+ if (likely(wr->sg_list[i].length)) {
	334	+ set_data_seg_v2(dseg, wr->sg_list + i);
	335	+ dseg++;
	336	+ }
133	337	}
134		-
135		- roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
136		- 1);
137	338	} else {
138		- if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
139		- for (i = 0; i < wr->num_sge; i++) {
140		- if (likely(wr->sg_list[i].length)) {
141		- set_data_seg_v2(dseg, wr->sg_list + i);
142		- dseg++;
143		- }
	339	+ for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) {
	340	+ if (likely(wr->sg_list[i].length)) {
	341	+ set_data_seg_v2(dseg, wr->sg_list + i);
	342	+ dseg++;
	343	+ j++;
144	344	}
145		- } else {
146		- roce_set_field(rc_sq_wqe->byte_20,
147		- V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
148		- V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
149		- (*sge_ind) & (qp->sge.sge_cnt - 1));
150		-
151		- for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
152		- if (likely(wr->sg_list[i].length)) {
153		- set_data_seg_v2(dseg, wr->sg_list + i);
154		- dseg++;
155		- }
156		- }
157		-
158		- set_extend_sge(qp, wr, sge_ind);
159	345	}
160	346
161		- roce_set_field(rc_sq_wqe->byte_16,
162		- V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
163		- V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
	347	+ set_extend_sge(qp, wr->sg_list + i, sge_ind,
	348	+ valid_num_sge - HNS_ROCE_SGE_IN_WQE);
	349	+ }
	350	+
	351	+ roce_set_field(rc_sq_wqe->byte_16,
	352	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
	353	+ V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
	354	+
	355	+ return 0;
	356	+}
	357	+
	358	+static int check_send_valid(struct hns_roce_dev *hr_dev,
	359	+ struct hns_roce_qp *hr_qp)
	360	+{
	361	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	362	+ struct ib_qp *ibqp = &hr_qp->ibqp;
	363	+
	364	+ if (unlikely(ibqp->qp_type != IB_QPT_RC &&
	365	+ ibqp->qp_type != IB_QPT_GSI &&
	366	+ ibqp->qp_type != IB_QPT_UD)) {
	367	+ ibdev_err(ibdev, "Not supported QP(0x%x)type!\n",
	368	+ ibqp->qp_type);
	369	+ return -EOPNOTSUPP;
	370	+ } else if (unlikely(hr_qp->state == IB_QPS_RESET \|\|
	371	+ hr_qp->state == IB_QPS_INIT \|\|
	372	+ hr_qp->state == IB_QPS_RTR)) {
	373	+ ibdev_err(ibdev, "failed to post WQE, QP state %hhu!\n",
	374	+ hr_qp->state);
	375	+ return -EINVAL;
	376	+ } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {
	377	+ ibdev_err(ibdev, "failed to post WQE, dev state %d!\n",
	378	+ hr_dev->state);
	379	+ return -EIO;
164	380	}
165	381
166	382	return 0;
167	383	}
168	384
169		-static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
170		- const struct ib_qp_attr *attr,
171		- int attr_mask, enum ib_qp_state cur_state,
172		- enum ib_qp_state new_state);
	385	+static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,
	386	+ unsigned int *sge_len)
	387	+{
	388	+ unsigned int valid_num = 0;
	389	+ unsigned int len = 0;
	390	+ int i;
	391	+
	392	+ for (i = 0; i < wr->num_sge; i++) {
	393	+ if (likely(wr->sg_list[i].length)) {
	394	+ len += wr->sg_list[i].length;
	395	+ valid_num++;
	396	+ }
	397	+ }
	398	+
	399	+ *sge_len = len;
	400	+ return valid_num;
	401	+}
	402	+
	403	+static __le32 get_immtdata(const struct ib_send_wr *wr)
	404	+{
	405	+ switch (wr->opcode) {
	406	+ case IB_WR_SEND_WITH_IMM:
	407	+ case IB_WR_RDMA_WRITE_WITH_IMM:
	408	+ return cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
	409	+ default:
	410	+ return 0;
	411	+ }
	412	+}
	413	+
	414	+static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
	415	+ const struct ib_send_wr *wr)
	416	+{
	417	+ u32 ib_op = wr->opcode;
	418	+
	419	+ if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM)
	420	+ return -EINVAL;
	421	+
	422	+ ud_sq_wqe->immtdata = get_immtdata(wr);
	423	+
	424	+ roce_set_field(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
	425	+ V2_UD_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op));
	426	+
	427	+ return 0;
	428	+}
	429	+
	430	+static inline int set_ud_wqe(struct hns_roce_qp *qp,
	431	+ const struct ib_send_wr *wr,
	432	+ void wqe, unsigned int sge_idx,
	433	+ unsigned int owner_bit)
	434	+{
	435	+ struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
	436	+ struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
	437	+ struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
	438	+ unsigned int curr_idx = *sge_idx;
	439	+ int valid_num_sge;
	440	+ u32 msg_len = 0;
	441	+ int ret;
	442	+
	443	+ valid_num_sge = calc_wr_sge_num(wr, &msg_len);
	444	+ memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
	445	+
	446	+ ret = set_ud_opcode(ud_sq_wqe, wr);
	447	+ if (WARN_ON(ret))
	448	+ return ret;
	449	+
	450	+ roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
	451	+ V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
	452	+ roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
	453	+ V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
	454	+ roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
	455	+ V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
	456	+ roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
	457	+ V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
	458	+ roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
	459	+ V2_UD_SEND_WQE_BYTE_48_DMAC_4_S, ah->av.mac[4]);
	460	+ roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
	461	+ V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, ah->av.mac[5]);
	462	+
	463	+ ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
	464	+
	465	+ /* Set sig attr */
	466	+ roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S,
	467	+ (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
	468	+
	469	+ /* Set se attr */
	470	+ roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S,
	471	+ (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
	472	+
	473	+ roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S,
	474	+ owner_bit);
	475	+
	476	+ roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M,
	477	+ V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn);
	478	+
	479	+ roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
	480	+ V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
	481	+
	482	+ roce_set_field(ud_sq_wqe->byte_20,
	483	+ V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
	484	+ V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
	485	+ curr_idx & (qp->sge.sge_cnt - 1));
	486	+
	487	+ roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
	488	+ V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
	489	+ ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
	490	+ qp->qkey : ud_wr(wr)->remote_qkey);
	491	+ roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
	492	+ V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn);
	493	+
	494	+ roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
	495	+ V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
	496	+ roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
	497	+ V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
	498	+ roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
	499	+ V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
	500	+ roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
	501	+ V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
	502	+ roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M,
	503	+ V2_UD_SEND_WQE_BYTE_40_PORTN_S, qp->port);
	504	+
	505	+ roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
	506	+ V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index);
	507	+
	508	+ if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) {
	509	+ roce_set_bit(ud_sq_wqe->byte_40,
	510	+ V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
	511	+ ah->av.vlan_en);
	512	+ roce_set_field(ud_sq_wqe->byte_36,
	513	+ V2_UD_SEND_WQE_BYTE_36_VLAN_M,
	514	+ V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
	515	+ }
	516	+
	517	+ memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2);
	518	+
	519	+ set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
	520	+
	521	+ *sge_idx = curr_idx;
	522	+
	523	+ return 0;
	524	+}
	525	+
	526	+static int set_rc_opcode(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
	527	+ const struct ib_send_wr *wr)
	528	+{
	529	+ u32 ib_op = wr->opcode;
	530	+
	531	+ rc_sq_wqe->immtdata = get_immtdata(wr);
	532	+
	533	+ switch (ib_op) {
	534	+ case IB_WR_RDMA_READ:
	535	+ case IB_WR_RDMA_WRITE:
	536	+ case IB_WR_RDMA_WRITE_WITH_IMM:
	537	+ rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
	538	+ rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
	539	+ break;
	540	+ case IB_WR_SEND:
	541	+ case IB_WR_SEND_WITH_IMM:
	542	+ break;
	543	+ case IB_WR_ATOMIC_CMP_AND_SWP:
	544	+ case IB_WR_ATOMIC_FETCH_AND_ADD:
	545	+ rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey);
	546	+ rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr);
	547	+ break;
	548	+ case IB_WR_REG_MR:
	549	+ set_frmr_seg(rc_sq_wqe, reg_wr(wr));
	550	+ break;
	551	+ case IB_WR_LOCAL_INV:
	552	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
	553	+ fallthrough;
	554	+ case IB_WR_SEND_WITH_INV:
	555	+ rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
	556	+ break;
	557	+ default:
	558	+ return -EINVAL;
	559	+ }
	560	+
	561	+ roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
	562	+ V2_RC_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op));
	563	+
	564	+ return 0;
	565	+}
	566	+static inline int set_rc_wqe(struct hns_roce_qp *qp,
	567	+ const struct ib_send_wr *wr,
	568	+ void wqe, unsigned int sge_idx,
	569	+ unsigned int owner_bit)
	570	+{
	571	+ struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
	572	+ unsigned int curr_idx = *sge_idx;
	573	+ unsigned int valid_num_sge;
	574	+ u32 msg_len = 0;
	575	+ int ret;
	576	+
	577	+ valid_num_sge = calc_wr_sge_num(wr, &msg_len);
	578	+ memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
	579	+
	580	+ rc_sq_wqe->msg_len = cpu_to_le32(msg_len);
	581	+
	582	+ ret = set_rc_opcode(rc_sq_wqe, wr);
	583	+ if (WARN_ON(ret))
	584	+ return ret;
	585	+
	586	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S,
	587	+ (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
	588	+
	589	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S,
	590	+ (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
	591	+
	592	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S,
	593	+ (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
	594	+
	595	+ roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
	596	+ owner_bit);
	597	+
	598	+ if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP \|\|
	599	+ wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
	600	+ set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
	601	+ else if (wr->opcode != IB_WR_REG_MR)
	602	+ ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
	603	+ &curr_idx, valid_num_sge);
	604	+
	605	+ *sge_idx = curr_idx;
	606	+
	607	+ return ret;
	608	+}
	609	+
	610	+static inline void update_sq_db(struct hns_roce_dev *hr_dev,
	611	+ struct hns_roce_qp *qp)
	612	+{
	613	+ /*
	614	+ * Hip08 hardware cannot flush the WQEs in SQ if the QP state
	615	+ * gets into errored mode. Hence, as a workaround to this
	616	+ * hardware limitation, driver needs to assist in flushing. But
	617	+ * the flushing operation uses mailbox to convey the QP state to
	618	+ * the hardware and which can sleep due to the mutex protection
	619	+ * around the mailbox calls. Hence, use the deferred flush for
	620	+ * now.
	621	+ */
	622	+ if (qp->state == IB_QPS_ERR) {
	623	+ if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
	624	+ init_flush_work(hr_dev, qp);
	625	+ } else {
	626	+ struct hns_roce_v2_db sq_db = {};
	627	+
	628	+ roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
	629	+ V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
	630	+ roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
	631	+ V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
	632	+ roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
	633	+ V2_DB_PARAMETER_IDX_S, qp->sq.head);
	634	+ roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
	635	+ V2_DB_PARAMETER_SL_S, qp->sl);
	636	+
	637	+ hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
	638	+ }
	639	+}
173	640
174	641	static int hns_roce_v2_post_send(struct ib_qp *ibqp,
175	642	const struct ib_send_wr *wr,
176	643	const struct ib_send_wr **bad_wr)
177	644	{
178	645	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
179		- struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
180		- struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
181		- struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
	646	+ struct ib_device *ibdev = &hr_dev->ib_dev;
182	647	struct hns_roce_qp *qp = to_hr_qp(ibqp);
183		- struct device *dev = hr_dev->dev;
184		- struct hns_roce_v2_db sq_db;
185		- struct ib_qp_attr attr;
186		- unsigned int sge_ind = 0;
	648	+ unsigned long flags = 0;
187	649	unsigned int owner_bit;
188		- unsigned long flags;
189		- unsigned int ind;
	650	+ unsigned int sge_idx;
	651	+ unsigned int wqe_idx;
190	652	void *wqe = NULL;
191		- bool loopback;
192		- int attr_mask;
193		- u32 tmp_len;
194		- int ret = 0;
195		- u8 *smac;
196	653	int nreq;
197		- int i;
198		-
199		- if (unlikely(ibqp->qp_type != IB_QPT_RC &&
200		- ibqp->qp_type != IB_QPT_GSI &&
201		- ibqp->qp_type != IB_QPT_UD)) {
202		- dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
203		- *bad_wr = wr;
204		- return -EOPNOTSUPP;
205		- }
206		-
207		- if (unlikely(qp->state == IB_QPS_RESET \|\| qp->state == IB_QPS_INIT \|\|
208		- qp->state == IB_QPS_RTR)) {
209		- dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
210		- *bad_wr = wr;
211		- return -EINVAL;
212		- }
	654	+ int ret;
213	655
214	656	spin_lock_irqsave(&qp->sq.lock, flags);
215		- ind = qp->sq_next_wqe;
216		- sge_ind = qp->next_sge;
	657	+
	658	+ ret = check_send_valid(hr_dev, qp);
	659	+ if (unlikely(ret)) {
	660	+ *bad_wr = wr;
	661	+ nreq = 0;
	662	+ goto out;
	663	+ }
	664	+
	665	+ sge_idx = qp->next_sge;
217	666
218	667	for (nreq = 0; wr; ++nreq, wr = wr->next) {
219	668	if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
..	..	@@ -222,334 +671,56 @@
222	671	goto out;
223	672	}
224	673
	674	+ wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
	675	+
225	676	if (unlikely(wr->num_sge > qp->sq.max_gs)) {
226		- dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
227		- wr->num_sge, qp->sq.max_gs);
	677	+ ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n",
	678	+ wr->num_sge, qp->sq.max_gs);
228	679	ret = -EINVAL;
229	680	*bad_wr = wr;
230	681	goto out;
231	682	}
232	683
233		- wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
234		- qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
235		- wr->wr_id;
236		-
	684	+ wqe = hns_roce_get_send_wqe(qp, wqe_idx);
	685	+ qp->sq.wrid[wqe_idx] = wr->wr_id;
237	686	owner_bit =
238	687	~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
239		- tmp_len = 0;
240	688
241	689	/* Corresponding to the QP type, wqe process separately */
242		- if (ibqp->qp_type == IB_QPT_GSI) {
243		- ud_sq_wqe = wqe;
244		- memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
	690	+ if (ibqp->qp_type == IB_QPT_GSI)
	691	+ ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
	692	+ else if (ibqp->qp_type == IB_QPT_RC)
	693	+ ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
245	694
246		- roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
247		- V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
248		- roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
249		- V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
250		- roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
251		- V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
252		- roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
253		- V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
254		- roce_set_field(ud_sq_wqe->byte_48,
255		- V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
256		- V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
257		- ah->av.mac[4]);
258		- roce_set_field(ud_sq_wqe->byte_48,
259		- V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
260		- V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
261		- ah->av.mac[5]);
262		-
263		- /* MAC loopback */
264		- smac = (u8 *)hr_dev->dev_addr[qp->port];
265		- loopback = ether_addr_equal_unaligned(ah->av.mac,
266		- smac) ? 1 : 0;
267		-
268		- roce_set_bit(ud_sq_wqe->byte_40,
269		- V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
270		-
271		- roce_set_field(ud_sq_wqe->byte_4,
272		- V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
273		- V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
274		- HNS_ROCE_V2_WQE_OP_SEND);
275		-
276		- for (i = 0; i < wr->num_sge; i++)
277		- tmp_len += wr->sg_list[i].length;
278		-
279		- ud_sq_wqe->msg_len =
280		- cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
281		-
282		- switch (wr->opcode) {
283		- case IB_WR_SEND_WITH_IMM:
284		- case IB_WR_RDMA_WRITE_WITH_IMM:
285		- ud_sq_wqe->immtdata =
286		- cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
287		- break;
288		- default:
289		- ud_sq_wqe->immtdata = 0;
290		- break;
291		- }
292		-
293		- /* Set sig attr */
294		- roce_set_bit(ud_sq_wqe->byte_4,
295		- V2_UD_SEND_WQE_BYTE_4_CQE_S,
296		- (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
297		-
298		- /* Set se attr */
299		- roce_set_bit(ud_sq_wqe->byte_4,
300		- V2_UD_SEND_WQE_BYTE_4_SE_S,
301		- (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
302		-
303		- roce_set_bit(ud_sq_wqe->byte_4,
304		- V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
305		-
306		- roce_set_field(ud_sq_wqe->byte_16,
307		- V2_UD_SEND_WQE_BYTE_16_PD_M,
308		- V2_UD_SEND_WQE_BYTE_16_PD_S,
309		- to_hr_pd(ibqp->pd)->pdn);
310		-
311		- roce_set_field(ud_sq_wqe->byte_16,
312		- V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
313		- V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
314		- wr->num_sge);
315		-
316		- roce_set_field(ud_sq_wqe->byte_20,
317		- V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
318		- V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
319		- sge_ind & (qp->sge.sge_cnt - 1));
320		-
321		- roce_set_field(ud_sq_wqe->byte_24,
322		- V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
323		- V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
324		- ud_sq_wqe->qkey =
325		- cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
326		- qp->qkey : ud_wr(wr)->remote_qkey);
327		- roce_set_field(ud_sq_wqe->byte_32,
328		- V2_UD_SEND_WQE_BYTE_32_DQPN_M,
329		- V2_UD_SEND_WQE_BYTE_32_DQPN_S,
330		- ud_wr(wr)->remote_qpn);
331		-
332		- roce_set_field(ud_sq_wqe->byte_36,
333		- V2_UD_SEND_WQE_BYTE_36_VLAN_M,
334		- V2_UD_SEND_WQE_BYTE_36_VLAN_S,
335		- le16_to_cpu(ah->av.vlan));
336		- roce_set_field(ud_sq_wqe->byte_36,
337		- V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
338		- V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
339		- ah->av.hop_limit);
340		- roce_set_field(ud_sq_wqe->byte_36,
341		- V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
342		- V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
343		- ah->av.sl_tclass_flowlabel >>
344		- HNS_ROCE_TCLASS_SHIFT);
345		- roce_set_field(ud_sq_wqe->byte_40,
346		- V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
347		- V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,
348		- ah->av.sl_tclass_flowlabel &
349		- HNS_ROCE_FLOW_LABEL_MASK);
350		- roce_set_field(ud_sq_wqe->byte_40,
351		- V2_UD_SEND_WQE_BYTE_40_SL_M,
352		- V2_UD_SEND_WQE_BYTE_40_SL_S,
353		- le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
354		- HNS_ROCE_SL_SHIFT);
355		- roce_set_field(ud_sq_wqe->byte_40,
356		- V2_UD_SEND_WQE_BYTE_40_PORTN_M,
357		- V2_UD_SEND_WQE_BYTE_40_PORTN_S,
358		- qp->port);
359		-
360		- roce_set_field(ud_sq_wqe->byte_48,
361		- V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
362		- V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
363		- hns_get_gid_index(hr_dev, qp->phy_port,
364		- ah->av.gid_index));
365		-
366		- memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
367		- GID_LEN_V2);
368		-
369		- set_extend_sge(qp, wr, &sge_ind);
370		- ind++;
371		- } else if (ibqp->qp_type == IB_QPT_RC) {
372		- rc_sq_wqe = wqe;
373		- memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
374		- for (i = 0; i < wr->num_sge; i++)
375		- tmp_len += wr->sg_list[i].length;
376		-
377		- rc_sq_wqe->msg_len =
378		- cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
379		-
380		- switch (wr->opcode) {
381		- case IB_WR_SEND_WITH_IMM:
382		- case IB_WR_RDMA_WRITE_WITH_IMM:
383		- rc_sq_wqe->immtdata =
384		- cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
385		- break;
386		- case IB_WR_SEND_WITH_INV:
387		- rc_sq_wqe->inv_key =
388		- cpu_to_le32(wr->ex.invalidate_rkey);
389		- break;
390		- default:
391		- rc_sq_wqe->immtdata = 0;
392		- break;
393		- }
394		-
395		- roce_set_bit(rc_sq_wqe->byte_4,
396		- V2_RC_SEND_WQE_BYTE_4_FENCE_S,
397		- (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
398		-
399		- roce_set_bit(rc_sq_wqe->byte_4,
400		- V2_RC_SEND_WQE_BYTE_4_SE_S,
401		- (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
402		-
403		- roce_set_bit(rc_sq_wqe->byte_4,
404		- V2_RC_SEND_WQE_BYTE_4_CQE_S,
405		- (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
406		-
407		- roce_set_bit(rc_sq_wqe->byte_4,
408		- V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
409		-
410		- switch (wr->opcode) {
411		- case IB_WR_RDMA_READ:
412		- roce_set_field(rc_sq_wqe->byte_4,
413		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
414		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
415		- HNS_ROCE_V2_WQE_OP_RDMA_READ);
416		- rc_sq_wqe->rkey =
417		- cpu_to_le32(rdma_wr(wr)->rkey);
418		- rc_sq_wqe->va =
419		- cpu_to_le64(rdma_wr(wr)->remote_addr);
420		- break;
421		- case IB_WR_RDMA_WRITE:
422		- roce_set_field(rc_sq_wqe->byte_4,
423		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
424		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
425		- HNS_ROCE_V2_WQE_OP_RDMA_WRITE);
426		- rc_sq_wqe->rkey =
427		- cpu_to_le32(rdma_wr(wr)->rkey);
428		- rc_sq_wqe->va =
429		- cpu_to_le64(rdma_wr(wr)->remote_addr);
430		- break;
431		- case IB_WR_RDMA_WRITE_WITH_IMM:
432		- roce_set_field(rc_sq_wqe->byte_4,
433		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
434		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
435		- HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM);
436		- rc_sq_wqe->rkey =
437		- cpu_to_le32(rdma_wr(wr)->rkey);
438		- rc_sq_wqe->va =
439		- cpu_to_le64(rdma_wr(wr)->remote_addr);
440		- break;
441		- case IB_WR_SEND:
442		- roce_set_field(rc_sq_wqe->byte_4,
443		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
444		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
445		- HNS_ROCE_V2_WQE_OP_SEND);
446		- break;
447		- case IB_WR_SEND_WITH_INV:
448		- roce_set_field(rc_sq_wqe->byte_4,
449		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
450		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
451		- HNS_ROCE_V2_WQE_OP_SEND_WITH_INV);
452		- break;
453		- case IB_WR_SEND_WITH_IMM:
454		- roce_set_field(rc_sq_wqe->byte_4,
455		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
456		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
457		- HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM);
458		- break;
459		- case IB_WR_LOCAL_INV:
460		- roce_set_field(rc_sq_wqe->byte_4,
461		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
462		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
463		- HNS_ROCE_V2_WQE_OP_LOCAL_INV);
464		- break;
465		- case IB_WR_ATOMIC_CMP_AND_SWP:
466		- roce_set_field(rc_sq_wqe->byte_4,
467		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
468		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
469		- HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP);
470		- break;
471		- case IB_WR_ATOMIC_FETCH_AND_ADD:
472		- roce_set_field(rc_sq_wqe->byte_4,
473		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
474		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
475		- HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD);
476		- break;
477		- case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
478		- roce_set_field(rc_sq_wqe->byte_4,
479		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
480		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
481		- HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP);
482		- break;
483		- case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
484		- roce_set_field(rc_sq_wqe->byte_4,
485		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
486		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
487		- HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD);
488		- break;
489		- default:
490		- roce_set_field(rc_sq_wqe->byte_4,
491		- V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
492		- V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
493		- HNS_ROCE_V2_WQE_OP_MASK);
494		- break;
495		- }
496		-
497		- wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
498		-
499		- ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe, wqe,
500		- &sge_ind, bad_wr);
501		- if (ret)
502		- goto out;
503		- ind++;
504		- } else {
505		- dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
506		- spin_unlock_irqrestore(&qp->sq.lock, flags);
	695	+ if (unlikely(ret)) {
507	696	*bad_wr = wr;
508		- return -EOPNOTSUPP;
	697	+ goto out;
509	698	}
510	699	}
511	700
512	701	out:
513	702	if (likely(nreq)) {
514	703	qp->sq.head += nreq;
	704	+ qp->next_sge = sge_idx;
515	705	/* Memory barrier */
516	706	wmb();
517		-
518		- sq_db.byte_4 = 0;
519		- sq_db.parameter = 0;
520		-
521		- roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
522		- V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
523		- roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
524		- V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
525		- roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
526		- V2_DB_PARAMETER_IDX_S,
527		- qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
528		- roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
529		- V2_DB_PARAMETER_SL_S, qp->sl);
530		-
531		- hns_roce_write64_k((__le32 *)&sq_db, qp->sq.db_reg_l);
532		-
533		- qp->sq_next_wqe = ind;
534		- qp->next_sge = sge_ind;
535		-
536		- if (qp->state == IB_QPS_ERR) {
537		- attr_mask = IB_QP_STATE;
538		- attr.qp_state = IB_QPS_ERR;
539		-
540		- ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask,
541		- qp->state, IB_QPS_ERR);
542		- if (ret) {
543		- spin_unlock_irqrestore(&qp->sq.lock, flags);
544		- *bad_wr = wr;
545		- return ret;
546		- }
547		- }
	707	+ update_sq_db(hr_dev, qp);
548	708	}
549	709
550	710	spin_unlock_irqrestore(&qp->sq.lock, flags);
551	711
552	712	return ret;
	713	+}
	714	+
	715	+static int check_recv_valid(struct hns_roce_dev *hr_dev,
	716	+ struct hns_roce_qp *hr_qp)
	717	+{
	718	+ if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
	719	+ return -EIO;
	720	+ else if (hr_qp->state == IB_QPS_RESET)
	721	+ return -EINVAL;
	722	+
	723	+ return 0;
553	724	}
554	725
555	726	static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
..	..	@@ -558,44 +729,44 @@
558	729	{
559	730	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
560	731	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
	732	+ struct ib_device *ibdev = &hr_dev->ib_dev;
561	733	struct hns_roce_v2_wqe_data_seg *dseg;
562	734	struct hns_roce_rinl_sge *sge_list;
563		- struct device *dev = hr_dev->dev;
564		- struct ib_qp_attr attr;
565	735	unsigned long flags;
566	736	void *wqe = NULL;
567		- int attr_mask;
568		- int ret = 0;
	737	+ u32 wqe_idx;
569	738	int nreq;
570		- int ind;
	739	+ int ret;
571	740	int i;
572	741
573	742	spin_lock_irqsave(&hr_qp->rq.lock, flags);
574		- ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
575	743
576		- if (hr_qp->state == IB_QPS_RESET) {
577		- spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
	744	+ ret = check_recv_valid(hr_dev, hr_qp);
	745	+ if (unlikely(ret)) {
578	746	*bad_wr = wr;
579		- return -EINVAL;
	747	+ nreq = 0;
	748	+ goto out;
580	749	}
581	750
582	751	for (nreq = 0; wr; ++nreq, wr = wr->next) {
583		- if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
584		- hr_qp->ibqp.recv_cq)) {
	752	+ if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,
	753	+ hr_qp->ibqp.recv_cq))) {
585	754	ret = -ENOMEM;
586	755	*bad_wr = wr;
587	756	goto out;
588	757	}
589	758
	759	+ wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
	760	+
590	761	if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
591		- dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
592		- wr->num_sge, hr_qp->rq.max_gs);
	762	+ ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n",
	763	+ wr->num_sge, hr_qp->rq.max_gs);
593	764	ret = -EINVAL;
594	765	*bad_wr = wr;
595	766	goto out;
596	767	}
597	768
598		- wqe = get_recv_wqe(hr_qp, ind);
	769	+ wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
599	770	dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
600	771	for (i = 0; i < wr->num_sge; i++) {
601	772	if (!wr->sg_list[i].length)
..	..	@@ -604,15 +775,15 @@
604	775	dseg++;
605	776	}
606	777
607		- if (i < hr_qp->rq.max_gs) {
	778	+ if (wr->num_sge < hr_qp->rq.max_gs) {
608	779	dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
609	780	dseg->addr = 0;
610	781	}
611	782
612	783	/* rq support inline data */
613		- if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
614		- sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
615		- hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt =
	784	+ if (hr_qp->rq_inl_buf.wqe_cnt) {
	785	+ sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
	786	+ hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt =
616	787	(u32)wr->num_sge;
617	788	for (i = 0; i < wr->num_sge; i++) {
618	789	sge_list[i].addr =
..	..	@@ -621,9 +792,7 @@
621	792	}
622	793	}
623	794
624		- hr_qp->rq.wrid[ind] = wr->wr_id;
625		-
626		- ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
	795	+ hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
627	796	}
628	797
629	798	out:
..	..	@@ -632,25 +801,265 @@
632	801	/* Memory barrier */
633	802	wmb();
634	803
635		- *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
636		-
	804	+ /*
	805	+ * Hip08 hardware cannot flush the WQEs in RQ if the QP state
	806	+ * gets into errored mode. Hence, as a workaround to this
	807	+ * hardware limitation, driver needs to assist in flushing. But
	808	+ * the flushing operation uses mailbox to convey the QP state to
	809	+ * the hardware and which can sleep due to the mutex protection
	810	+ * around the mailbox calls. Hence, use the deferred flush for
	811	+ * now.
	812	+ */
637	813	if (hr_qp->state == IB_QPS_ERR) {
638		- attr_mask = IB_QP_STATE;
639		- attr.qp_state = IB_QPS_ERR;
640		-
641		- ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr,
642		- attr_mask, hr_qp->state,
643		- IB_QPS_ERR);
644		- if (ret) {
645		- spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
646		- *bad_wr = wr;
647		- return ret;
648		- }
	814	+ if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG,
	815	+ &hr_qp->flush_flag))
	816	+ init_flush_work(hr_dev, hr_qp);
	817	+ } else {
	818	+ *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
649	819	}
650	820	}
651	821	spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
652	822
653	823	return ret;
	824	+}
	825	+
	826	+static void get_srq_wqe(struct hns_roce_srq srq, int n)
	827	+{
	828	+ return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
	829	+}
	830	+
	831	+static void get_idx_buf(struct hns_roce_idx_que idx_que, int n)
	832	+{
	833	+ return hns_roce_buf_offset(idx_que->mtr.kmem,
	834	+ n << idx_que->entry_shift);
	835	+}
	836	+
	837	+static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
	838	+{
	839	+ /* always called with interrupts disabled. */
	840	+ spin_lock(&srq->lock);
	841	+
	842	+ bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
	843	+ srq->tail++;
	844	+
	845	+ spin_unlock(&srq->lock);
	846	+}
	847	+
	848	+static int find_empty_entry(struct hns_roce_idx_que *idx_que,
	849	+ unsigned long size)
	850	+{
	851	+ int wqe_idx;
	852	+
	853	+ if (unlikely(bitmap_full(idx_que->bitmap, size)))
	854	+ return -ENOSPC;
	855	+
	856	+ wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
	857	+
	858	+ bitmap_set(idx_que->bitmap, wqe_idx, 1);
	859	+
	860	+ return wqe_idx;
	861	+}
	862	+
	863	+static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
	864	+ const struct ib_recv_wr *wr,
	865	+ const struct ib_recv_wr **bad_wr)
	866	+{
	867	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	868	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	869	+ struct hns_roce_v2_wqe_data_seg *dseg;
	870	+ struct hns_roce_v2_db srq_db;
	871	+ unsigned long flags;
	872	+ __le32 *srq_idx;
	873	+ int ret = 0;
	874	+ int wqe_idx;
	875	+ void *wqe;
	876	+ int nreq;
	877	+ int ind;
	878	+ int i;
	879	+
	880	+ spin_lock_irqsave(&srq->lock, flags);
	881	+
	882	+ ind = srq->head & (srq->wqe_cnt - 1);
	883	+
	884	+ for (nreq = 0; wr; ++nreq, wr = wr->next) {
	885	+ if (unlikely(wr->num_sge >= srq->max_gs)) {
	886	+ ret = -EINVAL;
	887	+ *bad_wr = wr;
	888	+ break;
	889	+ }
	890	+
	891	+ if (unlikely(srq->head == srq->tail)) {
	892	+ ret = -ENOMEM;
	893	+ *bad_wr = wr;
	894	+ break;
	895	+ }
	896	+
	897	+ wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt);
	898	+ if (unlikely(wqe_idx < 0)) {
	899	+ ret = -ENOMEM;
	900	+ *bad_wr = wr;
	901	+ break;
	902	+ }
	903	+
	904	+ wqe = get_srq_wqe(srq, wqe_idx);
	905	+ dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
	906	+
	907	+ for (i = 0; i < wr->num_sge; ++i) {
	908	+ dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
	909	+ dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
	910	+ dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
	911	+ }
	912	+
	913	+ if (wr->num_sge < srq->max_gs) {
	914	+ dseg[i].len = 0;
	915	+ dseg[i].lkey = cpu_to_le32(0x100);
	916	+ dseg[i].addr = 0;
	917	+ }
	918	+
	919	+ srq_idx = get_idx_buf(&srq->idx_que, ind);
	920	+ *srq_idx = cpu_to_le32(wqe_idx);
	921	+
	922	+ srq->wrid[wqe_idx] = wr->wr_id;
	923	+ ind = (ind + 1) & (srq->wqe_cnt - 1);
	924	+ }
	925	+
	926	+ if (likely(nreq)) {
	927	+ srq->head += nreq;
	928	+
	929	+ /*
	930	+ * Make sure that descriptors are written before
	931	+ * doorbell record.
	932	+ */
	933	+ wmb();
	934	+
	935	+ srq_db.byte_4 =
	936	+ cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S \|
	937	+ (srq->srqn & V2_DB_BYTE_4_TAG_M));
	938	+ srq_db.parameter =
	939	+ cpu_to_le32(srq->head & V2_DB_PARAMETER_IDX_M);
	940	+
	941	+ hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
	942	+ }
	943	+
	944	+ spin_unlock_irqrestore(&srq->lock, flags);
	945	+
	946	+ return ret;
	947	+}
	948	+
	949	+static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
	950	+ unsigned long instance_stage,
	951	+ unsigned long reset_stage)
	952	+{
	953	+ /* When hardware reset has been completed once or more, we should stop
	954	+ * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
	955	+ * function, we should exit with error. If now at HNAE3_INIT_CLIENT
	956	+ * stage of soft reset process, we should exit with error, and then
	957	+ * HNAE3_INIT_CLIENT related process can rollback the operation like
	958	+ * notifing hardware to free resources, HNAE3_INIT_CLIENT related
	959	+ * process will exit with error to notify NIC driver to reschedule soft
	960	+ * reset process once again.
	961	+ */
	962	+ hr_dev->is_reset = true;
	963	+ hr_dev->dis_db = true;
	964	+
	965	+ if (reset_stage == HNS_ROCE_STATE_RST_INIT \|\|
	966	+ instance_stage == HNS_ROCE_STATE_INIT)
	967	+ return CMD_RST_PRC_EBUSY;
	968	+
	969	+ return CMD_RST_PRC_SUCCESS;
	970	+}
	971	+
	972	+static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
	973	+ unsigned long instance_stage,
	974	+ unsigned long reset_stage)
	975	+{
	976	+#define HW_RESET_TIMEOUT_US 1000000
	977	+#define HW_RESET_SLEEP_US 1000
	978	+
	979	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	980	+ struct hnae3_handle *handle = priv->handle;
	981	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	982	+ unsigned long val;
	983	+ int ret;
	984	+
	985	+ /* When hardware reset is detected, we should stop sending mailbox&cmq&
	986	+ * doorbell to hardware. If now in .init_instance() function, we should
	987	+ * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
	988	+ * process, we should exit with error, and then HNAE3_INIT_CLIENT
	989	+ * related process can rollback the operation like notifing hardware to
	990	+ * free resources, HNAE3_INIT_CLIENT related process will exit with
	991	+ * error to notify NIC driver to reschedule soft reset process once
	992	+ * again.
	993	+ */
	994	+ hr_dev->dis_db = true;
	995	+
	996	+ ret = read_poll_timeout(ops->ae_dev_reset_cnt, val,
	997	+ val > hr_dev->reset_cnt, HW_RESET_SLEEP_US,
	998	+ HW_RESET_TIMEOUT_US, false, handle);
	999	+ if (!ret)
	1000	+ hr_dev->is_reset = true;
	1001	+
	1002	+ if (!hr_dev->is_reset \|\| reset_stage == HNS_ROCE_STATE_RST_INIT \|\|
	1003	+ instance_stage == HNS_ROCE_STATE_INIT)
	1004	+ return CMD_RST_PRC_EBUSY;
	1005	+
	1006	+ return CMD_RST_PRC_SUCCESS;
	1007	+}
	1008	+
	1009	+static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
	1010	+{
	1011	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	1012	+ struct hnae3_handle *handle = priv->handle;
	1013	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	1014	+
	1015	+ /* When software reset is detected at .init_instance() function, we
	1016	+ * should stop sending mailbox&cmq&doorbell to hardware, and exit
	1017	+ * with error.
	1018	+ */
	1019	+ hr_dev->dis_db = true;
	1020	+ if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
	1021	+ hr_dev->is_reset = true;
	1022	+
	1023	+ return CMD_RST_PRC_EBUSY;
	1024	+}
	1025	+
	1026	+static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
	1027	+{
	1028	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	1029	+ struct hnae3_handle *handle = priv->handle;
	1030	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	1031	+ unsigned long instance_stage; /* the current instance stage */
	1032	+ unsigned long reset_stage; /* the current reset stage */
	1033	+ unsigned long reset_cnt;
	1034	+ bool sw_resetting;
	1035	+ bool hw_resetting;
	1036	+
	1037	+ if (hr_dev->is_reset)
	1038	+ return CMD_RST_PRC_SUCCESS;
	1039	+
	1040	+ /* Get information about reset from NIC driver or RoCE driver itself,
	1041	+ * the meaning of the following variables from NIC driver are described
	1042	+ * as below:
	1043	+ * reset_cnt -- The count value of completed hardware reset.
	1044	+ * hw_resetting -- Whether hardware device is resetting now.
	1045	+ * sw_resetting -- Whether NIC's software reset process is running now.
	1046	+ */
	1047	+ instance_stage = handle->rinfo.instance_state;
	1048	+ reset_stage = handle->rinfo.reset_state;
	1049	+ reset_cnt = ops->ae_dev_reset_cnt(handle);
	1050	+ hw_resetting = ops->get_cmdq_stat(handle);
	1051	+ sw_resetting = ops->ae_dev_resetting(handle);
	1052	+
	1053	+ if (reset_cnt != hr_dev->reset_cnt)
	1054	+ return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
	1055	+ reset_stage);
	1056	+ else if (hw_resetting)
	1057	+ return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
	1058	+ reset_stage);
	1059	+ else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
	1060	+ return hns_roce_v2_cmd_sw_resetting(hr_dev);
	1061	+
	1062	+ return 0;
654	1063	}
655	1064
656	1065	static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
..	..	@@ -696,7 +1105,7 @@
696	1105
697	1106	static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
698	1107	{
699		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1108	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
700	1109	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
701	1110	&priv->cmq.csq : &priv->cmq.crq;
702	1111
..	..	@@ -709,7 +1118,7 @@
709	1118
710	1119	static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
711	1120	{
712		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1121	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
713	1122	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
714	1123	&priv->cmq.csq : &priv->cmq.crq;
715	1124	dma_addr_t dma = ring->desc_dma_addr;
..	..	@@ -719,8 +1128,7 @@
719	1128	roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
720	1129	upper_32_bits(dma));
721	1130	roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
722		- (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) \|
723		- HNS_ROCE_CMQ_ENABLE);
	1131	+ ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
724	1132	roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
725	1133	roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
726	1134	} else {
..	..	@@ -728,8 +1136,7 @@
728	1136	roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
729	1137	upper_32_bits(dma));
730	1138	roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
731		- (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) \|
732		- HNS_ROCE_CMQ_ENABLE);
	1139	+ ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
733	1140	roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
734	1141	roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
735	1142	}
..	..	@@ -737,7 +1144,7 @@
737	1144
738	1145	static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
739	1146	{
740		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1147	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
741	1148	int ret;
742	1149
743	1150	/* Setup the queue entries for command queue */
..	..	@@ -781,7 +1188,7 @@
781	1188
782	1189	static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
783	1190	{
784		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1191	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
785	1192
786	1193	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
787	1194	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
..	..	@@ -803,15 +1210,15 @@
803	1210
804	1211	static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
805	1212	{
806		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
807	1213	u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
	1214	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
808	1215
809	1216	return head == priv->cmq.csq.next_to_use;
810	1217	}
811	1218
812	1219	static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
813	1220	{
814		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1221	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
815	1222	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
816	1223	struct hns_roce_cmq_desc *desc;
817	1224	u16 ntc = csq->next_to_clean;
..	..	@@ -833,21 +1240,18 @@
833	1240	return clean;
834	1241	}
835	1242
836		-static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
837		- struct hns_roce_cmq_desc *desc, int num)
	1243	+static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
	1244	+ struct hns_roce_cmq_desc *desc, int num)
838	1245	{
839		- struct hns_roce_v2_priv priv = (struct hns_roce_v2_priv )hr_dev->priv;
	1246	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
840	1247	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
841	1248	struct hns_roce_cmq_desc *desc_to_use;
842	1249	bool complete = false;
843	1250	u32 timeout = 0;
844	1251	int handle = 0;
845	1252	u16 desc_ret;
846		- int ret = 0;
	1253	+ int ret;
847	1254	int ntc;
848		-
849		- if (hr_dev->is_reset)
850		- return 0;
851	1255
852	1256	spin_lock_bh(&csq->lock);
853	1257
..	..	@@ -879,7 +1283,7 @@
879	1283	* If the command is sync, wait for the firmware to write back,
880	1284	* if multi descriptors to be sent, use the first one to check
881	1285	*/
882		- if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
	1286	+ if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
883	1287	do {
884	1288	if (hns_roce_cmq_csq_done(hr_dev))
885	1289	break;
..	..	@@ -891,15 +1295,14 @@
891	1295	if (hns_roce_cmq_csq_done(hr_dev)) {
892	1296	complete = true;
893	1297	handle = 0;
	1298	+ ret = 0;
894	1299	while (handle < num) {
895	1300	/* get the result of hardware write back */
896	1301	desc_to_use = &csq->desc[ntc];
897	1302	desc[handle] = *desc_to_use;
898	1303	dev_dbg(hr_dev->dev, "Get cmq desc:\n");
899		- desc_ret = desc[handle].retval;
900		- if (desc_ret == CMD_EXEC_SUCCESS)
901		- ret = 0;
902		- else
	1304	+ desc_ret = le16_to_cpu(desc[handle].retval);
	1305	+ if (unlikely(desc_ret != CMD_EXEC_SUCCESS))
903	1306	ret = -EIO;
904	1307	priv->cmq.last_status = desc_ret;
905	1308	ntc++;
..	..	@@ -923,6 +1326,30 @@
923	1326	return ret;
924	1327	}
925	1328
	1329	+static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
	1330	+ struct hns_roce_cmq_desc *desc, int num)
	1331	+{
	1332	+ int retval;
	1333	+ int ret;
	1334	+
	1335	+ ret = hns_roce_v2_rst_process_cmd(hr_dev);
	1336	+ if (ret == CMD_RST_PRC_SUCCESS)
	1337	+ return 0;
	1338	+ if (ret == CMD_RST_PRC_EBUSY)
	1339	+ return -EBUSY;
	1340	+
	1341	+ ret = __hns_roce_cmq_send(hr_dev, desc, num);
	1342	+ if (ret) {
	1343	+ retval = hns_roce_v2_rst_process_cmd(hr_dev);
	1344	+ if (retval == CMD_RST_PRC_SUCCESS)
	1345	+ return 0;
	1346	+ else if (retval == CMD_RST_PRC_EBUSY)
	1347	+ return -EBUSY;
	1348	+ }
	1349	+
	1350	+ return ret;
	1351	+}
	1352	+
926	1353	static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
927	1354	{
928	1355	struct hns_roce_query_version *resp;
..	..	@@ -935,8 +1362,157 @@
935	1362	return ret;
936	1363
937	1364	resp = (struct hns_roce_query_version *)desc.data;
938		- hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version);
939		- hr_dev->vendor_id = le32_to_cpu(resp->rocee_vendor_id);
	1365	+ hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
	1366	+ hr_dev->vendor_id = hr_dev->pci_dev->vendor;
	1367	+
	1368	+ return 0;
	1369	+}
	1370	+
	1371	+static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
	1372	+{
	1373	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	1374	+ struct hnae3_handle *handle = priv->handle;
	1375	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	1376	+ unsigned long reset_cnt;
	1377	+ bool sw_resetting;
	1378	+ bool hw_resetting;
	1379	+
	1380	+ reset_cnt = ops->ae_dev_reset_cnt(handle);
	1381	+ hw_resetting = ops->get_hw_reset_stat(handle);
	1382	+ sw_resetting = ops->ae_dev_resetting(handle);
	1383	+
	1384	+ if (reset_cnt != hr_dev->reset_cnt \|\| hw_resetting \|\| sw_resetting)
	1385	+ return true;
	1386	+
	1387	+ return false;
	1388	+}
	1389	+
	1390	+static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval,
	1391	+ int flag)
	1392	+{
	1393	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
	1394	+ struct hnae3_handle *handle = priv->handle;
	1395	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	1396	+ unsigned long instance_stage;
	1397	+ unsigned long reset_cnt;
	1398	+ unsigned long end;
	1399	+ bool sw_resetting;
	1400	+ bool hw_resetting;
	1401	+
	1402	+ instance_stage = handle->rinfo.instance_state;
	1403	+ reset_cnt = ops->ae_dev_reset_cnt(handle);
	1404	+ hw_resetting = ops->get_hw_reset_stat(handle);
	1405	+ sw_resetting = ops->ae_dev_resetting(handle);
	1406	+
	1407	+ if (reset_cnt != hr_dev->reset_cnt) {
	1408	+ hr_dev->dis_db = true;
	1409	+ hr_dev->is_reset = true;
	1410	+ dev_info(hr_dev->dev, "Func clear success after reset.\n");
	1411	+ } else if (hw_resetting) {
	1412	+ hr_dev->dis_db = true;
	1413	+
	1414	+ dev_warn(hr_dev->dev,
	1415	+ "Func clear is pending, device in resetting state.\n");
	1416	+ end = HNS_ROCE_V2_HW_RST_TIMEOUT;
	1417	+ while (end) {
	1418	+ if (!ops->get_hw_reset_stat(handle)) {
	1419	+ hr_dev->is_reset = true;
	1420	+ dev_info(hr_dev->dev,
	1421	+ "Func clear success after reset.\n");
	1422	+ return;
	1423	+ }
	1424	+ msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
	1425	+ end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
	1426	+ }
	1427	+
	1428	+ dev_warn(hr_dev->dev, "Func clear failed.\n");
	1429	+ } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) {
	1430	+ hr_dev->dis_db = true;
	1431	+
	1432	+ dev_warn(hr_dev->dev,
	1433	+ "Func clear is pending, device in resetting state.\n");
	1434	+ end = HNS_ROCE_V2_HW_RST_TIMEOUT;
	1435	+ while (end) {
	1436	+ if (ops->ae_dev_reset_cnt(handle) !=
	1437	+ hr_dev->reset_cnt) {
	1438	+ hr_dev->is_reset = true;
	1439	+ dev_info(hr_dev->dev,
	1440	+ "Func clear success after sw reset\n");
	1441	+ return;
	1442	+ }
	1443	+ msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
	1444	+ end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
	1445	+ }
	1446	+
	1447	+ dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n");
	1448	+ } else {
	1449	+ if (retval && !flag)
	1450	+ dev_warn(hr_dev->dev,
	1451	+ "Func clear read failed, ret = %d.\n", retval);
	1452	+
	1453	+ dev_warn(hr_dev->dev, "Func clear failed.\n");
	1454	+ }
	1455	+}
	1456	+static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
	1457	+{
	1458	+ bool fclr_write_fail_flag = false;
	1459	+ struct hns_roce_func_clear *resp;
	1460	+ struct hns_roce_cmq_desc desc;
	1461	+ unsigned long end;
	1462	+ int ret = 0;
	1463	+
	1464	+ if (hns_roce_func_clr_chk_rst(hr_dev))
	1465	+ goto out;
	1466	+
	1467	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
	1468	+ resp = (struct hns_roce_func_clear *)desc.data;
	1469	+
	1470	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1471	+ if (ret) {
	1472	+ fclr_write_fail_flag = true;
	1473	+ dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
	1474	+ ret);
	1475	+ goto out;
	1476	+ }
	1477	+
	1478	+ msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
	1479	+ end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
	1480	+ while (end) {
	1481	+ if (hns_roce_func_clr_chk_rst(hr_dev))
	1482	+ goto out;
	1483	+ msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
	1484	+ end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
	1485	+
	1486	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
	1487	+ true);
	1488	+
	1489	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1490	+ if (ret)
	1491	+ continue;
	1492	+
	1493	+ if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
	1494	+ hr_dev->is_reset = true;
	1495	+ return;
	1496	+ }
	1497	+ }
	1498	+
	1499	+out:
	1500	+ hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag);
	1501	+}
	1502	+
	1503	+static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
	1504	+{
	1505	+ struct hns_roce_query_fw_info *resp;
	1506	+ struct hns_roce_cmq_desc desc;
	1507	+ int ret;
	1508	+
	1509	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
	1510	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1511	+ if (ret)
	1512	+ return ret;
	1513	+
	1514	+ resp = (struct hns_roce_query_fw_info *)desc.data;
	1515	+ hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
940	1516
941	1517	return 0;
942	1518	}
..	..	@@ -1002,8 +1578,63 @@
1002	1578	hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1003	1579	PF_RES_DATA_3_PF_SL_NUM_M,
1004	1580	PF_RES_DATA_3_PF_SL_NUM_S);
	1581	+ hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
	1582	+ PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
	1583	+ PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
1005	1584
1006	1585	return 0;
	1586	+}
	1587	+
	1588	+static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
	1589	+{
	1590	+ struct hns_roce_pf_timer_res_a *req_a;
	1591	+ struct hns_roce_cmq_desc desc;
	1592	+ int ret;
	1593	+
	1594	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
	1595	+ true);
	1596	+
	1597	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1598	+ if (ret)
	1599	+ return ret;
	1600	+
	1601	+ req_a = (struct hns_roce_pf_timer_res_a *)desc.data;
	1602	+
	1603	+ hr_dev->caps.qpc_timer_bt_num =
	1604	+ roce_get_field(req_a->qpc_timer_bt_idx_num,
	1605	+ PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
	1606	+ PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
	1607	+ hr_dev->caps.cqc_timer_bt_num =
	1608	+ roce_get_field(req_a->cqc_timer_bt_idx_num,
	1609	+ PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
	1610	+ PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
	1611	+
	1612	+ return 0;
	1613	+}
	1614	+
	1615	+static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id)
	1616	+{
	1617	+ struct hns_roce_cmq_desc desc;
	1618	+ struct hns_roce_vf_switch *swt;
	1619	+ int ret;
	1620	+
	1621	+ swt = (struct hns_roce_vf_switch *)desc.data;
	1622	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
	1623	+ swt->rocee_sel \|= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
	1624	+ roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M,
	1625	+ VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id);
	1626	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	1627	+ if (ret)
	1628	+ return ret;
	1629	+
	1630	+ desc.flag =
	1631	+ cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR \| HNS_ROCE_CMD_FLAG_IN);
	1632	+ desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
	1633	+ roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
	1634	+ roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0);
	1635	+ roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
	1636	+
	1637	+ return hns_roce_cmq_send(hr_dev, &desc, 1);
1007	1638	}
1008	1639
1009	1640	static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
..	..	@@ -1015,8 +1646,6 @@
1015	1646
1016	1647	req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1017	1648	req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1018		- memset(req_a, 0, sizeof(*req_a));
1019		- memset(req_b, 0, sizeof(*req_b));
1020	1649	for (i = 0; i < 2; i++) {
1021	1650	hns_roce_cmq_setup_basic_desc(&desc[i],
1022	1651	HNS_ROCE_OPC_ALLOC_VF_RES, false);
..	..	@@ -1025,73 +1654,62 @@
1025	1654	desc[i].flag \|= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1026	1655	else
1027	1656	desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1028		-
1029		- if (i == 0) {
1030		- roce_set_field(req_a->vf_qpc_bt_idx_num,
1031		- VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1032		- VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1033		- roce_set_field(req_a->vf_qpc_bt_idx_num,
1034		- VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1035		- VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
1036		- HNS_ROCE_VF_QPC_BT_NUM);
1037		-
1038		- roce_set_field(req_a->vf_srqc_bt_idx_num,
1039		- VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1040		- VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1041		- roce_set_field(req_a->vf_srqc_bt_idx_num,
1042		- VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1043		- VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1044		- HNS_ROCE_VF_SRQC_BT_NUM);
1045		-
1046		- roce_set_field(req_a->vf_cqc_bt_idx_num,
1047		- VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1048		- VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1049		- roce_set_field(req_a->vf_cqc_bt_idx_num,
1050		- VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1051		- VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
1052		- HNS_ROCE_VF_CQC_BT_NUM);
1053		-
1054		- roce_set_field(req_a->vf_mpt_bt_idx_num,
1055		- VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1056		- VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1057		- roce_set_field(req_a->vf_mpt_bt_idx_num,
1058		- VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1059		- VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
1060		- HNS_ROCE_VF_MPT_BT_NUM);
1061		-
1062		- roce_set_field(req_a->vf_eqc_bt_idx_num,
1063		- VF_RES_A_DATA_5_VF_EQC_IDX_M,
1064		- VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1065		- roce_set_field(req_a->vf_eqc_bt_idx_num,
1066		- VF_RES_A_DATA_5_VF_EQC_NUM_M,
1067		- VF_RES_A_DATA_5_VF_EQC_NUM_S,
1068		- HNS_ROCE_VF_EQC_NUM);
1069		- } else {
1070		- roce_set_field(req_b->vf_smac_idx_num,
1071		- VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1072		- VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1073		- roce_set_field(req_b->vf_smac_idx_num,
1074		- VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1075		- VF_RES_B_DATA_1_VF_SMAC_NUM_S,
1076		- HNS_ROCE_VF_SMAC_NUM);
1077		-
1078		- roce_set_field(req_b->vf_sgid_idx_num,
1079		- VF_RES_B_DATA_2_VF_SGID_IDX_M,
1080		- VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1081		- roce_set_field(req_b->vf_sgid_idx_num,
1082		- VF_RES_B_DATA_2_VF_SGID_NUM_M,
1083		- VF_RES_B_DATA_2_VF_SGID_NUM_S,
1084		- HNS_ROCE_VF_SGID_NUM);
1085		-
1086		- roce_set_field(req_b->vf_qid_idx_sl_num,
1087		- VF_RES_B_DATA_3_VF_QID_IDX_M,
1088		- VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1089		- roce_set_field(req_b->vf_qid_idx_sl_num,
1090		- VF_RES_B_DATA_3_VF_SL_NUM_M,
1091		- VF_RES_B_DATA_3_VF_SL_NUM_S,
1092		- HNS_ROCE_VF_SL_NUM);
1093		- }
1094	1657	}
	1658	+
	1659	+ roce_set_field(req_a->vf_qpc_bt_idx_num,
	1660	+ VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
	1661	+ VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
	1662	+ roce_set_field(req_a->vf_qpc_bt_idx_num,
	1663	+ VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
	1664	+ VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, HNS_ROCE_VF_QPC_BT_NUM);
	1665	+
	1666	+ roce_set_field(req_a->vf_srqc_bt_idx_num,
	1667	+ VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
	1668	+ VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
	1669	+ roce_set_field(req_a->vf_srqc_bt_idx_num,
	1670	+ VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
	1671	+ VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
	1672	+ HNS_ROCE_VF_SRQC_BT_NUM);
	1673	+
	1674	+ roce_set_field(req_a->vf_cqc_bt_idx_num,
	1675	+ VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
	1676	+ VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
	1677	+ roce_set_field(req_a->vf_cqc_bt_idx_num,
	1678	+ VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
	1679	+ VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, HNS_ROCE_VF_CQC_BT_NUM);
	1680	+
	1681	+ roce_set_field(req_a->vf_mpt_bt_idx_num,
	1682	+ VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
	1683	+ VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
	1684	+ roce_set_field(req_a->vf_mpt_bt_idx_num,
	1685	+ VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
	1686	+ VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, HNS_ROCE_VF_MPT_BT_NUM);
	1687	+
	1688	+ roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_IDX_M,
	1689	+ VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
	1690	+ roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_NUM_M,
	1691	+ VF_RES_A_DATA_5_VF_EQC_NUM_S, HNS_ROCE_VF_EQC_NUM);
	1692	+
	1693	+ roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_IDX_M,
	1694	+ VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
	1695	+ roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_NUM_M,
	1696	+ VF_RES_B_DATA_1_VF_SMAC_NUM_S, HNS_ROCE_VF_SMAC_NUM);
	1697	+
	1698	+ roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_IDX_M,
	1699	+ VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
	1700	+ roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_NUM_M,
	1701	+ VF_RES_B_DATA_2_VF_SGID_NUM_S, HNS_ROCE_VF_SGID_NUM);
	1702	+
	1703	+ roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_QID_IDX_M,
	1704	+ VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
	1705	+ roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_SL_NUM_M,
	1706	+ VF_RES_B_DATA_3_VF_SL_NUM_S, HNS_ROCE_VF_SL_NUM);
	1707	+
	1708	+ roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
	1709	+ VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
	1710	+ roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
	1711	+ VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
	1712	+ HNS_ROCE_VF_SCCC_BT_NUM);
1095	1713
1096	1714	return hns_roce_cmq_send(hr_dev, desc, 2);
1097	1715	}
..	..	@@ -1102,6 +1720,7 @@
1102	1720	u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1103	1721	u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1104	1722	u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
	1723	+ u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
1105	1724	struct hns_roce_cfg_bt_attr *req;
1106	1725	struct hns_roce_cmq_desc desc;
1107	1726
..	..	@@ -1149,7 +1768,458 @@
1149	1768	CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1150	1769	mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1151	1770
	1771	+ roce_set_field(req->vf_sccc_cfg,
	1772	+ CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
	1773	+ CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
	1774	+ hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
	1775	+ roce_set_field(req->vf_sccc_cfg,
	1776	+ CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
	1777	+ CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
	1778	+ hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
	1779	+ roce_set_field(req->vf_sccc_cfg,
	1780	+ CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
	1781	+ CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
	1782	+ sccc_hop_num ==
	1783	+ HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);
	1784	+
1152	1785	return hns_roce_cmq_send(hr_dev, &desc, 1);
	1786	+}
	1787	+
	1788	+static void set_default_caps(struct hns_roce_dev *hr_dev)
	1789	+{
	1790	+ struct hns_roce_caps *caps = &hr_dev->caps;
	1791	+
	1792	+ caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM;
	1793	+ caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM;
	1794	+ caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM;
	1795	+ caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM;
	1796	+ caps->min_cqes = HNS_ROCE_MIN_CQE_NUM;
	1797	+ caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM;
	1798	+ caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
	1799	+ caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
	1800	+ caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
	1801	+ caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
	1802	+ caps->num_uars = HNS_ROCE_V2_UAR_NUM;
	1803	+ caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM;
	1804	+ caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM;
	1805	+ caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM;
	1806	+ caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
	1807	+ caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM;
	1808	+ caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
	1809	+ caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
	1810	+ caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
	1811	+ caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
	1812	+ caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM;
	1813	+ caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
	1814	+ caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
	1815	+ caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
	1816	+ caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
	1817	+ caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
	1818	+ caps->qpc_sz = HNS_ROCE_V2_QPC_SZ;
	1819	+ caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ;
	1820	+ caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ;
	1821	+ caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ;
	1822	+ caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ;
	1823	+ caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ;
	1824	+ caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
	1825	+ caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ;
	1826	+ caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE;
	1827	+ caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
	1828	+ caps->reserved_lkey = 0;
	1829	+ caps->reserved_pds = 0;
	1830	+ caps->reserved_mrws = 1;
	1831	+ caps->reserved_uars = 0;
	1832	+ caps->reserved_cqs = 0;
	1833	+ caps->reserved_srqs = 0;
	1834	+ caps->reserved_qps = HNS_ROCE_V2_RSV_QPS;
	1835	+
	1836	+ caps->qpc_ba_pg_sz = 0;
	1837	+ caps->qpc_buf_pg_sz = 0;
	1838	+ caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
	1839	+ caps->srqc_ba_pg_sz = 0;
	1840	+ caps->srqc_buf_pg_sz = 0;
	1841	+ caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
	1842	+ caps->cqc_ba_pg_sz = 0;
	1843	+ caps->cqc_buf_pg_sz = 0;
	1844	+ caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
	1845	+ caps->mpt_ba_pg_sz = 0;
	1846	+ caps->mpt_buf_pg_sz = 0;
	1847	+ caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
	1848	+ caps->mtt_ba_pg_sz = 0;
	1849	+ caps->mtt_buf_pg_sz = 0;
	1850	+ caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
	1851	+ caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM;
	1852	+ caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM;
	1853	+ caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM;
	1854	+ caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;
	1855	+ caps->cqe_buf_pg_sz = 0;
	1856	+ caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
	1857	+ caps->srqwqe_ba_pg_sz = 0;
	1858	+ caps->srqwqe_buf_pg_sz = 0;
	1859	+ caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM;
	1860	+ caps->idx_ba_pg_sz = 0;
	1861	+ caps->idx_buf_pg_sz = 0;
	1862	+ caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM;
	1863	+ caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
	1864	+
	1865	+ caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR \|
	1866	+ HNS_ROCE_CAP_FLAG_ROCE_V1_V2 \|
	1867	+ HNS_ROCE_CAP_FLAG_RECORD_DB \|
	1868	+ HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
	1869	+
	1870	+ caps->pkey_table_len[0] = 1;
	1871	+ caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
	1872	+ caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
	1873	+ caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM;
	1874	+ caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
	1875	+ caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
	1876	+ caps->local_ca_ack_delay = 0;
	1877	+ caps->max_mtu = IB_MTU_4096;
	1878	+
	1879	+ caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;
	1880	+ caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;
	1881	+
	1882	+ caps->flags \|= HNS_ROCE_CAP_FLAG_ATOMIC \| HNS_ROCE_CAP_FLAG_MW \|
	1883	+ HNS_ROCE_CAP_FLAG_SRQ \| HNS_ROCE_CAP_FLAG_FRMR \|
	1884	+ HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
	1885	+
	1886	+ caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
	1887	+ caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
	1888	+ caps->qpc_timer_ba_pg_sz = 0;
	1889	+ caps->qpc_timer_buf_pg_sz = 0;
	1890	+ caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
	1891	+ caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
	1892	+ caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
	1893	+ caps->cqc_timer_ba_pg_sz = 0;
	1894	+ caps->cqc_timer_buf_pg_sz = 0;
	1895	+ caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
	1896	+
	1897	+ caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ;
	1898	+ caps->sccc_ba_pg_sz = 0;
	1899	+ caps->sccc_buf_pg_sz = 0;
	1900	+ caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
	1901	+
	1902	+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
	1903	+ caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
	1904	+ caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
	1905	+ caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
	1906	+ caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
	1907	+ }
	1908	+}
	1909	+
	1910	+static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num,
	1911	+ int buf_page_size, int bt_page_size, u32 hem_type)
	1912	+{
	1913	+ u64 obj_per_chunk;
	1914	+ u64 bt_chunk_size = PAGE_SIZE;
	1915	+ u64 buf_chunk_size = PAGE_SIZE;
	1916	+ u64 obj_per_chunk_default = buf_chunk_size / obj_size;
	1917	+
	1918	+ *buf_page_size = 0;
	1919	+ *bt_page_size = 0;
	1920	+
	1921	+ switch (hop_num) {
	1922	+ case 3:
	1923	+ obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
	1924	+ (bt_chunk_size / BA_BYTE_LEN) *
	1925	+ (bt_chunk_size / BA_BYTE_LEN) *
	1926	+ obj_per_chunk_default;
	1927	+ break;
	1928	+ case 2:
	1929	+ obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
	1930	+ (bt_chunk_size / BA_BYTE_LEN) *
	1931	+ obj_per_chunk_default;
	1932	+ break;
	1933	+ case 1:
	1934	+ obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
	1935	+ obj_per_chunk_default;
	1936	+ break;
	1937	+ case HNS_ROCE_HOP_NUM_0:
	1938	+ obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
	1939	+ break;
	1940	+ default:
	1941	+ pr_err("table %u not support hop_num = %u!\n", hem_type,
	1942	+ hop_num);
	1943	+ return;
	1944	+ }
	1945	+
	1946	+ if (hem_type >= HEM_TYPE_MTT)
	1947	+ *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
	1948	+ else
	1949	+ *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
	1950	+}
	1951	+
	1952	+static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
	1953	+{
	1954	+ struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
	1955	+ struct hns_roce_caps *caps = &hr_dev->caps;
	1956	+ struct hns_roce_query_pf_caps_a *resp_a;
	1957	+ struct hns_roce_query_pf_caps_b *resp_b;
	1958	+ struct hns_roce_query_pf_caps_c *resp_c;
	1959	+ struct hns_roce_query_pf_caps_d *resp_d;
	1960	+ struct hns_roce_query_pf_caps_e *resp_e;
	1961	+ int ctx_hop_num;
	1962	+ int pbl_hop_num;
	1963	+ int ret;
	1964	+ int i;
	1965	+
	1966	+ for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
	1967	+ hns_roce_cmq_setup_basic_desc(&desc[i],
	1968	+ HNS_ROCE_OPC_QUERY_PF_CAPS_NUM,
	1969	+ true);
	1970	+ if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
	1971	+ desc[i].flag \|= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
	1972	+ else
	1973	+ desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
	1974	+ }
	1975	+
	1976	+ ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);
	1977	+ if (ret)
	1978	+ return ret;
	1979	+
	1980	+ resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;
	1981	+ resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;
	1982	+ resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
	1983	+ resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
	1984	+ resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
	1985	+
	1986	+ caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;
	1987	+ caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);
	1988	+ caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline);
	1989	+ caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg);
	1990	+ caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg);
	1991	+ caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer);
	1992	+ caps->num_cqc_timer = le16_to_cpu(resp_a->num_cqc_timer);
	1993	+ caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges);
	1994	+ caps->num_aeq_vectors = resp_a->num_aeq_vectors;
	1995	+ caps->num_other_vectors = resp_a->num_other_vectors;
	1996	+ caps->max_sq_desc_sz = resp_a->max_sq_desc_sz;
	1997	+ caps->max_rq_desc_sz = resp_a->max_rq_desc_sz;
	1998	+ caps->max_srq_desc_sz = resp_a->max_srq_desc_sz;
	1999	+ caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE;
	2000	+
	2001	+ caps->mtpt_entry_sz = resp_b->mtpt_entry_sz;
	2002	+ caps->irrl_entry_sz = resp_b->irrl_entry_sz;
	2003	+ caps->trrl_entry_sz = resp_b->trrl_entry_sz;
	2004	+ caps->cqc_entry_sz = resp_b->cqc_entry_sz;
	2005	+ caps->srqc_entry_sz = resp_b->srqc_entry_sz;
	2006	+ caps->idx_entry_sz = resp_b->idx_entry_sz;
	2007	+ caps->sccc_sz = resp_b->sccc_sz;
	2008	+ caps->max_mtu = resp_b->max_mtu;
	2009	+ caps->qpc_sz = HNS_ROCE_V2_QPC_SZ;
	2010	+ caps->min_cqes = resp_b->min_cqes;
	2011	+ caps->min_wqes = resp_b->min_wqes;
	2012	+ caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap);
	2013	+ caps->pkey_table_len[0] = resp_b->pkey_table_len;
	2014	+ caps->phy_num_uars = resp_b->phy_num_uars;
	2015	+ ctx_hop_num = resp_b->ctx_hop_num;
	2016	+ pbl_hop_num = resp_b->pbl_hop_num;
	2017	+
	2018	+ caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds,
	2019	+ V2_QUERY_PF_CAPS_C_NUM_PDS_M,
	2020	+ V2_QUERY_PF_CAPS_C_NUM_PDS_S);
	2021	+ caps->flags = roce_get_field(resp_c->cap_flags_num_pds,
	2022	+ V2_QUERY_PF_CAPS_C_CAP_FLAGS_M,
	2023	+ V2_QUERY_PF_CAPS_C_CAP_FLAGS_S);
	2024	+ caps->flags \|= le16_to_cpu(resp_d->cap_flags_ex) <<
	2025	+ HNS_ROCE_CAP_FLAGS_EX_SHIFT;
	2026	+
	2027	+ caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs,
	2028	+ V2_QUERY_PF_CAPS_C_NUM_CQS_M,
	2029	+ V2_QUERY_PF_CAPS_C_NUM_CQS_S);
	2030	+ caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs,
	2031	+ V2_QUERY_PF_CAPS_C_MAX_GID_M,
	2032	+ V2_QUERY_PF_CAPS_C_MAX_GID_S);
	2033	+ caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth,
	2034	+ V2_QUERY_PF_CAPS_C_CQ_DEPTH_M,
	2035	+ V2_QUERY_PF_CAPS_C_CQ_DEPTH_S);
	2036	+ caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws,
	2037	+ V2_QUERY_PF_CAPS_C_NUM_MRWS_M,
	2038	+ V2_QUERY_PF_CAPS_C_NUM_MRWS_S);
	2039	+ caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps,
	2040	+ V2_QUERY_PF_CAPS_C_NUM_QPS_M,
	2041	+ V2_QUERY_PF_CAPS_C_NUM_QPS_S);
	2042	+ caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps,
	2043	+ V2_QUERY_PF_CAPS_C_MAX_ORD_M,
	2044	+ V2_QUERY_PF_CAPS_C_MAX_ORD_S);
	2045	+ caps->max_qp_dest_rdma = caps->max_qp_init_rdma;
	2046	+ caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
	2047	+ caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs,
	2048	+ V2_QUERY_PF_CAPS_D_NUM_SRQS_M,
	2049	+ V2_QUERY_PF_CAPS_D_NUM_SRQS_S);
	2050	+ caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
	2051	+ caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth,
	2052	+ V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M,
	2053	+ V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S);
	2054	+ caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth,
	2055	+ V2_QUERY_PF_CAPS_D_NUM_CEQS_M,
	2056	+ V2_QUERY_PF_CAPS_D_NUM_CEQS_S);
	2057	+ caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth,
	2058	+ V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M,
	2059	+ V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S);
	2060	+ caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
	2061	+ V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M,
	2062	+ V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S);
	2063	+ caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
	2064	+ V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M,
	2065	+ V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S);
	2066	+ caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds,
	2067	+ V2_QUERY_PF_CAPS_D_RSV_PDS_M,
	2068	+ V2_QUERY_PF_CAPS_D_RSV_PDS_S);
	2069	+ caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds,
	2070	+ V2_QUERY_PF_CAPS_D_NUM_UARS_M,
	2071	+ V2_QUERY_PF_CAPS_D_NUM_UARS_S);
	2072	+ caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps,
	2073	+ V2_QUERY_PF_CAPS_D_RSV_QPS_M,
	2074	+ V2_QUERY_PF_CAPS_D_RSV_QPS_S);
	2075	+ caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps,
	2076	+ V2_QUERY_PF_CAPS_D_RSV_UARS_M,
	2077	+ V2_QUERY_PF_CAPS_D_RSV_UARS_S);
	2078	+ caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
	2079	+ V2_QUERY_PF_CAPS_E_RSV_MRWS_M,
	2080	+ V2_QUERY_PF_CAPS_E_RSV_MRWS_S);
	2081	+ caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
	2082	+ V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M,
	2083	+ V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S);
	2084	+ caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs,
	2085	+ V2_QUERY_PF_CAPS_E_RSV_CQS_M,
	2086	+ V2_QUERY_PF_CAPS_E_RSV_CQS_S);
	2087	+ caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs,
	2088	+ V2_QUERY_PF_CAPS_E_RSV_SRQS_M,
	2089	+ V2_QUERY_PF_CAPS_E_RSV_SRQS_S);
	2090	+ caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey,
	2091	+ V2_QUERY_PF_CAPS_E_RSV_LKEYS_M,
	2092	+ V2_QUERY_PF_CAPS_E_RSV_LKEYS_S);
	2093	+ caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
	2094	+ caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
	2095	+ caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
	2096	+ caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
	2097	+
	2098	+ caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
	2099	+ caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
	2100	+ caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
	2101	+ caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
	2102	+ caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
	2103	+ caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
	2104	+ caps->mtt_ba_pg_sz = 0;
	2105	+ caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
	2106	+ caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
	2107	+ caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
	2108	+
	2109	+ caps->qpc_hop_num = ctx_hop_num;
	2110	+ caps->srqc_hop_num = ctx_hop_num;
	2111	+ caps->cqc_hop_num = ctx_hop_num;
	2112	+ caps->mpt_hop_num = ctx_hop_num;
	2113	+ caps->mtt_hop_num = pbl_hop_num;
	2114	+ caps->cqe_hop_num = pbl_hop_num;
	2115	+ caps->srqwqe_hop_num = pbl_hop_num;
	2116	+ caps->idx_hop_num = pbl_hop_num;
	2117	+ caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
	2118	+ V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M,
	2119	+ V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S);
	2120	+ caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
	2121	+ V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M,
	2122	+ V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S);
	2123	+ caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
	2124	+ V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M,
	2125	+ V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S);
	2126	+
	2127	+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
	2128	+ caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
	2129	+ caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
	2130	+ caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
	2131	+ caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
	2132	+ caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
	2133	+ }
	2134	+
	2135	+ calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,
	2136	+ caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,
	2137	+ HEM_TYPE_QPC);
	2138	+ calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,
	2139	+ caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,
	2140	+ HEM_TYPE_MTPT);
	2141	+ calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,
	2142	+ caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,
	2143	+ HEM_TYPE_CQC);
	2144	+ calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num,
	2145	+ caps->srqc_bt_num, &caps->srqc_buf_pg_sz,
	2146	+ &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC);
	2147	+
	2148	+ caps->sccc_hop_num = ctx_hop_num;
	2149	+ caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
	2150	+ caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
	2151	+
	2152	+ calc_pg_sz(caps->num_qps, caps->sccc_sz,
	2153	+ caps->sccc_hop_num, caps->sccc_bt_num,
	2154	+ &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,
	2155	+ HEM_TYPE_SCCC);
	2156	+ calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,
	2157	+ caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,
	2158	+ &caps->cqc_timer_buf_pg_sz,
	2159	+ &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);
	2160	+
	2161	+ calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num,
	2162	+ 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
	2163	+ calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,
	2164	+ caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,
	2165	+ &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);
	2166	+ calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num,
	2167	+ 1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX);
	2168	+
	2169	+ if (!(caps->page_size_cap & PAGE_SIZE))
	2170	+ caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
	2171	+
	2172	+ return 0;
	2173	+}
	2174	+
	2175	+static int hns_roce_config_qpc_size(struct hns_roce_dev *hr_dev)
	2176	+{
	2177	+ struct hns_roce_cmq_desc desc;
	2178	+ struct hns_roce_cfg_entry_size *cfg_size =
	2179	+ (struct hns_roce_cfg_entry_size *)desc.data;
	2180	+
	2181	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
	2182	+ false);
	2183	+
	2184	+ cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_QPC_SIZE);
	2185	+ cfg_size->size = cpu_to_le32(hr_dev->caps.qpc_sz);
	2186	+
	2187	+ return hns_roce_cmq_send(hr_dev, &desc, 1);
	2188	+}
	2189	+
	2190	+static int hns_roce_config_sccc_size(struct hns_roce_dev *hr_dev)
	2191	+{
	2192	+ struct hns_roce_cmq_desc desc;
	2193	+ struct hns_roce_cfg_entry_size *cfg_size =
	2194	+ (struct hns_roce_cfg_entry_size *)desc.data;
	2195	+
	2196	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
	2197	+ false);
	2198	+
	2199	+ cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_SCCC_SIZE);
	2200	+ cfg_size->size = cpu_to_le32(hr_dev->caps.sccc_sz);
	2201	+
	2202	+ return hns_roce_cmq_send(hr_dev, &desc, 1);
	2203	+}
	2204	+
	2205	+static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev)
	2206	+{
	2207	+ int ret;
	2208	+
	2209	+ if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
	2210	+ return 0;
	2211	+
	2212	+ ret = hns_roce_config_qpc_size(hr_dev);
	2213	+ if (ret) {
	2214	+ dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret);
	2215	+ return ret;
	2216	+ }
	2217	+
	2218	+ ret = hns_roce_config_sccc_size(hr_dev);
	2219	+ if (ret)
	2220	+ dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret);
	2221	+
	2222	+ return ret;
1153	2223	}
1154	2224
1155	2225	static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
..	..	@@ -1158,6 +2228,13 @@
1158	2228	int ret;
1159	2229
1160	2230	ret = hns_roce_cmq_query_hw_info(hr_dev);
	2231	+ if (ret) {
	2232	+ dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
	2233	+ ret);
	2234	+ return ret;
	2235	+ }
	2236	+
	2237	+ ret = hns_roce_query_fw_ver(hr_dev);
1161	2238	if (ret) {
1162	2239	dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1163	2240	ret);
..	..	@@ -1179,6 +2256,36 @@
1179	2256	return ret;
1180	2257	}
1181	2258
	2259	+ ret = hns_roce_query_pf_timer_resource(hr_dev);
	2260	+ if (ret) {
	2261	+ dev_err(hr_dev->dev,
	2262	+ "failed to query pf timer resource, ret = %d.\n", ret);
	2263	+ return ret;
	2264	+ }
	2265	+
	2266	+ ret = hns_roce_set_vf_switch_param(hr_dev, 0);
	2267	+ if (ret) {
	2268	+ dev_err(hr_dev->dev,
	2269	+ "failed to set function switch param, ret = %d.\n",
	2270	+ ret);
	2271	+ return ret;
	2272	+ }
	2273	+
	2274	+ hr_dev->vendor_part_id = hr_dev->pci_dev->device;
	2275	+ hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
	2276	+
	2277	+ caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
	2278	+ caps->pbl_buf_pg_sz = 0;
	2279	+ caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
	2280	+ caps->eqe_ba_pg_sz = 0;
	2281	+ caps->eqe_buf_pg_sz = 0;
	2282	+ caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
	2283	+ caps->tsq_buf_pg_sz = 0;
	2284	+
	2285	+ ret = hns_roce_query_pf_caps(hr_dev);
	2286	+ if (ret)
	2287	+ set_default_caps(hr_dev);
	2288	+
1182	2289	ret = hns_roce_alloc_vf_resource(hr_dev);
1183	2290	if (ret) {
1184	2291	dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
..	..	@@ -1186,89 +2293,15 @@
1186	2293	return ret;
1187	2294	}
1188	2295
1189		- hr_dev->vendor_part_id = 0;
1190		- hr_dev->sys_image_guid = 0;
1191		-
1192		- caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM;
1193		- caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM;
1194		- caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM;
1195		- caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM;
1196		- caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1197		- caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1198		- caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1199		- caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
1200		- caps->num_uars = HNS_ROCE_V2_UAR_NUM;
1201		- caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM;
1202		- caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM;
1203		- caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM;
1204		- caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1205		- caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM;
1206		- caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
1207		- caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
1208		- caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM;
1209		- caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1210		- caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1211		- caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1212		- caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1213		- caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1214		- caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ;
1215		- caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1216		- caps->trrl_entry_sz = HNS_ROCE_V2_TRRL_ENTRY_SZ;
1217		- caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ;
1218		- caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1219		- caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
1220		- caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1221		- caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1222		- caps->reserved_lkey = 0;
1223		- caps->reserved_pds = 0;
1224		- caps->reserved_mrws = 1;
1225		- caps->reserved_uars = 0;
1226		- caps->reserved_cqs = 0;
1227		- caps->reserved_qps = HNS_ROCE_V2_RSV_QPS;
1228		-
1229		- caps->qpc_ba_pg_sz = 0;
1230		- caps->qpc_buf_pg_sz = 0;
1231		- caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1232		- caps->srqc_ba_pg_sz = 0;
1233		- caps->srqc_buf_pg_sz = 0;
1234		- caps->srqc_hop_num = HNS_ROCE_HOP_NUM_0;
1235		- caps->cqc_ba_pg_sz = 0;
1236		- caps->cqc_buf_pg_sz = 0;
1237		- caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1238		- caps->mpt_ba_pg_sz = 0;
1239		- caps->mpt_buf_pg_sz = 0;
1240		- caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1241		- caps->pbl_ba_pg_sz = 0;
1242		- caps->pbl_buf_pg_sz = 0;
1243		- caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
1244		- caps->mtt_ba_pg_sz = 0;
1245		- caps->mtt_buf_pg_sz = 0;
1246		- caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
1247		- caps->cqe_ba_pg_sz = 0;
1248		- caps->cqe_buf_pg_sz = 0;
1249		- caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
1250		- caps->eqe_ba_pg_sz = 0;
1251		- caps->eqe_buf_pg_sz = 0;
1252		- caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
1253		- caps->tsq_buf_pg_sz = 0;
1254		- caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1255		-
1256		- caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR \|
1257		- HNS_ROCE_CAP_FLAG_ROCE_V1_V2 \|
1258		- HNS_ROCE_CAP_FLAG_RQ_INLINE \|
1259		- HNS_ROCE_CAP_FLAG_RECORD_DB \|
1260		- HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
1261		- caps->pkey_table_len[0] = 1;
1262		- caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1263		- caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
1264		- caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM;
1265		- caps->local_ca_ack_delay = 0;
1266		- caps->max_mtu = IB_MTU_4096;
1267		-
1268	2296	ret = hns_roce_v2_set_bt(hr_dev);
1269		- if (ret)
1270		- dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1271		- ret);
	2297	+ if (ret) {
	2298	+ dev_err(hr_dev->dev,
	2299	+ "Configure bt attribute fail, ret = %d.\n", ret);
	2300	+ return ret;
	2301	+ }
	2302	+
	2303	+ /* Configure the size of QPC, SCCC, etc. */
	2304	+ ret = hns_roce_config_entry_size(hr_dev);
1272	2305
1273	2306	return ret;
1274	2307	}
..	..	@@ -1303,8 +2336,6 @@
1303	2336
1304	2337	page_num = link_tbl->npages;
1305	2338	entry = link_tbl->table.buf;
1306		- memset(req_a, 0, sizeof(*req_a));
1307		- memset(req_b, 0, sizeof(*req_b));
1308	2339
1309	2340	for (i = 0; i < 2; i++) {
1310	2341	hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
..	..	@@ -1313,41 +2344,30 @@
1313	2344	desc[i].flag \|= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1314	2345	else
1315	2346	desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1316		-
1317		- if (i == 0) {
1318		- req_a->base_addr_l = link_tbl->table.map & 0xffffffff;
1319		- req_a->base_addr_h = (link_tbl->table.map >> 32) &
1320		- 0xffffffff;
1321		- roce_set_field(req_a->depth_pgsz_init_en,
1322		- CFG_LLM_QUE_DEPTH_M,
1323		- CFG_LLM_QUE_DEPTH_S,
1324		- link_tbl->npages);
1325		- roce_set_field(req_a->depth_pgsz_init_en,
1326		- CFG_LLM_QUE_PGSZ_M,
1327		- CFG_LLM_QUE_PGSZ_S,
1328		- link_tbl->pg_sz);
1329		- req_a->head_ba_l = entry[0].blk_ba0;
1330		- req_a->head_ba_h_nxtptr = entry[0].blk_ba1_nxt_ptr;
1331		- roce_set_field(req_a->head_ptr,
1332		- CFG_LLM_HEAD_PTR_M,
1333		- CFG_LLM_HEAD_PTR_S, 0);
1334		- } else {
1335		- req_b->tail_ba_l = entry[page_num - 1].blk_ba0;
1336		- roce_set_field(req_b->tail_ba_h,
1337		- CFG_LLM_TAIL_BA_H_M,
1338		- CFG_LLM_TAIL_BA_H_S,
1339		- entry[page_num - 1].blk_ba1_nxt_ptr &
1340		- HNS_ROCE_LINK_TABLE_BA1_M);
1341		- roce_set_field(req_b->tail_ptr,
1342		- CFG_LLM_TAIL_PTR_M,
1343		- CFG_LLM_TAIL_PTR_S,
1344		- (entry[page_num - 2].blk_ba1_nxt_ptr &
1345		- HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
1346		- HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1347		- }
1348	2347	}
1349		- roce_set_field(req_a->depth_pgsz_init_en,
1350		- CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1);
	2348	+
	2349	+ req_a->base_addr_l = cpu_to_le32(link_tbl->table.map & 0xffffffff);
	2350	+ req_a->base_addr_h = cpu_to_le32(link_tbl->table.map >> 32);
	2351	+ roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_DEPTH_M,
	2352	+ CFG_LLM_QUE_DEPTH_S, link_tbl->npages);
	2353	+ roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_PGSZ_M,
	2354	+ CFG_LLM_QUE_PGSZ_S, link_tbl->pg_sz);
	2355	+ roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M,
	2356	+ CFG_LLM_INIT_EN_S, 1);
	2357	+ req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
	2358	+ req_a->head_ba_h_nxtptr = cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
	2359	+ roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M, CFG_LLM_HEAD_PTR_S,
	2360	+ 0);
	2361	+
	2362	+ req_b->tail_ba_l = cpu_to_le32(entry[page_num - 1].blk_ba0);
	2363	+ roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M,
	2364	+ CFG_LLM_TAIL_BA_H_S,
	2365	+ entry[page_num - 1].blk_ba1_nxt_ptr &
	2366	+ HNS_ROCE_LINK_TABLE_BA1_M);
	2367	+ roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M, CFG_LLM_TAIL_PTR_S,
	2368	+ (entry[page_num - 2].blk_ba1_nxt_ptr &
	2369	+ HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
	2370	+ HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1351	2371
1352	2372	return hns_roce_cmq_send(hr_dev, desc, 2);
1353	2373	}
..	..	@@ -1407,19 +2427,13 @@
1407	2427	goto err_alloc_buf_failed;
1408	2428
1409	2429	link_tbl->pg_list[i].map = t;
1410		- memset(link_tbl->pg_list[i].buf, 0, buf_chk_sz);
1411	2430
1412		- entry[i].blk_ba0 = (t >> 12) & 0xffffffff;
1413		- roce_set_field(entry[i].blk_ba1_nxt_ptr,
1414		- HNS_ROCE_LINK_TABLE_BA1_M,
1415		- HNS_ROCE_LINK_TABLE_BA1_S,
1416		- t >> 44);
	2431	+ entry[i].blk_ba0 = (u32)(t >> 12);
	2432	+ entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44);
1417	2433
1418	2434	if (i < (pg_num - 1))
1419		- roce_set_field(entry[i].blk_ba1_nxt_ptr,
1420		- HNS_ROCE_LINK_TABLE_NXT_PTR_M,
1421		- HNS_ROCE_LINK_TABLE_NXT_PTR_S,
1422		- i + 1);
	2435	+ entry[i].blk_ba1_nxt_ptr \|=
	2436	+ (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S;
1423	2437	}
1424	2438	link_tbl->npages = pg_num;
1425	2439	link_tbl->pg_sz = buf_chk_sz;
..	..	@@ -1464,7 +2478,8 @@
1464	2478	static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
1465	2479	{
1466	2480	struct hns_roce_v2_priv *priv = hr_dev->priv;
1467		- int ret;
	2481	+ int qpc_count, cqc_count;
	2482	+ int ret, i;
1468	2483
1469	2484	/* TSQ includes SQ doorbell and ack doorbell */
1470	2485	ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
..	..	@@ -1479,7 +2494,39 @@
1479	2494	goto err_tpq_init_failed;
1480	2495	}
1481	2496
	2497	+ /* Alloc memory for QPC Timer buffer space chunk */
	2498	+ for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
	2499	+ qpc_count++) {
	2500	+ ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
	2501	+ qpc_count);
	2502	+ if (ret) {
	2503	+ dev_err(hr_dev->dev, "QPC Timer get failed\n");
	2504	+ goto err_qpc_timer_failed;
	2505	+ }
	2506	+ }
	2507	+
	2508	+ /* Alloc memory for CQC Timer buffer space chunk */
	2509	+ for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
	2510	+ cqc_count++) {
	2511	+ ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
	2512	+ cqc_count);
	2513	+ if (ret) {
	2514	+ dev_err(hr_dev->dev, "CQC Timer get failed\n");
	2515	+ goto err_cqc_timer_failed;
	2516	+ }
	2517	+ }
	2518	+
1482	2519	return 0;
	2520	+
	2521	+err_cqc_timer_failed:
	2522	+ for (i = 0; i < cqc_count; i++)
	2523	+ hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
	2524	+
	2525	+err_qpc_timer_failed:
	2526	+ for (i = 0; i < qpc_count; i++)
	2527	+ hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
	2528	+
	2529	+ hns_roce_free_link_table(hr_dev, &priv->tpq);
1483	2530
1484	2531	err_tpq_init_failed:
1485	2532	hns_roce_free_link_table(hr_dev, &priv->tsq);
..	..	@@ -1491,22 +2538,59 @@
1491	2538	{
1492	2539	struct hns_roce_v2_priv *priv = hr_dev->priv;
1493	2540
	2541	+ hns_roce_function_clear(hr_dev);
	2542	+
1494	2543	hns_roce_free_link_table(hr_dev, &priv->tpq);
1495	2544	hns_roce_free_link_table(hr_dev, &priv->tsq);
1496	2545	}
1497	2546
	2547	+static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
	2548	+{
	2549	+ struct hns_roce_cmq_desc desc;
	2550	+ struct hns_roce_mbox_status *mb_st =
	2551	+ (struct hns_roce_mbox_status *)desc.data;
	2552	+ enum hns_roce_cmd_return_status status;
	2553	+
	2554	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
	2555	+
	2556	+ status = hns_roce_cmq_send(hr_dev, &desc, 1);
	2557	+ if (status)
	2558	+ return status;
	2559	+
	2560	+ return le32_to_cpu(mb_st->mb_status_hw_run);
	2561	+}
	2562	+
1498	2563	static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
1499	2564	{
1500		- u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
	2565	+ u32 status = hns_roce_query_mbox_status(hr_dev);
1501	2566
1502	2567	return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
1503	2568	}
1504	2569
1505	2570	static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
1506	2571	{
1507		- u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
	2572	+ u32 status = hns_roce_query_mbox_status(hr_dev);
1508	2573
1509	2574	return status & HNS_ROCE_HW_MB_STATUS_MASK;
	2575	+}
	2576	+
	2577	+static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
	2578	+ u64 out_param, u32 in_modifier, u8 op_modifier,
	2579	+ u16 op, u16 token, int event)
	2580	+{
	2581	+ struct hns_roce_cmq_desc desc;
	2582	+ struct hns_roce_post_mbox mb = (struct hns_roce_post_mbox )desc.data;
	2583	+
	2584	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
	2585	+
	2586	+ mb->in_param_l = cpu_to_le32(in_param);
	2587	+ mb->in_param_h = cpu_to_le32(in_param >> 32);
	2588	+ mb->out_param_l = cpu_to_le32(out_param);
	2589	+ mb->out_param_h = cpu_to_le32(out_param >> 32);
	2590	+ mb->cmd_tag = cpu_to_le32(in_modifier << 8 \| op);
	2591	+ mb->token_event_en = cpu_to_le32(event << 16 \| token);
	2592	+
	2593	+ return hns_roce_cmq_send(hr_dev, &desc, 1);
1510	2594	}
1511	2595
1512	2596	static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
..	..	@@ -1514,11 +2598,8 @@
1514	2598	u16 op, u16 token, int event)
1515	2599	{
1516	2600	struct device *dev = hr_dev->dev;
1517		- u32 __iomem hcr = (u32 __iomem )(hr_dev->reg_base +
1518		- ROCEE_VF_MB_CFG0_REG);
1519	2601	unsigned long end;
1520		- u32 val0 = 0;
1521		- u32 val1 = 0;
	2602	+ int ret;
1522	2603
1523	2604	end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
1524	2605	while (hns_roce_v2_cmd_pending(hr_dev)) {
..	..	@@ -1530,34 +2611,19 @@
1530	2611	cond_resched();
1531	2612	}
1532	2613
1533		- roce_set_field(val0, HNS_ROCE_VF_MB4_TAG_MASK,
1534		- HNS_ROCE_VF_MB4_TAG_SHIFT, in_modifier);
1535		- roce_set_field(val0, HNS_ROCE_VF_MB4_CMD_MASK,
1536		- HNS_ROCE_VF_MB4_CMD_SHIFT, op);
1537		- roce_set_field(val1, HNS_ROCE_VF_MB5_EVENT_MASK,
1538		- HNS_ROCE_VF_MB5_EVENT_SHIFT, event);
1539		- roce_set_field(val1, HNS_ROCE_VF_MB5_TOKEN_MASK,
1540		- HNS_ROCE_VF_MB5_TOKEN_SHIFT, token);
	2614	+ ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
	2615	+ op_modifier, op, token, event);
	2616	+ if (ret)
	2617	+ dev_err(dev, "Post mailbox fail(%d)\n", ret);
1541	2618
1542		- writeq(in_param, hcr + 0);
1543		- writeq(out_param, hcr + 2);
1544		-
1545		- /* Memory barrier */
1546		- wmb();
1547		-
1548		- writel(val0, hcr + 4);
1549		- writel(val1, hcr + 5);
1550		-
1551		- mmiowb();
1552		-
1553		- return 0;
	2619	+ return ret;
1554	2620	}
1555	2621
1556	2622	static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
1557	2623	unsigned long timeout)
1558	2624	{
1559	2625	struct device *dev = hr_dev->dev;
1560		- unsigned long end = 0;
	2626	+ unsigned long end;
1561	2627	u32 status;
1562	2628
1563	2629	end = msecs_to_jiffies(timeout) + jiffies;
..	..	@@ -1571,6 +2637,9 @@
1571	2637
1572	2638	status = hns_roce_v2_cmd_complete(hr_dev);
1573	2639	if (status != 0x1) {
	2640	+ if (status == CMD_RST_PRC_EBUSY)
	2641	+ return status;
	2642	+
1574	2643	dev_err(dev, "mailbox status 0x%x!\n", status);
1575	2644	return -EBUSY;
1576	2645	}
..	..	@@ -1589,11 +2658,9 @@
1589	2658
1590	2659	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
1591	2660
1592		- roce_set_field(sgid_tb->table_idx_rsv,
1593		- CFG_SGID_TB_TABLE_IDX_M,
	2661	+ roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M,
1594	2662	CFG_SGID_TB_TABLE_IDX_S, gid_index);
1595		- roce_set_field(sgid_tb->vf_sgid_type_rsv,
1596		- CFG_SGID_TB_VF_SGID_TYPE_M,
	2663	+ roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M,
1597	2664	CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
1598	2665
1599	2666	p = (u32 *)&gid->raw[0];
..	..	@@ -1633,7 +2700,9 @@
1633	2700
1634	2701	ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
1635	2702	if (ret)
1636		- dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret);
	2703	+ ibdev_err(&hr_dev->ib_dev,
	2704	+ "failed to configure sgid table, ret = %d!\n",
	2705	+ ret);
1637	2706
1638	2707	return ret;
1639	2708	}
..	..	@@ -1652,52 +2721,43 @@
1652	2721	reg_smac_l = (u32 )(&addr[0]);
1653	2722	reg_smac_h = (u16 )(&addr[4]);
1654	2723
1655		- memset(smac_tb, 0, sizeof(*smac_tb));
1656		- roce_set_field(smac_tb->tb_idx_rsv,
1657		- CFG_SMAC_TB_IDX_M,
	2724	+ roce_set_field(smac_tb->tb_idx_rsv, CFG_SMAC_TB_IDX_M,
1658	2725	CFG_SMAC_TB_IDX_S, phy_port);
1659		- roce_set_field(smac_tb->vf_smac_h_rsv,
1660		- CFG_SMAC_TB_VF_SMAC_H_M,
	2726	+ roce_set_field(smac_tb->vf_smac_h_rsv, CFG_SMAC_TB_VF_SMAC_H_M,
1661	2727	CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
1662		- smac_tb->vf_smac_l = reg_smac_l;
	2728	+ smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
1663	2729
1664	2730	return hns_roce_cmq_send(hr_dev, &desc, 1);
1665	2731	}
1666	2732
1667		-static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
	2733	+static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,
	2734	+ struct hns_roce_v2_mpt_entry *mpt_entry,
1668	2735	struct hns_roce_mr *mr)
1669	2736	{
1670		- struct scatterlist *sg;
1671		- u64 page_addr;
1672		- u64 *pages;
1673		- int i, j;
1674		- int len;
1675		- int entry;
	2737	+ u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
	2738	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	2739	+ dma_addr_t pbl_ba;
	2740	+ int i, count;
1676	2741
1677		- mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
1678		- mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
	2742	+ count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
	2743	+ min_t(int, ARRAY_SIZE(pages), mr->npages),
	2744	+ &pbl_ba);
	2745	+ if (count < 1) {
	2746	+ ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
	2747	+ count);
	2748	+ return -ENOBUFS;
	2749	+ }
	2750	+
	2751	+ /* Aligned to the hardware address access unit */
	2752	+ for (i = 0; i < count; i++)
	2753	+ pages[i] >>= 6;
	2754	+
	2755	+ mpt_entry->pbl_size = cpu_to_le32(mr->npages);
	2756	+ mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3);
1679	2757	roce_set_field(mpt_entry->byte_48_mode_ba,
1680	2758	V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
1681		- upper_32_bits(mr->pbl_ba >> 3));
	2759	+ upper_32_bits(pbl_ba >> 3));
1682	2760
1683		- pages = (u64 *)__get_free_page(GFP_KERNEL);
1684		- if (!pages)
1685		- return -ENOMEM;
1686		-
1687		- i = 0;
1688		- for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1689		- len = sg_dma_len(sg) >> PAGE_SHIFT;
1690		- for (j = 0; j < len; ++j) {
1691		- page_addr = sg_dma_address(sg) +
1692		- (j << mr->umem->page_shift);
1693		- pages[i] = page_addr >> 6;
1694		- /* Record the first 2 entry directly to MTPT table */
1695		- if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
1696		- goto found;
1697		- i++;
1698		- }
1699		- }
1700		-found:
1701	2761	mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
1702	2762	roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
1703	2763	V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
..	..	@@ -1708,14 +2768,13 @@
1708	2768	roce_set_field(mpt_entry->byte_64_buf_pa1,
1709	2769	V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
1710	2770	V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
1711		- mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
1712		-
1713		- free_page((unsigned long)pages);
	2771	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
1714	2772
1715	2773	return 0;
1716	2774	}
1717	2775
1718		-static int hns_roce_v2_write_mtpt(void mb_buf, struct hns_roce_mr mr,
	2776	+static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
	2777	+ void mb_buf, struct hns_roce_mr mr,
1719	2778	unsigned long mtpt_idx)
1720	2779	{
1721	2780	struct hns_roce_v2_mpt_entry *mpt_entry;
..	..	@@ -1732,16 +2791,17 @@
1732	2791	roce_set_field(mpt_entry->byte_4_pd_hop_st,
1733	2792	V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
1734	2793	V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
1735		- mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
	2794	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
1736	2795	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1737	2796	V2_MPT_BYTE_4_PD_S, mr->pd);
1738	2797
1739	2798	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
1740		- roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
1741		- roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 0);
	2799	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
	2800	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
1742	2801	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
1743	2802	(mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1744		- roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S, 0);
	2803	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
	2804	+ mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
1745	2805	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
1746	2806	(mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1747	2807	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
..	..	@@ -1763,7 +2823,7 @@
1763	2823	if (mr->type == MR_TYPE_DMA)
1764	2824	return 0;
1765	2825
1766		- ret = set_mtpt_pbl(mpt_entry, mr);
	2826	+ ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
1767	2827
1768	2828	return ret;
1769	2829	}
..	..	@@ -1809,16 +2869,105 @@
1809	2869	mr->iova = iova;
1810	2870	mr->size = size;
1811	2871
1812		- ret = set_mtpt_pbl(mpt_entry, mr);
	2872	+ ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
1813	2873	}
1814	2874
1815	2875	return ret;
1816	2876	}
1817	2877
	2878	+static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev,
	2879	+ void mb_buf, struct hns_roce_mr mr)
	2880	+{
	2881	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	2882	+ struct hns_roce_v2_mpt_entry *mpt_entry;
	2883	+ dma_addr_t pbl_ba = 0;
	2884	+
	2885	+ mpt_entry = mb_buf;
	2886	+ memset(mpt_entry, 0, sizeof(*mpt_entry));
	2887	+
	2888	+ if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) {
	2889	+ ibdev_err(ibdev, "failed to find frmr mtr.\n");
	2890	+ return -ENOBUFS;
	2891	+ }
	2892	+
	2893	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
	2894	+ V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
	2895	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
	2896	+ V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
	2897	+ roce_set_field(mpt_entry->byte_4_pd_hop_st,
	2898	+ V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
	2899	+ V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
	2900	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
	2901	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
	2902	+ V2_MPT_BYTE_4_PD_S, mr->pd);
	2903	+
	2904	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
	2905	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
	2906	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
	2907	+
	2908	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
	2909	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
	2910	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
	2911	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
	2912	+
	2913	+ mpt_entry->pbl_size = cpu_to_le32(mr->npages);
	2914	+
	2915	+ mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3));
	2916	+ roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
	2917	+ V2_MPT_BYTE_48_PBL_BA_H_S,
	2918	+ upper_32_bits(pbl_ba >> 3));
	2919	+
	2920	+ roce_set_field(mpt_entry->byte_64_buf_pa1,
	2921	+ V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
	2922	+ V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
	2923	+ to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
	2924	+
	2925	+ return 0;
	2926	+}
	2927	+
	2928	+static int hns_roce_v2_mw_write_mtpt(void mb_buf, struct hns_roce_mw mw)
	2929	+{
	2930	+ struct hns_roce_v2_mpt_entry *mpt_entry;
	2931	+
	2932	+ mpt_entry = mb_buf;
	2933	+ memset(mpt_entry, 0, sizeof(*mpt_entry));
	2934	+
	2935	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
	2936	+ V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
	2937	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
	2938	+ V2_MPT_BYTE_4_PD_S, mw->pdn);
	2939	+ roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
	2940	+ V2_MPT_BYTE_4_PBL_HOP_NUM_S,
	2941	+ mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
	2942	+ mw->pbl_hop_num);
	2943	+ roce_set_field(mpt_entry->byte_4_pd_hop_st,
	2944	+ V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
	2945	+ V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
	2946	+ mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
	2947	+
	2948	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
	2949	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
	2950	+ roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 1);
	2951	+
	2952	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
	2953	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
	2954	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
	2955	+ roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
	2956	+ mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
	2957	+
	2958	+ roce_set_field(mpt_entry->byte_64_buf_pa1,
	2959	+ V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
	2960	+ V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
	2961	+ mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
	2962	+
	2963	+ mpt_entry->lkey = cpu_to_le32(mw->rkey);
	2964	+
	2965	+ return 0;
	2966	+}
	2967	+
1818	2968	static void get_cqe_v2(struct hns_roce_cq hr_cq, int n)
1819	2969	{
1820		- return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1821		- n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
	2970	+ return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);
1822	2971	}
1823	2972
1824	2973	static void get_sw_cqe_v2(struct hns_roce_cq hr_cq, int n)
..	..	@@ -1827,17 +2976,12 @@
1827	2976
1828	2977	/* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1829	2978	return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
1830		- !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL;
	2979	+ !!(n & hr_cq->cq_depth)) ? cqe : NULL;
1831	2980	}
1832	2981
1833		-static struct hns_roce_v2_cqe next_cqe_sw_v2(struct hns_roce_cq hr_cq)
	2982	+static inline void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 ci)
1834	2983	{
1835		- return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
1836		-}
1837		-
1838		-static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
1839		-{
1840		- *hr_cq->set_ci_db = cons_index & 0xffffff;
	2984	+ *hr_cq->set_ci_db = ci & V2_CQ_DB_PARAMETER_CONS_IDX_M;
1841	2985	}
1842	2986
1843	2987	static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
..	..	@@ -1846,11 +2990,12 @@
1846	2990	struct hns_roce_v2_cqe cqe, dest;
1847	2991	u32 prod_index;
1848	2992	int nfreed = 0;
	2993	+ int wqe_index;
1849	2994	u8 owner_bit;
1850	2995
1851	2996	for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
1852	2997	++prod_index) {
1853		- if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
	2998	+ if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
1854	2999	break;
1855	3000	}
1856	3001
..	..	@@ -1863,7 +3008,13 @@
1863	3008	if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
1864	3009	V2_CQE_BYTE_16_LCL_QPN_S) &
1865	3010	HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
1866		- /* In v1 engine, not support SRQ */
	3011	+ if (srq &&
	3012	+ roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
	3013	+ wqe_index = roce_get_field(cqe->byte_4,
	3014	+ V2_CQE_BYTE_4_WQE_INDX_M,
	3015	+ V2_CQE_BYTE_4_WQE_INDX_S);
	3016	+ hns_roce_free_srq_wqe(srq, wqe_index);
	3017	+ }
1867	3018	++nfreed;
1868	3019	} else if (nfreed) {
1869	3020	dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
..	..	@@ -1897,8 +3048,7 @@
1897	3048
1898	3049	static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
1899	3050	struct hns_roce_cq hr_cq, void mb_buf,
1900		- u64 *mtts, dma_addr_t dma_handle, int nent,
1901		- u32 vector)
	3051	+ u64 *mtts, dma_addr_t dma_handle)
1902	3052	{
1903	3053	struct hns_roce_v2_cq_context *cq_context;
1904	3054
..	..	@@ -1910,55 +3060,56 @@
1910	3060	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
1911	3061	V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
1912	3062	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
1913		- V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));
	3063	+ V2_CQC_BYTE_4_SHIFT_S, ilog2(hr_cq->cq_depth));
1914	3064	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
1915		- V2_CQC_BYTE_4_CEQN_S, vector);
1916		- cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn);
	3065	+ V2_CQC_BYTE_4_CEQN_S, hr_cq->vector);
1917	3066
1918	3067	roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
1919	3068	V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
1920	3069
1921		- cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
1922		- cq_context->cqe_cur_blk_addr =
1923		- cpu_to_le32(cq_context->cqe_cur_blk_addr);
	3070	+ roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQE_SIZE_M,
	3071	+ V2_CQC_BYTE_8_CQE_SIZE_S, hr_cq->cqe_size ==
	3072	+ HNS_ROCE_V3_CQE_SIZE ? 1 : 0);
	3073	+
	3074	+ cq_context->cqe_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
1924	3075
1925	3076	roce_set_field(cq_context->byte_16_hop_addr,
1926	3077	V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
1927	3078	V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
1928		- cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT)));
	3079	+ upper_32_bits(to_hr_hw_page_addr(mtts[0])));
1929	3080	roce_set_field(cq_context->byte_16_hop_addr,
1930	3081	V2_CQC_BYTE_16_CQE_HOP_NUM_M,
1931	3082	V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
1932	3083	HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
1933	3084
1934		- cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
	3085	+ cq_context->cqe_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
1935	3086	roce_set_field(cq_context->byte_24_pgsz_addr,
1936	3087	V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
1937	3088	V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
1938		- cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT)));
	3089	+ upper_32_bits(to_hr_hw_page_addr(mtts[1])));
1939	3090	roce_set_field(cq_context->byte_24_pgsz_addr,
1940	3091	V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
1941	3092	V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
1942		- hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
	3093	+ to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));
1943	3094	roce_set_field(cq_context->byte_24_pgsz_addr,
1944	3095	V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
1945	3096	V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
1946		- hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
	3097	+ to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));
1947	3098
1948		- cq_context->cqe_ba = (u32)(dma_handle >> 3);
	3099	+ cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3);
1949	3100
1950	3101	roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
1951	3102	V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
1952	3103
1953		- if (hr_cq->db_en)
1954		- roce_set_bit(cq_context->byte_44_db_record,
1955		- V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
	3104	+ roce_set_bit(cq_context->byte_44_db_record,
	3105	+ V2_CQC_BYTE_44_DB_RECORD_EN_S,
	3106	+ (hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB) ? 1 : 0);
1956	3107
1957	3108	roce_set_field(cq_context->byte_44_db_record,
1958	3109	V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
1959	3110	V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
1960	3111	((u32)hr_cq->db.dma) >> 1);
1961		- cq_context->db_record_addr = hr_cq->db.dma >> 32;
	3112	+ cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32);
1962	3113
1963	3114	roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
1964	3115	V2_CQC_BYTE_56_CQ_MAX_CNT_M,
..	..	@@ -1973,9 +3124,10 @@
1973	3124	static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
1974	3125	enum ib_cq_notify_flags flags)
1975	3126	{
	3127	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
1976	3128	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1977	3129	u32 notification_flag;
1978		- u32 doorbell[2];
	3130	+ __le32 doorbell[2];
1979	3131
1980	3132	doorbell[0] = 0;
1981	3133	doorbell[1] = 0;
..	..	@@ -1991,14 +3143,13 @@
1991	3143	roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
1992	3144	HNS_ROCE_V2_CQ_DB_NTR);
1993	3145	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
1994		- V2_CQ_DB_PARAMETER_CONS_IDX_S,
1995		- hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
	3146	+ V2_CQ_DB_PARAMETER_CONS_IDX_S, hr_cq->cons_index);
1996	3147	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
1997	3148	V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
1998	3149	roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
1999	3150	notification_flag);
2000	3151
2001		- hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
	3152	+ hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);
2002	3153
2003	3154	return 0;
2004	3155	}
..	..	@@ -2018,7 +3169,7 @@
2018	3169
2019	3170	sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
2020	3171	sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
2021		- wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
	3172	+ wqe_buf = hns_roce_get_recv_wqe(*cur_qp, wr_cnt);
2022	3173	data_len = wc->byte_len;
2023	3174
2024	3175	for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
..	..	@@ -2029,7 +3180,7 @@
2029	3180	wqe_buf += size;
2030	3181	}
2031	3182
2032		- if (data_len) {
	3183	+ if (unlikely(data_len)) {
2033	3184	wc->status = IB_WC_LOC_LEN_ERR;
2034	3185	return -EAGAIN;
2035	3186	}
..	..	@@ -2037,24 +3188,137 @@
2037	3188	return 0;
2038	3189	}
2039	3190
	3191	+static int sw_comp(struct hns_roce_qp hr_qp, struct hns_roce_wq wq,
	3192	+ int num_entries, struct ib_wc *wc)
	3193	+{
	3194	+ unsigned int left;
	3195	+ int npolled = 0;
	3196	+
	3197	+ left = wq->head - wq->tail;
	3198	+ if (left == 0)
	3199	+ return 0;
	3200	+
	3201	+ left = min_t(unsigned int, (unsigned int)num_entries, left);
	3202	+ while (npolled < left) {
	3203	+ wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
	3204	+ wc->status = IB_WC_WR_FLUSH_ERR;
	3205	+ wc->vendor_err = 0;
	3206	+ wc->qp = &hr_qp->ibqp;
	3207	+
	3208	+ wq->tail++;
	3209	+ wc++;
	3210	+ npolled++;
	3211	+ }
	3212	+
	3213	+ return npolled;
	3214	+}
	3215	+
	3216	+static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,
	3217	+ struct ib_wc *wc)
	3218	+{
	3219	+ struct hns_roce_qp *hr_qp;
	3220	+ int npolled = 0;
	3221	+
	3222	+ list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {
	3223	+ npolled += sw_comp(hr_qp, &hr_qp->sq,
	3224	+ num_entries - npolled, wc + npolled);
	3225	+ if (npolled >= num_entries)
	3226	+ goto out;
	3227	+ }
	3228	+
	3229	+ list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {
	3230	+ npolled += sw_comp(hr_qp, &hr_qp->rq,
	3231	+ num_entries - npolled, wc + npolled);
	3232	+ if (npolled >= num_entries)
	3233	+ goto out;
	3234	+ }
	3235	+
	3236	+out:
	3237	+ return npolled;
	3238	+}
	3239	+
	3240	+static void get_cqe_status(struct hns_roce_dev hr_dev, struct hns_roce_qp qp,
	3241	+ struct hns_roce_cq cq, struct hns_roce_v2_cqe cqe,
	3242	+ struct ib_wc *wc)
	3243	+{
	3244	+ static const struct {
	3245	+ u32 cqe_status;
	3246	+ enum ib_wc_status wc_status;
	3247	+ } map[] = {
	3248	+ { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },
	3249	+ { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },
	3250	+ { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
	3251	+ { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },
	3252	+ { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
	3253	+ { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },
	3254	+ { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
	3255	+ { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
	3256	+ { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
	3257	+ { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
	3258	+ { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },
	3259	+ { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,
	3260	+ IB_WC_RETRY_EXC_ERR },
	3261	+ { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
	3262	+ { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
	3263	+ { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}
	3264	+ };
	3265	+
	3266	+ u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
	3267	+ V2_CQE_BYTE_4_STATUS_S);
	3268	+ int i;
	3269	+
	3270	+ wc->status = IB_WC_GENERAL_ERR;
	3271	+ for (i = 0; i < ARRAY_SIZE(map); i++)
	3272	+ if (cqe_status == map[i].cqe_status) {
	3273	+ wc->status = map[i].wc_status;
	3274	+ break;
	3275	+ }
	3276	+
	3277	+ if (likely(wc->status == IB_WC_SUCCESS \|\|
	3278	+ wc->status == IB_WC_WR_FLUSH_ERR))
	3279	+ return;
	3280	+
	3281	+ ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status);
	3282	+ print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe,
	3283	+ cq->cqe_size, false);
	3284	+
	3285	+ /*
	3286	+ * For hns ROCEE, GENERAL_ERR is an error type that is not defined in
	3287	+ * the standard protocol, the driver must ignore it and needn't to set
	3288	+ * the QP to an error state.
	3289	+ */
	3290	+ if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)
	3291	+ return;
	3292	+
	3293	+ /*
	3294	+ * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets
	3295	+ * into errored mode. Hence, as a workaround to this hardware
	3296	+ * limitation, driver needs to assist in flushing. But the flushing
	3297	+ * operation uses mailbox to convey the QP state to the hardware and
	3298	+ * which can sleep due to the mutex protection around the mailbox calls.
	3299	+ * Hence, use the deferred flush for now. Once wc error detected, the
	3300	+ * flushing operation is needed.
	3301	+ */
	3302	+ if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
	3303	+ init_flush_work(hr_dev, qp);
	3304	+}
	3305	+
2040	3306	static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2041	3307	struct hns_roce_qp *cur_qp, struct ib_wc wc)
2042	3308	{
2043		- struct hns_roce_dev *hr_dev;
	3309	+ struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
	3310	+ struct hns_roce_srq *srq = NULL;
2044	3311	struct hns_roce_v2_cqe *cqe;
2045	3312	struct hns_roce_qp *hr_qp;
2046	3313	struct hns_roce_wq *wq;
2047		- struct ib_qp_attr attr;
2048		- int attr_mask;
2049	3314	int is_send;
2050	3315	u16 wqe_ctr;
2051	3316	u32 opcode;
2052		- u32 status;
2053	3317	int qpn;
2054	3318	int ret;
2055	3319
2056	3320	/* Find cqe according to consumer index */
2057		- cqe = next_cqe_sw_v2(hr_cq);
	3321	+ cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2058	3322	if (!cqe)
2059	3323	return -EAGAIN;
2060	3324
..	..	@@ -2069,11 +3333,11 @@
2069	3333	V2_CQE_BYTE_16_LCL_QPN_S);
2070	3334
2071	3335	if (!cur_qp \|\| (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (cur_qp)->qpn) {
2072		- hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2073	3336	hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2074	3337	if (unlikely(!hr_qp)) {
2075		- dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
2076		- hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
	3338	+ ibdev_err(&hr_dev->ib_dev,
	3339	+ "CQ %06lx with entry for unknown QPN %06x\n",
	3340	+ hr_cq->cqn, qpn & HNS_ROCE_V2_CQE_QPN_MASK);
2077	3341	return -EINVAL;
2078	3342	}
2079	3343	*cur_qp = hr_qp;
..	..	@@ -2082,126 +3346,7 @@
2082	3346	wc->qp = &(*cur_qp)->ibqp;
2083	3347	wc->vendor_err = 0;
2084	3348
2085		- status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2086		- V2_CQE_BYTE_4_STATUS_S);
2087		- switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
2088		- case HNS_ROCE_CQE_V2_SUCCESS:
2089		- wc->status = IB_WC_SUCCESS;
2090		- break;
2091		- case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
2092		- wc->status = IB_WC_LOC_LEN_ERR;
2093		- break;
2094		- case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
2095		- wc->status = IB_WC_LOC_QP_OP_ERR;
2096		- break;
2097		- case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
2098		- wc->status = IB_WC_LOC_PROT_ERR;
2099		- break;
2100		- case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
2101		- wc->status = IB_WC_WR_FLUSH_ERR;
2102		- break;
2103		- case HNS_ROCE_CQE_V2_MW_BIND_ERR:
2104		- wc->status = IB_WC_MW_BIND_ERR;
2105		- break;
2106		- case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
2107		- wc->status = IB_WC_BAD_RESP_ERR;
2108		- break;
2109		- case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
2110		- wc->status = IB_WC_LOC_ACCESS_ERR;
2111		- break;
2112		- case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
2113		- wc->status = IB_WC_REM_INV_REQ_ERR;
2114		- break;
2115		- case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
2116		- wc->status = IB_WC_REM_ACCESS_ERR;
2117		- break;
2118		- case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
2119		- wc->status = IB_WC_REM_OP_ERR;
2120		- break;
2121		- case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
2122		- wc->status = IB_WC_RETRY_EXC_ERR;
2123		- break;
2124		- case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
2125		- wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2126		- break;
2127		- case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
2128		- wc->status = IB_WC_REM_ABORT_ERR;
2129		- break;
2130		- default:
2131		- wc->status = IB_WC_GENERAL_ERR;
2132		- break;
2133		- }
2134		-
2135		- /* flush cqe if wc status is error, excluding flush error */
2136		- if ((wc->status != IB_WC_SUCCESS) &&
2137		- (wc->status != IB_WC_WR_FLUSH_ERR)) {
2138		- attr_mask = IB_QP_STATE;
2139		- attr.qp_state = IB_QPS_ERR;
2140		- return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp,
2141		- &attr, attr_mask,
2142		- (*cur_qp)->state, IB_QPS_ERR);
2143		- }
2144		-
2145		- if (wc->status == IB_WC_WR_FLUSH_ERR)
2146		- return 0;
2147		-
2148	3349	if (is_send) {
2149		- wc->wc_flags = 0;
2150		- /* SQ corresponding to CQE */
2151		- switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2152		- V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
2153		- case HNS_ROCE_SQ_OPCODE_SEND:
2154		- wc->opcode = IB_WC_SEND;
2155		- break;
2156		- case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
2157		- wc->opcode = IB_WC_SEND;
2158		- break;
2159		- case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
2160		- wc->opcode = IB_WC_SEND;
2161		- wc->wc_flags \|= IB_WC_WITH_IMM;
2162		- break;
2163		- case HNS_ROCE_SQ_OPCODE_RDMA_READ:
2164		- wc->opcode = IB_WC_RDMA_READ;
2165		- wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2166		- break;
2167		- case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
2168		- wc->opcode = IB_WC_RDMA_WRITE;
2169		- break;
2170		- case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
2171		- wc->opcode = IB_WC_RDMA_WRITE;
2172		- wc->wc_flags \|= IB_WC_WITH_IMM;
2173		- break;
2174		- case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
2175		- wc->opcode = IB_WC_LOCAL_INV;
2176		- wc->wc_flags \|= IB_WC_WITH_INVALIDATE;
2177		- break;
2178		- case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
2179		- wc->opcode = IB_WC_COMP_SWAP;
2180		- wc->byte_len = 8;
2181		- break;
2182		- case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
2183		- wc->opcode = IB_WC_FETCH_ADD;
2184		- wc->byte_len = 8;
2185		- break;
2186		- case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
2187		- wc->opcode = IB_WC_MASKED_COMP_SWAP;
2188		- wc->byte_len = 8;
2189		- break;
2190		- case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
2191		- wc->opcode = IB_WC_MASKED_FETCH_ADD;
2192		- wc->byte_len = 8;
2193		- break;
2194		- case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
2195		- wc->opcode = IB_WC_REG_MR;
2196		- break;
2197		- case HNS_ROCE_SQ_OPCODE_BIND_MW:
2198		- wc->opcode = IB_WC_REG_MR;
2199		- break;
2200		- default:
2201		- wc->status = IB_WC_GENERAL_ERR;
2202		- break;
2203		- }
2204		-
2205	3350	wq = &(*cur_qp)->sq;
2206	3351	if ((*cur_qp)->sq_signal_bits) {
2207	3352	/*
..	..	@@ -2218,6 +3363,80 @@
2218	3363
2219	3364	wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2220	3365	++wq->tail;
	3366	+ } else if ((*cur_qp)->ibqp.srq) {
	3367	+ srq = to_hr_srq((*cur_qp)->ibqp.srq);
	3368	+ wqe_ctr = (u16)roce_get_field(cqe->byte_4,
	3369	+ V2_CQE_BYTE_4_WQE_INDX_M,
	3370	+ V2_CQE_BYTE_4_WQE_INDX_S);
	3371	+ wc->wr_id = srq->wrid[wqe_ctr];
	3372	+ hns_roce_free_srq_wqe(srq, wqe_ctr);
	3373	+ } else {
	3374	+ /* Update tail pointer, record wr_id */
	3375	+ wq = &(*cur_qp)->rq;
	3376	+ wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
	3377	+ ++wq->tail;
	3378	+ }
	3379	+
	3380	+ get_cqe_status(hr_dev, *cur_qp, hr_cq, cqe, wc);
	3381	+ if (unlikely(wc->status != IB_WC_SUCCESS))
	3382	+ return 0;
	3383	+
	3384	+ if (is_send) {
	3385	+ wc->wc_flags = 0;
	3386	+ /* SQ corresponding to CQE */
	3387	+ switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
	3388	+ V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
	3389	+ case HNS_ROCE_V2_WQE_OP_SEND:
	3390	+ wc->opcode = IB_WC_SEND;
	3391	+ break;
	3392	+ case HNS_ROCE_V2_WQE_OP_SEND_WITH_INV:
	3393	+ wc->opcode = IB_WC_SEND;
	3394	+ break;
	3395	+ case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:
	3396	+ wc->opcode = IB_WC_SEND;
	3397	+ wc->wc_flags \|= IB_WC_WITH_IMM;
	3398	+ break;
	3399	+ case HNS_ROCE_V2_WQE_OP_RDMA_READ:
	3400	+ wc->opcode = IB_WC_RDMA_READ;
	3401	+ wc->byte_len = le32_to_cpu(cqe->byte_cnt);
	3402	+ break;
	3403	+ case HNS_ROCE_V2_WQE_OP_RDMA_WRITE:
	3404	+ wc->opcode = IB_WC_RDMA_WRITE;
	3405	+ break;
	3406	+ case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:
	3407	+ wc->opcode = IB_WC_RDMA_WRITE;
	3408	+ wc->wc_flags \|= IB_WC_WITH_IMM;
	3409	+ break;
	3410	+ case HNS_ROCE_V2_WQE_OP_LOCAL_INV:
	3411	+ wc->opcode = IB_WC_LOCAL_INV;
	3412	+ wc->wc_flags \|= IB_WC_WITH_INVALIDATE;
	3413	+ break;
	3414	+ case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:
	3415	+ wc->opcode = IB_WC_COMP_SWAP;
	3416	+ wc->byte_len = 8;
	3417	+ break;
	3418	+ case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:
	3419	+ wc->opcode = IB_WC_FETCH_ADD;
	3420	+ wc->byte_len = 8;
	3421	+ break;
	3422	+ case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:
	3423	+ wc->opcode = IB_WC_MASKED_COMP_SWAP;
	3424	+ wc->byte_len = 8;
	3425	+ break;
	3426	+ case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:
	3427	+ wc->opcode = IB_WC_MASKED_FETCH_ADD;
	3428	+ wc->byte_len = 8;
	3429	+ break;
	3430	+ case HNS_ROCE_V2_WQE_OP_FAST_REG_PMR:
	3431	+ wc->opcode = IB_WC_REG_MR;
	3432	+ break;
	3433	+ case HNS_ROCE_V2_WQE_OP_BIND_MW:
	3434	+ wc->opcode = IB_WC_REG_MR;
	3435	+ break;
	3436	+ default:
	3437	+ wc->status = IB_WC_GENERAL_ERR;
	3438	+ break;
	3439	+ }
2221	3440	} else {
2222	3441	/* RQ correspond to CQE */
2223	3442	wc->byte_len = le32_to_cpu(cqe->byte_cnt);
..	..	@@ -2258,14 +3477,9 @@
2258	3477	opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
2259	3478	(roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
2260	3479	ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
2261		- if (ret)
	3480	+ if (unlikely(ret))
2262	3481	return -EAGAIN;
2263	3482	}
2264		-
2265		- /* Update tail pointer, record wr_id */
2266		- wq = &(*cur_qp)->rq;
2267		- wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2268		- ++wq->tail;
2269	3483
2270	3484	wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2271	3485	V2_CQE_BYTE_32_SL_S);
..	..	@@ -2279,15 +3493,16 @@
2279	3493	wc->port_num = roce_get_field(cqe->byte_32,
2280	3494	V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
2281	3495	wc->pkey_index = 0;
2282		- memcpy(wc->smac, cqe->smac, 4);
2283		- wc->smac[4] = roce_get_field(cqe->byte_28,
2284		- V2_CQE_BYTE_28_SMAC_4_M,
2285		- V2_CQE_BYTE_28_SMAC_4_S);
2286		- wc->smac[5] = roce_get_field(cqe->byte_28,
2287		- V2_CQE_BYTE_28_SMAC_5_M,
2288		- V2_CQE_BYTE_28_SMAC_5_S);
2289		- wc->vlan_id = 0xffff;
2290		- wc->wc_flags \|= (IB_WC_WITH_VLAN \| IB_WC_WITH_SMAC);
	3496	+
	3497	+ if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
	3498	+ wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
	3499	+ V2_CQE_BYTE_28_VID_M,
	3500	+ V2_CQE_BYTE_28_VID_S);
	3501	+ wc->wc_flags \|= IB_WC_WITH_VLAN;
	3502	+ } else {
	3503	+ wc->vlan_id = 0xffff;
	3504	+ }
	3505	+
2291	3506	wc->network_hdr_type = roce_get_field(cqe->byte_28,
2292	3507	V2_CQE_BYTE_28_PORT_TYPE_M,
2293	3508	V2_CQE_BYTE_28_PORT_TYPE_S);
..	..	@@ -2299,12 +3514,25 @@
2299	3514	static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
2300	3515	struct ib_wc *wc)
2301	3516	{
	3517	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
2302	3518	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2303	3519	struct hns_roce_qp *cur_qp = NULL;
2304	3520	unsigned long flags;
2305	3521	int npolled;
2306	3522
2307	3523	spin_lock_irqsave(&hr_cq->lock, flags);
	3524	+
	3525	+ /*
	3526	+ * When the device starts to reset, the state is RST_DOWN. At this time,
	3527	+ * there may still be some valid CQEs in the hardware that are not
	3528	+ * polled. Therefore, it is not allowed to switch to the software mode
	3529	+ * immediately. When the state changes to UNINIT, CQE no longer exists
	3530	+ * in the hardware, and then switch to software mode.
	3531	+ */
	3532	+ if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) {
	3533	+ npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);
	3534	+ goto out;
	3535	+ }
2308	3536
2309	3537	for (npolled = 0; npolled < num_entries; ++npolled) {
2310	3538	if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
..	..	@@ -2317,17 +3545,78 @@
2317	3545	hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2318	3546	}
2319	3547
	3548	+out:
2320	3549	spin_unlock_irqrestore(&hr_cq->lock, flags);
2321	3550
2322	3551	return npolled;
	3552	+}
	3553	+
	3554	+static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
	3555	+ int step_idx)
	3556	+{
	3557	+ int op;
	3558	+
	3559	+ if (type == HEM_TYPE_SCCC && step_idx)
	3560	+ return -EINVAL;
	3561	+
	3562	+ switch (type) {
	3563	+ case HEM_TYPE_QPC:
	3564	+ op = HNS_ROCE_CMD_WRITE_QPC_BT0;
	3565	+ break;
	3566	+ case HEM_TYPE_MTPT:
	3567	+ op = HNS_ROCE_CMD_WRITE_MPT_BT0;
	3568	+ break;
	3569	+ case HEM_TYPE_CQC:
	3570	+ op = HNS_ROCE_CMD_WRITE_CQC_BT0;
	3571	+ break;
	3572	+ case HEM_TYPE_SRQC:
	3573	+ op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
	3574	+ break;
	3575	+ case HEM_TYPE_SCCC:
	3576	+ op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
	3577	+ break;
	3578	+ case HEM_TYPE_QPC_TIMER:
	3579	+ op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
	3580	+ break;
	3581	+ case HEM_TYPE_CQC_TIMER:
	3582	+ op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
	3583	+ break;
	3584	+ default:
	3585	+ dev_warn(hr_dev->dev,
	3586	+ "table %u not to be written by mailbox!\n", type);
	3587	+ return -EINVAL;
	3588	+ }
	3589	+
	3590	+ return op + step_idx;
	3591	+}
	3592	+
	3593	+static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba,
	3594	+ u32 hem_type, int step_idx)
	3595	+{
	3596	+ struct hns_roce_cmd_mailbox *mailbox;
	3597	+ int ret;
	3598	+ int op;
	3599	+
	3600	+ op = get_op_for_set_hem(hr_dev, hem_type, step_idx);
	3601	+ if (op < 0)
	3602	+ return 0;
	3603	+
	3604	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	3605	+ if (IS_ERR(mailbox))
	3606	+ return PTR_ERR(mailbox);
	3607	+
	3608	+ ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
	3609	+ 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
	3610	+
	3611	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	3612	+
	3613	+ return ret;
2323	3614	}
2324	3615
2325	3616	static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
2326	3617	struct hns_roce_hem_table *table, int obj,
2327	3618	int step_idx)
2328	3619	{
2329		- struct device *dev = hr_dev->dev;
2330		- struct hns_roce_cmd_mailbox *mailbox;
2331	3620	struct hns_roce_hem_iter iter;
2332	3621	struct hns_roce_hem_mhop mhop;
2333	3622	struct hns_roce_hem *hem;
..	..	@@ -2339,7 +3628,6 @@
2339	3628	u64 bt_ba = 0;
2340	3629	u32 chunk_ba_num;
2341	3630	u32 hop_num;
2342		- u16 op = 0xff;
2343	3631
2344	3632	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2345	3633	return 0;
..	..	@@ -2361,40 +3649,16 @@
2361	3649	hem_idx = i;
2362	3650	}
2363	3651
2364		- switch (table->type) {
2365		- case HEM_TYPE_QPC:
2366		- op = HNS_ROCE_CMD_WRITE_QPC_BT0;
2367		- break;
2368		- case HEM_TYPE_MTPT:
2369		- op = HNS_ROCE_CMD_WRITE_MPT_BT0;
2370		- break;
2371		- case HEM_TYPE_CQC:
2372		- op = HNS_ROCE_CMD_WRITE_CQC_BT0;
2373		- break;
2374		- case HEM_TYPE_SRQC:
2375		- op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
2376		- break;
2377		- default:
2378		- dev_warn(dev, "Table %d not to be written by mailbox!\n",
2379		- table->type);
2380		- return 0;
2381		- }
2382		- op += step_idx;
2383		-
2384		- mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2385		- if (IS_ERR(mailbox))
2386		- return PTR_ERR(mailbox);
	3652	+ if (table->type == HEM_TYPE_SCCC)
	3653	+ obj = mhop.l0_idx;
2387	3654
2388	3655	if (check_whether_last_step(hop_num, step_idx)) {
2389	3656	hem = table->hem[hem_idx];
2390	3657	for (hns_roce_hem_first(hem, &iter);
2391	3658	!hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
2392	3659	bt_ba = hns_roce_hem_addr(&iter);
2393		-
2394		- /* configure the ba, tag, and op */
2395		- ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
2396		- obj, 0, op,
2397		- HNS_ROCE_CMD_TIMEOUT_MSECS);
	3660	+ ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type,
	3661	+ step_idx);
2398	3662	}
2399	3663	} else {
2400	3664	if (step_idx == 0)
..	..	@@ -2402,12 +3666,9 @@
2402	3666	else if (step_idx == 1 && hop_num == 2)
2403	3667	bt_ba = table->bt_l1_dma_addr[l1_idx];
2404	3668
2405		- /* configure the ba, tag, and op */
2406		- ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
2407		- 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
	3669	+ ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);
2408	3670	}
2409	3671
2410		- hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2411	3672	return ret;
2412	3673	}
2413	3674
..	..	@@ -2417,7 +3678,7 @@
2417	3678	{
2418	3679	struct device *dev = hr_dev->dev;
2419	3680	struct hns_roce_cmd_mailbox *mailbox;
2420		- int ret = 0;
	3681	+ int ret;
2421	3682	u16 op = 0xff;
2422	3683
2423	3684	if (!hns_roce_check_whether_mhop(hr_dev, table->type))
..	..	@@ -2433,14 +3694,24 @@
2433	3694	case HEM_TYPE_CQC:
2434	3695	op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
2435	3696	break;
	3697	+ case HEM_TYPE_SCCC:
	3698	+ case HEM_TYPE_QPC_TIMER:
	3699	+ case HEM_TYPE_CQC_TIMER:
	3700	+ break;
2436	3701	case HEM_TYPE_SRQC:
2437	3702	op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
2438	3703	break;
2439	3704	default:
2440		- dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
	3705	+ dev_warn(dev, "table %u not to be destroyed by mailbox!\n",
2441	3706	table->type);
2442	3707	return 0;
2443	3708	}
	3709	+
	3710	+ if (table->type == HEM_TYPE_SCCC \|\|
	3711	+ table->type == HEM_TYPE_QPC_TIMER \|\|
	3712	+ table->type == HEM_TYPE_CQC_TIMER)
	3713	+ return 0;
	3714	+
2444	3715	op += step_idx;
2445	3716
2446	3717	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
..	..	@@ -2456,20 +3727,22 @@
2456	3727	}
2457	3728
2458	3729	static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
2459		- struct hns_roce_mtt *mtt,
2460		- enum ib_qp_state cur_state,
2461		- enum ib_qp_state new_state,
2462	3730	struct hns_roce_v2_qp_context *context,
	3731	+ struct hns_roce_v2_qp_context *qpc_mask,
2463	3732	struct hns_roce_qp *hr_qp)
2464	3733	{
2465	3734	struct hns_roce_cmd_mailbox *mailbox;
	3735	+ int qpc_size;
2466	3736	int ret;
2467	3737
2468	3738	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2469	3739	if (IS_ERR(mailbox))
2470	3740	return PTR_ERR(mailbox);
2471	3741
2472		- memcpy(mailbox->buf, context, sizeof(context) 2);
	3742	+ /* The qpc size of HIP08 is only 256B, which is half of HIP09 */
	3743	+ qpc_size = hr_dev->caps.qpc_sz;
	3744	+ memcpy(mailbox->buf, context, qpc_size);
	3745	+ memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size);
2473	3746
2474	3747	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
2475	3748	HNS_ROCE_CMD_MODIFY_QPC,
..	..	@@ -2508,6 +3781,27 @@
2508	3781	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2509	3782	!!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
2510	3783	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
	3784	+ roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S,
	3785	+ !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
	3786	+ roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, 0);
	3787	+}
	3788	+
	3789	+static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
	3790	+ struct hns_roce_v2_qp_context *context,
	3791	+ struct hns_roce_v2_qp_context *qpc_mask)
	3792	+{
	3793	+ roce_set_field(context->byte_4_sqpn_tst,
	3794	+ V2_QPC_BYTE_4_SGE_SHIFT_M, V2_QPC_BYTE_4_SGE_SHIFT_S,
	3795	+ to_hr_hem_entries_shift(hr_qp->sge.sge_cnt,
	3796	+ hr_qp->sge.sge_shift));
	3797	+
	3798	+ roce_set_field(context->byte_20_smac_sgid_idx,
	3799	+ V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
	3800	+ ilog2(hr_qp->sq.wqe_cnt));
	3801	+
	3802	+ roce_set_field(context->byte_20_smac_sgid_idx,
	3803	+ V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
	3804	+ ilog2(hr_qp->rq.wqe_cnt));
2511	3805	}
2512	3806
2513	3807	static void modify_qp_reset_to_init(struct ib_qp *ibqp,
..	..	@@ -2527,310 +3821,50 @@
2527	3821	*/
2528	3822	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2529	3823	V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
2530		- roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2531		- V2_QPC_BYTE_4_TST_S, 0);
2532		-
2533		- if (ibqp->qp_type == IB_QPT_GSI)
2534		- roce_set_field(context->byte_4_sqpn_tst,
2535		- V2_QPC_BYTE_4_SGE_SHIFT_M,
2536		- V2_QPC_BYTE_4_SGE_SHIFT_S,
2537		- ilog2((unsigned int)hr_qp->sge.sge_cnt));
2538		- else
2539		- roce_set_field(context->byte_4_sqpn_tst,
2540		- V2_QPC_BYTE_4_SGE_SHIFT_M,
2541		- V2_QPC_BYTE_4_SGE_SHIFT_S,
2542		- hr_qp->sq.max_gs > 2 ?
2543		- ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2544		-
2545		- roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2546		- V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2547	3824
2548	3825	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2549	3826	V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2550		- roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2551		- V2_QPC_BYTE_4_SQPN_S, 0);
2552	3827
2553	3828	roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2554	3829	V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
2555		- roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2556		- V2_QPC_BYTE_16_PD_S, 0);
2557	3830
2558	3831	roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2559	3832	V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
2560		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2561		- V2_QPC_BYTE_20_RQWS_S, 0);
2562	3833
2563		- roce_set_field(context->byte_20_smac_sgid_idx,
2564		- V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2565		- ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2566		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2567		- V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2568		-
2569		- roce_set_field(context->byte_20_smac_sgid_idx,
2570		- V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2571		- ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2572		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2573		- V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
	3834	+ set_qpc_wqe_cnt(hr_qp, context, qpc_mask);
2574	3835
2575	3836	/* No VLAN need to set 0xFFF */
2576	3837	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
2577	3838	V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
2578		- roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
2579		- V2_QPC_BYTE_24_VLAN_ID_S, 0);
2580	3839
2581		- /*
2582		- * Set some fields in context to zero, Because the default values
2583		- * of all fields in context are zero, we need not set them to 0 again.
2584		- * but we should set the relevant fields of context mask to 0.
2585		- */
2586		- roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
2587		- roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
2588		- roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
2589		- roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
2590		-
2591		- roce_set_field(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_MAPID_M,
2592		- V2_QPC_BYTE_60_MAPID_S, 0);
2593		-
2594		- roce_set_bit(qpc_mask->byte_60_qpst_mapid,
2595		- V2_QPC_BYTE_60_INNER_MAP_IND_S, 0);
2596		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_MAP_IND_S,
2597		- 0);
2598		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_RQ_MAP_IND_S,
2599		- 0);
2600		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_EXT_MAP_IND_S,
2601		- 0);
2602		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_RLS_IND_S,
2603		- 0);
2604		- roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_EXT_IND_S,
2605		- 0);
2606		- roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
2607		- roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
2608		-
2609		- if (attr_mask & IB_QP_QKEY) {
2610		- context->qkey_xrcd = attr->qkey;
2611		- qpc_mask->qkey_xrcd = 0;
2612		- hr_qp->qkey = attr->qkey;
2613		- }
2614		-
2615		- if (hr_qp->rdb_en) {
	3840	+ if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
2616	3841	roce_set_bit(context->byte_68_rq_db,
2617	3842	V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
2618		- roce_set_bit(qpc_mask->byte_68_rq_db,
2619		- V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
2620		- }
2621	3843
2622	3844	roce_set_field(context->byte_68_rq_db,
2623	3845	V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
2624	3846	V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
2625	3847	((u32)hr_qp->rdb.dma) >> 1);
2626		- roce_set_field(qpc_mask->byte_68_rq_db,
2627		- V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
2628		- V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
2629		- context->rq_db_record_addr = hr_qp->rdb.dma >> 32;
2630		- qpc_mask->rq_db_record_addr = 0;
	3848	+ context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32);
2631	3849
2632	3850	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
2633	3851	(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
2634		- roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
2635	3852
2636	3853	roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2637	3854	V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
2638		- roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2639		- V2_QPC_BYTE_80_RX_CQN_S, 0);
2640	3855	if (ibqp->srq) {
2641	3856	roce_set_field(context->byte_76_srqn_op_en,
2642	3857	V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
2643	3858	to_hr_srq(ibqp->srq)->srqn);
2644		- roce_set_field(qpc_mask->byte_76_srqn_op_en,
2645		- V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2646	3859	roce_set_bit(context->byte_76_srqn_op_en,
2647	3860	V2_QPC_BYTE_76_SRQ_EN_S, 1);
2648		- roce_set_bit(qpc_mask->byte_76_srqn_op_en,
2649		- V2_QPC_BYTE_76_SRQ_EN_S, 0);
2650	3861	}
2651	3862
2652		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2653		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2654		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
2655		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2656		- V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
2657		- V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
2658		-
2659		- roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
2660		- V2_QPC_BYTE_92_SRQ_INFO_S, 0);
2661		-
2662		- roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
2663		- V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
2664		-
2665		- roce_set_field(qpc_mask->byte_104_rq_sge,
2666		- V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
2667		- V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
2668		-
2669		- roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2670		- V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
2671		- roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2672		- V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
2673		- V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
2674		- roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2675		- V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
2676		-
2677		- qpc_mask->rq_rnr_timer = 0;
2678		- qpc_mask->rx_msg_len = 0;
2679		- qpc_mask->rx_rkey_pkt_info = 0;
2680		- qpc_mask->rx_va = 0;
2681		-
2682		- roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
2683		- V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
2684		- roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
2685		- V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
2686		-
2687		- roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RSVD_RAQ_MAP_S, 0);
2688		- roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
2689		- V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
2690		- roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
2691		- V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
2692		-
2693		- roce_set_field(qpc_mask->byte_144_raq,
2694		- V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
2695		- V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
2696		- roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_RTY_INI_IND_S,
2697		- 0);
2698		- roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
2699		- V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
2700		- roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
2701		-
2702		- roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
2703		- V2_QPC_BYTE_148_RQ_MSN_S, 0);
2704		- roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
2705		- V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
2706		-
2707		- roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2708		- V2_QPC_BYTE_152_RAQ_PSN_S, 0);
2709		- roce_set_field(qpc_mask->byte_152_raq,
2710		- V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
2711		- V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
2712		-
2713		- roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
2714		- V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
2715		-
2716		- roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2717		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
2718		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
2719		- roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2720		- V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
2721		- V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
2722		-
2723		- roce_set_field(context->byte_168_irrl_idx,
2724		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2725		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_S,
2726		- ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2727		- roce_set_field(qpc_mask->byte_168_irrl_idx,
2728		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2729		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0);
2730		-
2731		- roce_set_bit(qpc_mask->byte_168_irrl_idx,
2732		- V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
2733		- roce_set_bit(qpc_mask->byte_168_irrl_idx,
2734		- V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
2735		- roce_set_field(qpc_mask->byte_168_irrl_idx,
2736		- V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
2737		- V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
2738		-
2739		- roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2740		- V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
2741		- roce_set_field(qpc_mask->byte_172_sq_psn,
2742		- V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2743		- V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
2744		-
2745		- roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
2746		- 0);
2747		-
2748		- roce_set_field(qpc_mask->byte_176_msg_pktn,
2749		- V2_QPC_BYTE_176_MSG_USE_PKTN_M,
2750		- V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
2751		- roce_set_field(qpc_mask->byte_176_msg_pktn,
2752		- V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
2753		- V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
2754		-
2755		- roce_set_field(qpc_mask->byte_184_irrl_idx,
2756		- V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
2757		- V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
2758		-
2759		- qpc_mask->cur_sge_offset = 0;
2760		-
2761		- roce_set_field(qpc_mask->byte_192_ext_sge,
2762		- V2_QPC_BYTE_192_CUR_SGE_IDX_M,
2763		- V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
2764		- roce_set_field(qpc_mask->byte_192_ext_sge,
2765		- V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
2766		- V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
2767		-
2768		- roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
2769		- V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
2770		-
2771		- roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
2772		- V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
2773		- roce_set_field(qpc_mask->byte_200_sq_max,
2774		- V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
2775		- V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
2776		-
2777		- roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
2778		- roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
2779		-
2780		- roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
2781		- V2_QPC_BYTE_212_CHECK_FLG_S, 0);
2782		-
2783		- qpc_mask->sq_timer = 0;
2784		-
2785		- roce_set_field(qpc_mask->byte_220_retry_psn_msn,
2786		- V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
2787		- V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
2788		- roce_set_field(qpc_mask->byte_232_irrl_sge,
2789		- V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
2790		- V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
2791		-
2792		- qpc_mask->irrl_cur_sge_offset = 0;
2793		-
2794		- roce_set_field(qpc_mask->byte_240_irrl_tail,
2795		- V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
2796		- V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
2797		- roce_set_field(qpc_mask->byte_240_irrl_tail,
2798		- V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
2799		- V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
2800		- roce_set_field(qpc_mask->byte_240_irrl_tail,
2801		- V2_QPC_BYTE_240_RX_ACK_MSN_M,
2802		- V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
2803		-
2804		- roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
2805		- V2_QPC_BYTE_248_IRRL_PSN_S, 0);
2806		- roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
2807		- 0);
2808		- roce_set_field(qpc_mask->byte_248_ack_psn,
2809		- V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
2810		- V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
2811		- roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
2812		- 0);
2813		- roce_set_bit(qpc_mask->byte_248_ack_psn,
2814		- V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
2815		- roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
2816		- 0);
	3863	+ roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
2817	3864
2818	3865	hr_qp->access_flags = attr->qp_access_flags;
2819		- hr_qp->pkey_index = attr->pkey_index;
2820	3866	roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2821	3867	V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
2822		- roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2823		- V2_QPC_BYTE_252_TX_CQN_S, 0);
2824		-
2825		- roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
2826		- V2_QPC_BYTE_252_ERR_TYPE_S, 0);
2827		-
2828		- roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
2829		- V2_QPC_BYTE_256_RQ_CQE_IDX_M,
2830		- V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
2831		- roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
2832		- V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
2833		- V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
2834	3868	}
2835	3869
2836	3870	static void modify_qp_init_to_init(struct ib_qp *ibqp,
..	..	@@ -2851,20 +3885,6 @@
2851	3885	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2852	3886	V2_QPC_BYTE_4_TST_S, 0);
2853	3887
2854		- if (ibqp->qp_type == IB_QPT_GSI)
2855		- roce_set_field(context->byte_4_sqpn_tst,
2856		- V2_QPC_BYTE_4_SGE_SHIFT_M,
2857		- V2_QPC_BYTE_4_SGE_SHIFT_S,
2858		- ilog2((unsigned int)hr_qp->sge.sge_cnt));
2859		- else
2860		- roce_set_field(context->byte_4_sqpn_tst,
2861		- V2_QPC_BYTE_4_SGE_SHIFT_M,
2862		- V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
2863		- ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2864		-
2865		- roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2866		- V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2867		-
2868	3888	if (attr_mask & IB_QP_ACCESS_FLAGS) {
2869	3889	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2870	3890	!!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
..	..	@@ -2882,6 +3902,12 @@
2882	3902	IB_ACCESS_REMOTE_ATOMIC));
2883	3903	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2884	3904	0);
	3905	+ roce_set_bit(context->byte_76_srqn_op_en,
	3906	+ V2_QPC_BYTE_76_EXT_ATE_S,
	3907	+ !!(attr->qp_access_flags &
	3908	+ IB_ACCESS_REMOTE_ATOMIC));
	3909	+ roce_set_bit(qpc_mask->byte_76_srqn_op_en,
	3910	+ V2_QPC_BYTE_76_EXT_ATE_S, 0);
2885	3911	} else {
2886	3912	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2887	3913	!!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
..	..	@@ -2897,19 +3923,12 @@
2897	3923	!!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
2898	3924	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2899	3925	0);
	3926	+ roce_set_bit(context->byte_76_srqn_op_en,
	3927	+ V2_QPC_BYTE_76_EXT_ATE_S,
	3928	+ !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
	3929	+ roce_set_bit(qpc_mask->byte_76_srqn_op_en,
	3930	+ V2_QPC_BYTE_76_EXT_ATE_S, 0);
2900	3931	}
2901		-
2902		- roce_set_field(context->byte_20_smac_sgid_idx,
2903		- V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2904		- ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2905		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2906		- V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2907		-
2908		- roce_set_field(context->byte_20_smac_sgid_idx,
2909		- V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2910		- ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2911		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2912		- V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
2913	3932
2914	3933	roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2915	3934	V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
..	..	@@ -2938,11 +3957,6 @@
2938	3957	V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2939	3958	}
2940	3959
2941		- if (attr_mask & IB_QP_QKEY) {
2942		- context->qkey_xrcd = attr->qkey;
2943		- qpc_mask->qkey_xrcd = 0;
2944		- }
2945		-
2946	3960	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2947	3961	V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2948	3962	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
..	..	@@ -2954,13 +3968,194 @@
2954	3968	roce_set_field(qpc_mask->byte_56_dqpn_err,
2955	3969	V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
2956	3970	}
	3971	+}
	3972	+
	3973	+static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
	3974	+ struct hns_roce_qp *hr_qp,
	3975	+ struct hns_roce_v2_qp_context *context,
	3976	+ struct hns_roce_v2_qp_context *qpc_mask)
	3977	+{
	3978	+ u64 mtts[MTT_MIN_COUNT] = { 0 };
	3979	+ u64 wqe_sge_ba;
	3980	+ int count;
	3981	+
	3982	+ /* Search qp buf's mtts */
	3983	+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
	3984	+ MTT_MIN_COUNT, &wqe_sge_ba);
	3985	+ if (hr_qp->rq.wqe_cnt && count < 1) {
	3986	+ ibdev_err(&hr_dev->ib_dev,
	3987	+ "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn);
	3988	+ return -EINVAL;
	3989	+ }
	3990	+
	3991	+ context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
	3992	+ qpc_mask->wqe_sge_ba = 0;
	3993	+
	3994	+ /*
	3995	+ * In v2 engine, software pass context and context mask to hardware
	3996	+ * when modifying qp. If software need modify some fields in context,
	3997	+ * we should set all bits of the relevant fields in context mask to
	3998	+ * 0 at the same time, else set them to 0x1.
	3999	+ */
	4000	+ roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
	4001	+ V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
	4002	+ roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
	4003	+ V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
	4004	+
	4005	+ roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
	4006	+ V2_QPC_BYTE_12_SQ_HOP_NUM_S,
	4007	+ to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num,
	4008	+ hr_qp->sq.wqe_cnt));
	4009	+ roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
	4010	+ V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
	4011	+
	4012	+ roce_set_field(context->byte_20_smac_sgid_idx,
	4013	+ V2_QPC_BYTE_20_SGE_HOP_NUM_M,
	4014	+ V2_QPC_BYTE_20_SGE_HOP_NUM_S,
	4015	+ to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num,
	4016	+ hr_qp->sge.sge_cnt));
	4017	+ roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
	4018	+ V2_QPC_BYTE_20_SGE_HOP_NUM_M,
	4019	+ V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
	4020	+
	4021	+ roce_set_field(context->byte_20_smac_sgid_idx,
	4022	+ V2_QPC_BYTE_20_RQ_HOP_NUM_M,
	4023	+ V2_QPC_BYTE_20_RQ_HOP_NUM_S,
	4024	+ to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num,
	4025	+ hr_qp->rq.wqe_cnt));
	4026	+
	4027	+ roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
	4028	+ V2_QPC_BYTE_20_RQ_HOP_NUM_M,
	4029	+ V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
	4030	+
	4031	+ roce_set_field(context->byte_16_buf_ba_pg_sz,
	4032	+ V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
	4033	+ V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
	4034	+ to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift));
	4035	+ roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
	4036	+ V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
	4037	+ V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
	4038	+
	4039	+ roce_set_field(context->byte_16_buf_ba_pg_sz,
	4040	+ V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
	4041	+ V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
	4042	+ to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift));
	4043	+ roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
	4044	+ V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
	4045	+ V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
	4046	+
	4047	+ context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
	4048	+ qpc_mask->rq_cur_blk_addr = 0;
	4049	+
	4050	+ roce_set_field(context->byte_92_srq_info,
	4051	+ V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
	4052	+ V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
	4053	+ upper_32_bits(to_hr_hw_page_addr(mtts[0])));
	4054	+ roce_set_field(qpc_mask->byte_92_srq_info,
	4055	+ V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
	4056	+ V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
	4057	+
	4058	+ context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
	4059	+ qpc_mask->rq_nxt_blk_addr = 0;
	4060	+
	4061	+ roce_set_field(context->byte_104_rq_sge,
	4062	+ V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
	4063	+ V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
	4064	+ upper_32_bits(to_hr_hw_page_addr(mtts[1])));
	4065	+ roce_set_field(qpc_mask->byte_104_rq_sge,
	4066	+ V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
	4067	+ V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
	4068	+
	4069	+ roce_set_field(context->byte_84_rq_ci_pi,
	4070	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
	4071	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
	4072	+ roce_set_field(qpc_mask->byte_84_rq_ci_pi,
	4073	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
	4074	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
	4075	+
	4076	+ roce_set_field(qpc_mask->byte_84_rq_ci_pi,
	4077	+ V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
	4078	+ V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
	4079	+
	4080	+ return 0;
	4081	+}
	4082	+
	4083	+static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,
	4084	+ struct hns_roce_qp *hr_qp,
	4085	+ struct hns_roce_v2_qp_context *context,
	4086	+ struct hns_roce_v2_qp_context *qpc_mask)
	4087	+{
	4088	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4089	+ u64 sge_cur_blk = 0;
	4090	+ u64 sq_cur_blk = 0;
	4091	+ int count;
	4092	+
	4093	+ /* search qp buf's mtts */
	4094	+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
	4095	+ if (count < 1) {
	4096	+ ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n",
	4097	+ hr_qp->qpn);
	4098	+ return -EINVAL;
	4099	+ }
	4100	+ if (hr_qp->sge.sge_cnt > 0) {
	4101	+ count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
	4102	+ hr_qp->sge.offset,
	4103	+ &sge_cur_blk, 1, NULL);
	4104	+ if (count < 1) {
	4105	+ ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",
	4106	+ hr_qp->qpn);
	4107	+ return -EINVAL;
	4108	+ }
	4109	+ }
	4110	+
	4111	+ /*
	4112	+ * In v2 engine, software pass context and context mask to hardware
	4113	+ * when modifying qp. If software need modify some fields in context,
	4114	+ * we should set all bits of the relevant fields in context mask to
	4115	+ * 0 at the same time, else set them to 0x1.
	4116	+ */
	4117	+ context->sq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk));
2957	4118	roce_set_field(context->byte_168_irrl_idx,
2958		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2959		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_S,
2960		- ilog2((unsigned int)hr_qp->sq.wqe_cnt));
	4119	+ V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
	4120	+ V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
	4121	+ upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
	4122	+ qpc_mask->sq_cur_blk_addr = 0;
2961	4123	roce_set_field(qpc_mask->byte_168_irrl_idx,
2962		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2963		- V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0);
	4124	+ V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
	4125	+ V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
	4126	+
	4127	+ context->sq_cur_sge_blk_addr =
	4128	+ cpu_to_le32(to_hr_hw_page_addr(sge_cur_blk));
	4129	+ roce_set_field(context->byte_184_irrl_idx,
	4130	+ V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
	4131	+ V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
	4132	+ upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
	4133	+ qpc_mask->sq_cur_sge_blk_addr = 0;
	4134	+ roce_set_field(qpc_mask->byte_184_irrl_idx,
	4135	+ V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
	4136	+ V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
	4137	+
	4138	+ context->rx_sq_cur_blk_addr =
	4139	+ cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk));
	4140	+ roce_set_field(context->byte_232_irrl_sge,
	4141	+ V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
	4142	+ V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
	4143	+ upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
	4144	+ qpc_mask->rx_sq_cur_blk_addr = 0;
	4145	+ roce_set_field(qpc_mask->byte_232_irrl_sge,
	4146	+ V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
	4147	+ V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
	4148	+
	4149	+ return 0;
	4150	+}
	4151	+
	4152	+static inline enum ib_mtu get_mtu(struct ib_qp *ibqp,
	4153	+ const struct ib_qp_attr *attr)
	4154	+{
	4155	+ if (ibqp->qp_type == IB_QPT_GSI \|\| ibqp->qp_type == IB_QPT_UD)
	4156	+ return IB_MTU_4096;
	4157	+
	4158	+ return attr->path_mtu;
2964	4159	}
2965	4160
2966	4161	static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
..	..	@@ -2971,162 +4166,63 @@
2971	4166	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2972	4167	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2973	4168	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2974		- struct device *dev = hr_dev->dev;
2975		- dma_addr_t dma_handle_3;
2976		- dma_addr_t dma_handle_2;
2977		- dma_addr_t dma_handle;
2978		- u32 page_size;
	4169	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4170	+ dma_addr_t trrl_ba;
	4171	+ dma_addr_t irrl_ba;
	4172	+ enum ib_mtu mtu;
	4173	+ u8 lp_pktn_ini;
2979	4174	u8 port_num;
2980		- u64 *mtts_3;
2981		- u64 *mtts_2;
2982	4175	u64 *mtts;
2983	4176	u8 *dmac;
2984	4177	u8 *smac;
2985	4178	int port;
	4179	+ int ret;
2986	4180
2987		- /* Search qp buf's mtts */
2988		- mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
2989		- hr_qp->mtt.first_seg, &dma_handle);
2990		- if (!mtts) {
2991		- dev_err(dev, "qp buf pa find failed\n");
2992		- return -EINVAL;
	4181	+ ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);
	4182	+ if (ret) {
	4183	+ ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);
	4184	+ return ret;
2993	4185	}
2994	4186
2995	4187	/* Search IRRL's mtts */
2996		- mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
2997		- hr_qp->qpn, &dma_handle_2);
2998		- if (!mtts_2) {
2999		- dev_err(dev, "qp irrl_table find failed\n");
	4188	+ mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
	4189	+ hr_qp->qpn, &irrl_ba);
	4190	+ if (!mtts) {
	4191	+ ibdev_err(ibdev, "failed to find qp irrl_table.\n");
3000	4192	return -EINVAL;
3001	4193	}
3002	4194
3003	4195	/* Search TRRL's mtts */
3004		- mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
3005		- hr_qp->qpn, &dma_handle_3);
3006		- if (!mtts_3) {
3007		- dev_err(dev, "qp trrl_table find failed\n");
	4196	+ mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
	4197	+ hr_qp->qpn, &trrl_ba);
	4198	+ if (!mtts) {
	4199	+ ibdev_err(ibdev, "failed to find qp trrl_table.\n");
3008	4200	return -EINVAL;
3009	4201	}
3010	4202
3011	4203	if (attr_mask & IB_QP_ALT_PATH) {
3012		- dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
	4204	+ ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",
	4205	+ attr_mask);
3013	4206	return -EINVAL;
3014	4207	}
3015	4208
3016		- dmac = (u8 *)attr->ah_attr.roce.dmac;
3017		- context->wqe_sge_ba = (u32)(dma_handle >> 3);
3018		- qpc_mask->wqe_sge_ba = 0;
3019		-
3020		- /*
3021		- * In v2 engine, software pass context and context mask to hardware
3022		- * when modifying qp. If software need modify some fields in context,
3023		- * we should set all bits of the relevant fields in context mask to
3024		- * 0 at the same time, else set them to 0x1.
3025		- */
3026		- roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3027		- V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3));
3028		- roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3029		- V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
3030		-
3031		- roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3032		- V2_QPC_BYTE_12_SQ_HOP_NUM_S,
3033		- hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
3034		- 0 : hr_dev->caps.mtt_hop_num);
3035		- roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3036		- V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
3037		-
3038		- roce_set_field(context->byte_20_smac_sgid_idx,
3039		- V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3040		- V2_QPC_BYTE_20_SGE_HOP_NUM_S,
3041		- ((ibqp->qp_type == IB_QPT_GSI) \|\| hr_qp->sq.max_gs > 2) ?
3042		- hr_dev->caps.mtt_hop_num : 0);
3043		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3044		- V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3045		- V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
3046		-
3047		- roce_set_field(context->byte_20_smac_sgid_idx,
3048		- V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3049		- V2_QPC_BYTE_20_RQ_HOP_NUM_S,
3050		- hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
3051		- 0 : hr_dev->caps.mtt_hop_num);
3052		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3053		- V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3054		- V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
3055		-
3056		- roce_set_field(context->byte_16_buf_ba_pg_sz,
3057		- V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3058		- V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
3059		- hr_dev->caps.mtt_ba_pg_sz + PG_SHIFT_OFFSET);
3060		- roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3061		- V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3062		- V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
3063		-
3064		- roce_set_field(context->byte_16_buf_ba_pg_sz,
3065		- V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3066		- V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
3067		- hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
3068		- roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3069		- V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3070		- V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
3071		-
3072		- roce_set_field(context->byte_80_rnr_rx_cqn,
3073		- V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3074		- V2_QPC_BYTE_80_MIN_RNR_TIME_S, attr->min_rnr_timer);
3075		- roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
3076		- V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3077		- V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
3078		-
3079		- page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3080		- context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size]
3081		- >> PAGE_ADDR_SHIFT);
3082		- qpc_mask->rq_cur_blk_addr = 0;
3083		-
3084		- roce_set_field(context->byte_92_srq_info,
3085		- V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3086		- V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
3087		- mtts[hr_qp->rq.offset / page_size]
3088		- >> (32 + PAGE_ADDR_SHIFT));
3089		- roce_set_field(qpc_mask->byte_92_srq_info,
3090		- V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3091		- V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
3092		-
3093		- context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1]
3094		- >> PAGE_ADDR_SHIFT);
3095		- qpc_mask->rq_nxt_blk_addr = 0;
3096		-
3097		- roce_set_field(context->byte_104_rq_sge,
3098		- V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3099		- V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
3100		- mtts[hr_qp->rq.offset / page_size + 1]
3101		- >> (32 + PAGE_ADDR_SHIFT));
3102		- roce_set_field(qpc_mask->byte_104_rq_sge,
3103		- V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3104		- V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
3105		-
3106		- roce_set_field(context->byte_108_rx_reqepsn,
3107		- V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3108		- V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
3109		- roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3110		- V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3111		- V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
3112		-
3113	4209	roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3114		- V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
	4210	+ V2_QPC_BYTE_132_TRRL_BA_S, trrl_ba >> 4);
3115	4211	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3116	4212	V2_QPC_BYTE_132_TRRL_BA_S, 0);
3117		- context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4));
	4213	+ context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4));
3118	4214	qpc_mask->trrl_ba = 0;
3119	4215	roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3120	4216	V2_QPC_BYTE_140_TRRL_BA_S,
3121		- (u32)(dma_handle_3 >> (32 + 16 + 4)));
	4217	+ (u32)(trrl_ba >> (32 + 16 + 4)));
3122	4218	roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3123	4219	V2_QPC_BYTE_140_TRRL_BA_S, 0);
3124	4220
3125		- context->irrl_ba = (u32)(dma_handle_2 >> 6);
	4221	+ context->irrl_ba = cpu_to_le32(irrl_ba >> 6);
3126	4222	qpc_mask->irrl_ba = 0;
3127	4223	roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3128	4224	V2_QPC_BYTE_208_IRRL_BA_S,
3129		- dma_handle_2 >> (32 + 6));
	4225	+ irrl_ba >> (32 + 6));
3130	4226	roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3131	4227	V2_QPC_BYTE_208_IRRL_BA_S, 0);
3132	4228
..	..	@@ -3141,20 +4237,12 @@
3141	4237	port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
3142	4238
3143	4239	smac = (u8 *)hr_dev->dev_addr[port];
	4240	+ dmac = (u8 *)attr->ah_attr.roce.dmac;
3144	4241	/* when dmac equals smac or loop_idc is 1, it should loopback */
3145	4242	if (ether_addr_equal_unaligned(dmac, smac) \|\|
3146	4243	hr_dev->loop_idc == 0x1) {
3147	4244	roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
3148	4245	roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
3149		- }
3150		-
3151		- if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
3152		- attr->max_dest_rd_atomic) {
3153		- roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
3154		- V2_QPC_BYTE_140_RR_MAX_S,
3155		- fls(attr->max_dest_rd_atomic - 1));
3156		- roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
3157		- V2_QPC_BYTE_140_RR_MAX_S, 0);
3158	4246	}
3159	4247
3160	4248	if (attr_mask & IB_QP_DEST_QPN) {
..	..	@@ -3167,45 +4255,45 @@
3167	4255	/* Configure GID index */
3168	4256	port_num = rdma_ah_get_port_num(&attr->ah_attr);
3169	4257	roce_set_field(context->byte_20_smac_sgid_idx,
3170		- V2_QPC_BYTE_20_SGID_IDX_M,
3171		- V2_QPC_BYTE_20_SGID_IDX_S,
	4258	+ V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
3172	4259	hns_get_gid_index(hr_dev, port_num - 1,
3173	4260	grh->sgid_index));
3174	4261	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3175		- V2_QPC_BYTE_20_SGID_IDX_M,
3176		- V2_QPC_BYTE_20_SGID_IDX_S, 0);
3177		- memcpy(&(context->dmac), dmac, 4);
	4262	+ V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
	4263	+
	4264	+ memcpy(&(context->dmac), dmac, sizeof(u32));
3178	4265	roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3179	4266	V2_QPC_BYTE_52_DMAC_S, ((u16 )(&dmac[4])));
3180	4267	qpc_mask->dmac = 0;
3181	4268	roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3182	4269	V2_QPC_BYTE_52_DMAC_S, 0);
3183	4270
	4271	+ mtu = get_mtu(ibqp, attr);
	4272	+ hr_qp->path_mtu = mtu;
	4273	+
	4274	+ if (attr_mask & IB_QP_PATH_MTU) {
	4275	+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
	4276	+ V2_QPC_BYTE_24_MTU_S, mtu);
	4277	+ roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
	4278	+ V2_QPC_BYTE_24_MTU_S, 0);
	4279	+ }
	4280	+
	4281	+#define MAX_LP_MSG_LEN 65536
	4282	+ /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 64KB */
	4283	+ lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / ib_mtu_enum_to_int(mtu));
	4284	+
3184	4285	roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3185		- V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
	4286	+ V2_QPC_BYTE_56_LP_PKTN_INI_S, lp_pktn_ini);
3186	4287	roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3187	4288	V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
3188	4289
3189		- if (ibqp->qp_type == IB_QPT_GSI \|\| ibqp->qp_type == IB_QPT_UD)
3190		- roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3191		- V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
3192		- else if (attr_mask & IB_QP_PATH_MTU)
3193		- roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3194		- V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
	4290	+ /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */
	4291	+ roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
	4292	+ V2_QPC_BYTE_172_ACK_REQ_FREQ_S, lp_pktn_ini);
	4293	+ roce_set_field(qpc_mask->byte_172_sq_psn,
	4294	+ V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
	4295	+ V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
3195	4296
3196		- roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3197		- V2_QPC_BYTE_24_MTU_S, 0);
3198		-
3199		- roce_set_field(context->byte_84_rq_ci_pi,
3200		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3201		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
3202		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3203		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3204		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3205		-
3206		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3207		- V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3208		- V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3209	4297	roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3210	4298	V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3211	4299	roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
..	..	@@ -3217,16 +4305,12 @@
3217	4305	context->rq_rnr_timer = 0;
3218	4306	qpc_mask->rq_rnr_timer = 0;
3219	4307
3220		- roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3221		- V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
3222		- roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3223		- V2_QPC_BYTE_152_RAQ_PSN_S, 0);
3224		-
3225	4308	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3226	4309	V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3227	4310	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3228	4311	V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3229	4312
	4313	+ /* rocee send 2^lp_sgen_ini segs every time */
3230	4314	roce_set_field(context->byte_168_irrl_idx,
3231	4315	V2_QPC_BYTE_168_LP_SGEN_INI_M,
3232	4316	V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
..	..	@@ -3244,74 +4328,20 @@
3244	4328	{
3245	4329	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3246	4330	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3247		- struct device *dev = hr_dev->dev;
3248		- dma_addr_t dma_handle;
3249		- u32 page_size;
3250		- u64 *mtts;
3251		-
3252		- /* Search qp buf's mtts */
3253		- mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
3254		- hr_qp->mtt.first_seg, &dma_handle);
3255		- if (!mtts) {
3256		- dev_err(dev, "qp buf pa find failed\n");
3257		- return -EINVAL;
3258		- }
	4331	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4332	+ int ret;
3259	4333
3260	4334	/* Not support alternate path and path migration */
3261		- if ((attr_mask & IB_QP_ALT_PATH) \|\|
3262		- (attr_mask & IB_QP_PATH_MIG_STATE)) {
3263		- dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
	4335	+ if (attr_mask & (IB_QP_ALT_PATH \| IB_QP_PATH_MIG_STATE)) {
	4336	+ ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
3264	4337	return -EINVAL;
3265	4338	}
3266	4339
3267		- /*
3268		- * In v2 engine, software pass context and context mask to hardware
3269		- * when modifying qp. If software need modify some fields in context,
3270		- * we should set all bits of the relevant fields in context mask to
3271		- * 0 at the same time, else set them to 0x1.
3272		- */
3273		- roce_set_field(context->byte_60_qpst_mapid,
3274		- V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M,
3275		- V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S, attr->retry_cnt);
3276		- roce_set_field(qpc_mask->byte_60_qpst_mapid,
3277		- V2_QPC_BYTE_60_RTY_NUM_INI_BAK_M,
3278		- V2_QPC_BYTE_60_RTY_NUM_INI_BAK_S, 0);
3279		-
3280		- context->sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
3281		- roce_set_field(context->byte_168_irrl_idx,
3282		- V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3283		- V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
3284		- mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3285		- qpc_mask->sq_cur_blk_addr = 0;
3286		- roce_set_field(qpc_mask->byte_168_irrl_idx,
3287		- V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3288		- V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
3289		-
3290		- page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3291		- context->sq_cur_sge_blk_addr =
3292		- ((ibqp->qp_type == IB_QPT_GSI) \|\| hr_qp->sq.max_gs > 2) ?
3293		- ((u32)(mtts[hr_qp->sge.offset / page_size]
3294		- >> PAGE_ADDR_SHIFT)) : 0;
3295		- roce_set_field(context->byte_184_irrl_idx,
3296		- V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3297		- V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
3298		- ((ibqp->qp_type == IB_QPT_GSI) \|\| hr_qp->sq.max_gs > 2) ?
3299		- (mtts[hr_qp->sge.offset / page_size] >>
3300		- (32 + PAGE_ADDR_SHIFT)) : 0);
3301		- qpc_mask->sq_cur_sge_blk_addr = 0;
3302		- roce_set_field(qpc_mask->byte_184_irrl_idx,
3303		- V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3304		- V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
3305		-
3306		- context->rx_sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
3307		- roce_set_field(context->byte_232_irrl_sge,
3308		- V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3309		- V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
3310		- mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3311		- qpc_mask->rx_sq_cur_blk_addr = 0;
3312		- roce_set_field(qpc_mask->byte_232_irrl_sge,
3313		- V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3314		- V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
	4340	+ ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);
	4341	+ if (ret) {
	4342	+ ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);
	4343	+ return ret;
	4344	+ }
3315	4345
3316	4346	/*
3317	4347	* Set some fields in context to zero, Because the default values
..	..	@@ -3326,13 +4356,6 @@
3326	4356	V2_QPC_BYTE_240_RX_ACK_MSN_M,
3327	4357	V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3328	4358
3329		- roce_set_field(context->byte_244_rnr_rxack,
3330		- V2_QPC_BYTE_244_RX_ACK_EPSN_M,
3331		- V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
3332		- roce_set_field(qpc_mask->byte_244_rnr_rxack,
3333		- V2_QPC_BYTE_244_RX_ACK_EPSN_M,
3334		- V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
3335		-
3336	4359	roce_set_field(qpc_mask->byte_248_ack_psn,
3337	4360	V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3338	4361	V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
..	..	@@ -3346,27 +4369,6 @@
3346	4369	V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3347	4370	V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3348	4371
3349		- roce_set_field(context->byte_220_retry_psn_msn,
3350		- V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
3351		- V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
3352		- roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3353		- V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
3354		- V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
3355		-
3356		- roce_set_field(context->byte_224_retry_msg,
3357		- V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
3358		- V2_QPC_BYTE_224_RETRY_MSG_PSN_S, attr->sq_psn >> 16);
3359		- roce_set_field(qpc_mask->byte_224_retry_msg,
3360		- V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
3361		- V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
3362		-
3363		- roce_set_field(context->byte_224_retry_msg,
3364		- V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
3365		- V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, attr->sq_psn);
3366		- roce_set_field(qpc_mask->byte_224_retry_msg,
3367		- V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
3368		- V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
3369		-
3370	4372	roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3371	4373	V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3372	4374	V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
..	..	@@ -3377,51 +4379,311 @@
3377	4379	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3378	4380	V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3379	4381
3380		- roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M,
3381		- V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt);
3382		- roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M,
3383		- V2_QPC_BYTE_212_RETRY_CNT_S, 0);
3384		-
3385		- roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
3386		- V2_QPC_BYTE_212_RETRY_NUM_INIT_S, attr->retry_cnt);
3387		- roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
3388		- V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
3389		-
3390		- roce_set_field(context->byte_244_rnr_rxack,
3391		- V2_QPC_BYTE_244_RNR_NUM_INIT_M,
3392		- V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
3393		- roce_set_field(qpc_mask->byte_244_rnr_rxack,
3394		- V2_QPC_BYTE_244_RNR_NUM_INIT_M,
3395		- V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
3396		-
3397		- roce_set_field(context->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M,
3398		- V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
3399		- roce_set_field(qpc_mask->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M,
3400		- V2_QPC_BYTE_244_RNR_CNT_S, 0);
3401		-
3402	4382	roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3403	4383	V2_QPC_BYTE_212_LSN_S, 0x100);
3404	4384	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3405	4385	V2_QPC_BYTE_212_LSN_S, 0);
3406	4386
3407		- if (attr_mask & IB_QP_TIMEOUT) {
3408		- roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_AT_M,
3409		- V2_QPC_BYTE_28_AT_S, attr->timeout);
3410		- roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_AT_M,
3411		- V2_QPC_BYTE_28_AT_S, 0);
3412		- }
3413		-
3414		- roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3415		- V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
3416		- roce_set_field(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3417		- V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
3418		-
3419	4387	roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3420	4388	V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3421		- roce_set_field(context->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
3422		- V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
3423		- roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
3424		- V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
	4389	+
	4390	+ return 0;
	4391	+}
	4392	+
	4393	+static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
	4394	+{
	4395	+ if (!fl)
	4396	+ fl = rdma_calc_flow_label(lqpn, rqpn);
	4397	+
	4398	+ return rdma_flow_label_to_udp_sport(fl);
	4399	+}
	4400	+
	4401	+static int hns_roce_v2_set_path(struct ib_qp *ibqp,
	4402	+ const struct ib_qp_attr *attr,
	4403	+ int attr_mask,
	4404	+ struct hns_roce_v2_qp_context *context,
	4405	+ struct hns_roce_v2_qp_context *qpc_mask)
	4406	+{
	4407	+ const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
	4408	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4409	+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
	4410	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4411	+ const struct ib_gid_attr *gid_attr = NULL;
	4412	+ int is_roce_protocol;
	4413	+ u16 vlan_id = 0xffff;
	4414	+ bool is_udp = false;
	4415	+ u8 ib_port;
	4416	+ u8 hr_port;
	4417	+ int ret;
	4418	+
	4419	+ ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
	4420	+ hr_port = ib_port - 1;
	4421	+ is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
	4422	+ rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
	4423	+
	4424	+ if (is_roce_protocol) {
	4425	+ gid_attr = attr->ah_attr.grh.sgid_attr;
	4426	+ ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
	4427	+ if (ret)
	4428	+ return ret;
	4429	+
	4430	+ if (gid_attr)
	4431	+ is_udp = (gid_attr->gid_type ==
	4432	+ IB_GID_TYPE_ROCE_UDP_ENCAP);
	4433	+ }
	4434	+
	4435	+ if (vlan_id < VLAN_N_VID) {
	4436	+ roce_set_bit(context->byte_76_srqn_op_en,
	4437	+ V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
	4438	+ roce_set_bit(qpc_mask->byte_76_srqn_op_en,
	4439	+ V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
	4440	+ roce_set_bit(context->byte_168_irrl_idx,
	4441	+ V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
	4442	+ roce_set_bit(qpc_mask->byte_168_irrl_idx,
	4443	+ V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
	4444	+ }
	4445	+
	4446	+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
	4447	+ V2_QPC_BYTE_24_VLAN_ID_S, vlan_id);
	4448	+ roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
	4449	+ V2_QPC_BYTE_24_VLAN_ID_S, 0);
	4450	+
	4451	+ if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
	4452	+ ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
	4453	+ grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
	4454	+ return -EINVAL;
	4455	+ }
	4456	+
	4457	+ if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
	4458	+ ibdev_err(ibdev, "ah attr is not RDMA roce type\n");
	4459	+ return -EINVAL;
	4460	+ }
	4461	+
	4462	+ roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
	4463	+ V2_QPC_BYTE_52_UDPSPN_S,
	4464	+ is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num,
	4465	+ attr->dest_qp_num) : 0);
	4466	+
	4467	+ roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
	4468	+ V2_QPC_BYTE_52_UDPSPN_S, 0);
	4469	+
	4470	+ roce_set_field(context->byte_20_smac_sgid_idx,
	4471	+ V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
	4472	+ grh->sgid_index);
	4473	+
	4474	+ roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
	4475	+ V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
	4476	+
	4477	+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
	4478	+ V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
	4479	+ roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
	4480	+ V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
	4481	+
	4482	+ roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
	4483	+ V2_QPC_BYTE_24_TC_S, get_tclass(&attr->ah_attr.grh));
	4484	+ roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
	4485	+ V2_QPC_BYTE_24_TC_S, 0);
	4486	+
	4487	+ roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
	4488	+ V2_QPC_BYTE_28_FL_S, grh->flow_label);
	4489	+ roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
	4490	+ V2_QPC_BYTE_28_FL_S, 0);
	4491	+ memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
	4492	+ memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
	4493	+
	4494	+ hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
	4495	+ if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) {
	4496	+ ibdev_err(ibdev,
	4497	+ "failed to fill QPC, sl (%d) shouldn't be larger than %d.\n",
	4498	+ hr_qp->sl, MAX_SERVICE_LEVEL);
	4499	+ return -EINVAL;
	4500	+ }
	4501	+
	4502	+ roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
	4503	+ V2_QPC_BYTE_28_SL_S, hr_qp->sl);
	4504	+ roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
	4505	+ V2_QPC_BYTE_28_SL_S, 0);
	4506	+
	4507	+ return 0;
	4508	+}
	4509	+
	4510	+static bool check_qp_state(enum ib_qp_state cur_state,
	4511	+ enum ib_qp_state new_state)
	4512	+{
	4513	+ static const bool sm[][IB_QPS_ERR + 1] = {
	4514	+ [IB_QPS_RESET] = { [IB_QPS_RESET] = true,
	4515	+ [IB_QPS_INIT] = true },
	4516	+ [IB_QPS_INIT] = { [IB_QPS_RESET] = true,
	4517	+ [IB_QPS_INIT] = true,
	4518	+ [IB_QPS_RTR] = true,
	4519	+ [IB_QPS_ERR] = true },
	4520	+ [IB_QPS_RTR] = { [IB_QPS_RESET] = true,
	4521	+ [IB_QPS_RTS] = true,
	4522	+ [IB_QPS_ERR] = true },
	4523	+ [IB_QPS_RTS] = { [IB_QPS_RESET] = true,
	4524	+ [IB_QPS_RTS] = true,
	4525	+ [IB_QPS_ERR] = true },
	4526	+ [IB_QPS_SQD] = {},
	4527	+ [IB_QPS_SQE] = {},
	4528	+ [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }
	4529	+ };
	4530	+
	4531	+ return sm[cur_state][new_state];
	4532	+}
	4533	+
	4534	+static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
	4535	+ const struct ib_qp_attr *attr,
	4536	+ int attr_mask,
	4537	+ enum ib_qp_state cur_state,
	4538	+ enum ib_qp_state new_state,
	4539	+ struct hns_roce_v2_qp_context *context,
	4540	+ struct hns_roce_v2_qp_context *qpc_mask)
	4541	+{
	4542	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4543	+ int ret = 0;
	4544	+
	4545	+ if (!check_qp_state(cur_state, new_state)) {
	4546	+ ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n");
	4547	+ return -EINVAL;
	4548	+ }
	4549	+
	4550	+ if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
	4551	+ memset(qpc_mask, 0, hr_dev->caps.qpc_sz);
	4552	+ modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
	4553	+ qpc_mask);
	4554	+ } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
	4555	+ modify_qp_init_to_init(ibqp, attr, attr_mask, context,
	4556	+ qpc_mask);
	4557	+ } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
	4558	+ ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
	4559	+ qpc_mask);
	4560	+ } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
	4561	+ ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
	4562	+ qpc_mask);
	4563	+ }
	4564	+
	4565	+ return ret;
	4566	+}
	4567	+
	4568	+static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
	4569	+ const struct ib_qp_attr *attr,
	4570	+ int attr_mask,
	4571	+ struct hns_roce_v2_qp_context *context,
	4572	+ struct hns_roce_v2_qp_context *qpc_mask)
	4573	+{
	4574	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4575	+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
	4576	+ int ret = 0;
	4577	+
	4578	+ if (attr_mask & IB_QP_AV) {
	4579	+ ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
	4580	+ qpc_mask);
	4581	+ if (ret)
	4582	+ return ret;
	4583	+ }
	4584	+
	4585	+ if (attr_mask & IB_QP_TIMEOUT) {
	4586	+ if (attr->timeout < 31) {
	4587	+ roce_set_field(context->byte_28_at_fl,
	4588	+ V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
	4589	+ attr->timeout);
	4590	+ roce_set_field(qpc_mask->byte_28_at_fl,
	4591	+ V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
	4592	+ 0);
	4593	+ } else {
	4594	+ ibdev_warn(&hr_dev->ib_dev,
	4595	+ "Local ACK timeout shall be 0 to 30.\n");
	4596	+ }
	4597	+ }
	4598	+
	4599	+ if (attr_mask & IB_QP_RETRY_CNT) {
	4600	+ roce_set_field(context->byte_212_lsn,
	4601	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
	4602	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
	4603	+ attr->retry_cnt);
	4604	+ roce_set_field(qpc_mask->byte_212_lsn,
	4605	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
	4606	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
	4607	+
	4608	+ roce_set_field(context->byte_212_lsn,
	4609	+ V2_QPC_BYTE_212_RETRY_CNT_M,
	4610	+ V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt);
	4611	+ roce_set_field(qpc_mask->byte_212_lsn,
	4612	+ V2_QPC_BYTE_212_RETRY_CNT_M,
	4613	+ V2_QPC_BYTE_212_RETRY_CNT_S, 0);
	4614	+ }
	4615	+
	4616	+ if (attr_mask & IB_QP_RNR_RETRY) {
	4617	+ roce_set_field(context->byte_244_rnr_rxack,
	4618	+ V2_QPC_BYTE_244_RNR_NUM_INIT_M,
	4619	+ V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
	4620	+ roce_set_field(qpc_mask->byte_244_rnr_rxack,
	4621	+ V2_QPC_BYTE_244_RNR_NUM_INIT_M,
	4622	+ V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
	4623	+
	4624	+ roce_set_field(context->byte_244_rnr_rxack,
	4625	+ V2_QPC_BYTE_244_RNR_CNT_M,
	4626	+ V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
	4627	+ roce_set_field(qpc_mask->byte_244_rnr_rxack,
	4628	+ V2_QPC_BYTE_244_RNR_CNT_M,
	4629	+ V2_QPC_BYTE_244_RNR_CNT_S, 0);
	4630	+ }
	4631	+
	4632	+ /* RC&UC&UD required attr */
	4633	+ if (attr_mask & IB_QP_SQ_PSN) {
	4634	+ roce_set_field(context->byte_172_sq_psn,
	4635	+ V2_QPC_BYTE_172_SQ_CUR_PSN_M,
	4636	+ V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
	4637	+ roce_set_field(qpc_mask->byte_172_sq_psn,
	4638	+ V2_QPC_BYTE_172_SQ_CUR_PSN_M,
	4639	+ V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
	4640	+
	4641	+ roce_set_field(context->byte_196_sq_psn,
	4642	+ V2_QPC_BYTE_196_SQ_MAX_PSN_M,
	4643	+ V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
	4644	+ roce_set_field(qpc_mask->byte_196_sq_psn,
	4645	+ V2_QPC_BYTE_196_SQ_MAX_PSN_M,
	4646	+ V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
	4647	+
	4648	+ roce_set_field(context->byte_220_retry_psn_msn,
	4649	+ V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
	4650	+ V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
	4651	+ roce_set_field(qpc_mask->byte_220_retry_psn_msn,
	4652	+ V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
	4653	+ V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
	4654	+
	4655	+ roce_set_field(context->byte_224_retry_msg,
	4656	+ V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
	4657	+ V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
	4658	+ attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
	4659	+ roce_set_field(qpc_mask->byte_224_retry_msg,
	4660	+ V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
	4661	+ V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
	4662	+
	4663	+ roce_set_field(context->byte_224_retry_msg,
	4664	+ V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
	4665	+ V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
	4666	+ attr->sq_psn);
	4667	+ roce_set_field(qpc_mask->byte_224_retry_msg,
	4668	+ V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
	4669	+ V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
	4670	+
	4671	+ roce_set_field(context->byte_244_rnr_rxack,
	4672	+ V2_QPC_BYTE_244_RX_ACK_EPSN_M,
	4673	+ V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
	4674	+ roce_set_field(qpc_mask->byte_244_rnr_rxack,
	4675	+ V2_QPC_BYTE_244_RX_ACK_EPSN_M,
	4676	+ V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
	4677	+ }
	4678	+
	4679	+ if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
	4680	+ attr->max_dest_rd_atomic) {
	4681	+ roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
	4682	+ V2_QPC_BYTE_140_RR_MAX_S,
	4683	+ fls(attr->max_dest_rd_atomic - 1));
	4684	+ roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
	4685	+ V2_QPC_BYTE_140_RR_MAX_S, 0);
	4686	+ }
3425	4687
3426	4688	if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
3427	4689	roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
..	..	@@ -3431,197 +4693,51 @@
3431	4693	V2_QPC_BYTE_208_SR_MAX_M,
3432	4694	V2_QPC_BYTE_208_SR_MAX_S, 0);
3433	4695	}
3434		- return 0;
3435		-}
3436		-
3437		-static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
3438		- const struct ib_qp_attr *attr,
3439		- int attr_mask, enum ib_qp_state cur_state,
3440		- enum ib_qp_state new_state)
3441		-{
3442		- struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3443		- struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3444		- struct hns_roce_v2_qp_context *context;
3445		- struct hns_roce_v2_qp_context *qpc_mask;
3446		- struct device *dev = hr_dev->dev;
3447		- int ret = -EINVAL;
3448		-
3449		- context = kcalloc(2, sizeof(*context), GFP_ATOMIC);
3450		- if (!context)
3451		- return -ENOMEM;
3452		-
3453		- qpc_mask = context + 1;
3454		- /*
3455		- * In v2 engine, software pass context and context mask to hardware
3456		- * when modifying qp. If software need modify some fields in context,
3457		- * we should set all bits of the relevant fields in context mask to
3458		- * 0 at the same time, else set them to 0x1.
3459		- */
3460		- memset(qpc_mask, 0xff, sizeof(*qpc_mask));
3461		- if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3462		- modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
3463		- qpc_mask);
3464		- } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
3465		- modify_qp_init_to_init(ibqp, attr, attr_mask, context,
3466		- qpc_mask);
3467		- } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3468		- ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
3469		- qpc_mask);
3470		- if (ret)
3471		- goto out;
3472		- } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
3473		- ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
3474		- qpc_mask);
3475		- if (ret)
3476		- goto out;
3477		- } else if ((cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) \|\|
3478		- (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS) \|\|
3479		- (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD) \|\|
3480		- (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD) \|\|
3481		- (cur_state == IB_QPS_SQD && new_state == IB_QPS_RTS) \|\|
3482		- (cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) \|\|
3483		- (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) \|\|
3484		- (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) \|\|
3485		- (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) \|\|
3486		- (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) \|\|
3487		- (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) \|\|
3488		- (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) \|\|
3489		- (cur_state == IB_QPS_SQD && new_state == IB_QPS_ERR) \|\|
3490		- (cur_state == IB_QPS_SQE && new_state == IB_QPS_ERR) \|\|
3491		- (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR)) {
3492		- /* Nothing */
3493		- ;
3494		- } else {
3495		- dev_err(dev, "Illegal state for QP!\n");
3496		- ret = -EINVAL;
3497		- goto out;
3498		- }
3499		-
3500		- /* When QP state is err, SQ and RQ WQE should be flushed */
3501		- if (new_state == IB_QPS_ERR) {
3502		- roce_set_field(context->byte_160_sq_ci_pi,
3503		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3504		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
3505		- hr_qp->sq.head);
3506		- roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3507		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3508		- V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
3509		-
3510		- if (!ibqp->srq) {
3511		- roce_set_field(context->byte_84_rq_ci_pi,
3512		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3513		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
3514		- hr_qp->rq.head);
3515		- roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3516		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3517		- V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3518		- }
3519		- }
3520		-
3521		- if (attr_mask & IB_QP_AV) {
3522		- const struct ib_global_route *grh =
3523		- rdma_ah_read_grh(&attr->ah_attr);
3524		- const struct ib_gid_attr *gid_attr = NULL;
3525		- u8 src_mac[ETH_ALEN];
3526		- int is_roce_protocol;
3527		- u16 vlan = 0xffff;
3528		- u8 ib_port;
3529		- u8 hr_port;
3530		-
3531		- ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num :
3532		- hr_qp->port + 1;
3533		- hr_port = ib_port - 1;
3534		- is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
3535		- rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
3536		-
3537		- if (is_roce_protocol) {
3538		- gid_attr = attr->ah_attr.grh.sgid_attr;
3539		- vlan = rdma_vlan_dev_vlan_id(gid_attr->ndev);
3540		- memcpy(src_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
3541		- }
3542		-
3543		- roce_set_field(context->byte_24_mtu_tc,
3544		- V2_QPC_BYTE_24_VLAN_ID_M,
3545		- V2_QPC_BYTE_24_VLAN_ID_S, vlan);
3546		- roce_set_field(qpc_mask->byte_24_mtu_tc,
3547		- V2_QPC_BYTE_24_VLAN_ID_M,
3548		- V2_QPC_BYTE_24_VLAN_ID_S, 0);
3549		-
3550		- if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
3551		- dev_err(hr_dev->dev,
3552		- "sgid_index(%u) too large. max is %d\n",
3553		- grh->sgid_index,
3554		- hr_dev->caps.gid_table_len[hr_port]);
3555		- ret = -EINVAL;
3556		- goto out;
3557		- }
3558		-
3559		- if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
3560		- dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n");
3561		- ret = -EINVAL;
3562		- goto out;
3563		- }
3564		-
3565		- roce_set_field(context->byte_52_udpspn_dmac,
3566		- V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S,
3567		- (gid_attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) ?
3568		- 0 : 0x12b7);
3569		-
3570		- roce_set_field(qpc_mask->byte_52_udpspn_dmac,
3571		- V2_QPC_BYTE_52_UDPSPN_M,
3572		- V2_QPC_BYTE_52_UDPSPN_S, 0);
3573		-
3574		- roce_set_field(context->byte_20_smac_sgid_idx,
3575		- V2_QPC_BYTE_20_SGID_IDX_M,
3576		- V2_QPC_BYTE_20_SGID_IDX_S, grh->sgid_index);
3577		-
3578		- roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3579		- V2_QPC_BYTE_20_SGID_IDX_M,
3580		- V2_QPC_BYTE_20_SGID_IDX_S, 0);
3581		-
3582		- roce_set_field(context->byte_24_mtu_tc,
3583		- V2_QPC_BYTE_24_HOP_LIMIT_M,
3584		- V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
3585		- roce_set_field(qpc_mask->byte_24_mtu_tc,
3586		- V2_QPC_BYTE_24_HOP_LIMIT_M,
3587		- V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
3588		-
3589		- roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
3590		- V2_QPC_BYTE_24_TC_S, grh->traffic_class);
3591		- roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
3592		- V2_QPC_BYTE_24_TC_S, 0);
3593		- roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
3594		- V2_QPC_BYTE_28_FL_S, grh->flow_label);
3595		- roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
3596		- V2_QPC_BYTE_28_FL_S, 0);
3597		- memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
3598		- memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
3599		- roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
3600		- V2_QPC_BYTE_28_SL_S,
3601		- rdma_ah_get_sl(&attr->ah_attr));
3602		- roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
3603		- V2_QPC_BYTE_28_SL_S, 0);
3604		- hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
3605		- }
3606	4696
3607	4697	if (attr_mask & (IB_QP_ACCESS_FLAGS \| IB_QP_MAX_DEST_RD_ATOMIC))
3608	4698	set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
3609	4699
3610		- /* Every status migrate must change state */
3611		- roce_set_field(context->byte_60_qpst_mapid, V2_QPC_BYTE_60_QP_ST_M,
3612		- V2_QPC_BYTE_60_QP_ST_S, new_state);
3613		- roce_set_field(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_QP_ST_M,
3614		- V2_QPC_BYTE_60_QP_ST_S, 0);
3615		-
3616		- /* SW pass context to HW */
3617		- ret = hns_roce_v2_qp_modify(hr_dev, &hr_qp->mtt, cur_state, new_state,
3618		- context, hr_qp);
3619		- if (ret) {
3620		- dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
3621		- goto out;
	4700	+ if (attr_mask & IB_QP_MIN_RNR_TIMER) {
	4701	+ roce_set_field(context->byte_80_rnr_rx_cqn,
	4702	+ V2_QPC_BYTE_80_MIN_RNR_TIME_M,
	4703	+ V2_QPC_BYTE_80_MIN_RNR_TIME_S,
	4704	+ attr->min_rnr_timer);
	4705	+ roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
	4706	+ V2_QPC_BYTE_80_MIN_RNR_TIME_M,
	4707	+ V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
3622	4708	}
3623	4709
3624		- hr_qp->state = new_state;
	4710	+ /* RC&UC required attr */
	4711	+ if (attr_mask & IB_QP_RQ_PSN) {
	4712	+ roce_set_field(context->byte_108_rx_reqepsn,
	4713	+ V2_QPC_BYTE_108_RX_REQ_EPSN_M,
	4714	+ V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
	4715	+ roce_set_field(qpc_mask->byte_108_rx_reqepsn,
	4716	+ V2_QPC_BYTE_108_RX_REQ_EPSN_M,
	4717	+ V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
	4718	+
	4719	+ roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
	4720	+ V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
	4721	+ roce_set_field(qpc_mask->byte_152_raq,
	4722	+ V2_QPC_BYTE_152_RAQ_PSN_M,
	4723	+ V2_QPC_BYTE_152_RAQ_PSN_S, 0);
	4724	+ }
	4725	+
	4726	+ if (attr_mask & IB_QP_QKEY) {
	4727	+ context->qkey_xrcd = cpu_to_le32(attr->qkey);
	4728	+ qpc_mask->qkey_xrcd = 0;
	4729	+ hr_qp->qkey = attr->qkey;
	4730	+ }
	4731	+
	4732	+ return ret;
	4733	+}
	4734	+
	4735	+static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
	4736	+ const struct ib_qp_attr *attr,
	4737	+ int attr_mask)
	4738	+{
	4739	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4740	+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3625	4741
3626	4742	if (attr_mask & IB_QP_ACCESS_FLAGS)
3627	4743	hr_qp->atomic_rd_en = attr->qp_access_flags;
..	..	@@ -3632,6 +4748,90 @@
3632	4748	hr_qp->port = attr->port_num - 1;
3633	4749	hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
3634	4750	}
	4751	+}
	4752	+
	4753	+static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
	4754	+ const struct ib_qp_attr *attr,
	4755	+ int attr_mask, enum ib_qp_state cur_state,
	4756	+ enum ib_qp_state new_state)
	4757	+{
	4758	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
	4759	+ struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
	4760	+ struct hns_roce_v2_qp_context ctx[2];
	4761	+ struct hns_roce_v2_qp_context *context = ctx;
	4762	+ struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
	4763	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	4764	+ unsigned long sq_flag = 0;
	4765	+ unsigned long rq_flag = 0;
	4766	+ int ret;
	4767	+
	4768	+ /*
	4769	+ * In v2 engine, software pass context and context mask to hardware
	4770	+ * when modifying qp. If software need modify some fields in context,
	4771	+ * we should set all bits of the relevant fields in context mask to
	4772	+ * 0 at the same time, else set them to 0x1.
	4773	+ */
	4774	+ memset(context, 0, hr_dev->caps.qpc_sz);
	4775	+ memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
	4776	+
	4777	+ ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
	4778	+ new_state, context, qpc_mask);
	4779	+ if (ret)
	4780	+ goto out;
	4781	+
	4782	+ /* When QP state is err, SQ and RQ WQE should be flushed */
	4783	+ if (new_state == IB_QPS_ERR) {
	4784	+ spin_lock_irqsave(&hr_qp->sq.lock, sq_flag);
	4785	+ hr_qp->state = IB_QPS_ERR;
	4786	+ roce_set_field(context->byte_160_sq_ci_pi,
	4787	+ V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
	4788	+ V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
	4789	+ hr_qp->sq.head);
	4790	+ roce_set_field(qpc_mask->byte_160_sq_ci_pi,
	4791	+ V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
	4792	+ V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
	4793	+ spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
	4794	+
	4795	+ if (!ibqp->srq) {
	4796	+ spin_lock_irqsave(&hr_qp->rq.lock, rq_flag);
	4797	+ roce_set_field(context->byte_84_rq_ci_pi,
	4798	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
	4799	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
	4800	+ hr_qp->rq.head);
	4801	+ roce_set_field(qpc_mask->byte_84_rq_ci_pi,
	4802	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
	4803	+ V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
	4804	+ spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag);
	4805	+ }
	4806	+ }
	4807	+
	4808	+ /* Configure the optional fields */
	4809	+ ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
	4810	+ qpc_mask);
	4811	+ if (ret)
	4812	+ goto out;
	4813	+
	4814	+ roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
	4815	+ ibqp->srq ? 1 : 0);
	4816	+ roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
	4817	+ V2_QPC_BYTE_108_INV_CREDIT_S, 0);
	4818	+
	4819	+ /* Every status migrate must change state */
	4820	+ roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
	4821	+ V2_QPC_BYTE_60_QP_ST_S, new_state);
	4822	+ roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
	4823	+ V2_QPC_BYTE_60_QP_ST_S, 0);
	4824	+
	4825	+ /* SW pass context to HW */
	4826	+ ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);
	4827	+ if (ret) {
	4828	+ ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret);
	4829	+ goto out;
	4830	+ }
	4831	+
	4832	+ hr_qp->state = new_state;
	4833	+
	4834	+ hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
3635	4835
3636	4836	if (new_state == IB_QPS_RESET && !ibqp->uobject) {
3637	4837	hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
..	..	@@ -3644,30 +4844,29 @@
3644	4844	hr_qp->rq.tail = 0;
3645	4845	hr_qp->sq.head = 0;
3646	4846	hr_qp->sq.tail = 0;
3647		- hr_qp->sq_next_wqe = 0;
3648	4847	hr_qp->next_sge = 0;
3649	4848	if (hr_qp->rq.wqe_cnt)
3650	4849	*hr_qp->rdb.db_record = 0;
3651	4850	}
3652	4851
3653	4852	out:
3654		- kfree(context);
3655	4853	return ret;
3656	4854	}
3657	4855
3658		-static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
	4856	+static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
3659	4857	{
3660		- switch (state) {
3661		- case HNS_ROCE_QP_ST_RST: return IB_QPS_RESET;
3662		- case HNS_ROCE_QP_ST_INIT: return IB_QPS_INIT;
3663		- case HNS_ROCE_QP_ST_RTR: return IB_QPS_RTR;
3664		- case HNS_ROCE_QP_ST_RTS: return IB_QPS_RTS;
3665		- case HNS_ROCE_QP_ST_SQ_DRAINING:
3666		- case HNS_ROCE_QP_ST_SQD: return IB_QPS_SQD;
3667		- case HNS_ROCE_QP_ST_SQER: return IB_QPS_SQE;
3668		- case HNS_ROCE_QP_ST_ERR: return IB_QPS_ERR;
3669		- default: return -1;
3670		- }
	4858	+ static const enum ib_qp_state map[] = {
	4859	+ [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,
	4860	+ [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,
	4861	+ [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,
	4862	+ [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,
	4863	+ [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,
	4864	+ [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,
	4865	+ [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,
	4866	+ [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD
	4867	+ };
	4868	+
	4869	+ return (state < ARRAY_SIZE(map)) ? map[state] : -1;
3671	4870	}
3672	4871
3673	4872	static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
..	..	@@ -3684,12 +4883,10 @@
3684	4883	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
3685	4884	HNS_ROCE_CMD_QUERY_QPC,
3686	4885	HNS_ROCE_CMD_TIMEOUT_MSECS);
3687		- if (ret) {
3688		- dev_err(hr_dev->dev, "QUERY QP cmd process error\n");
	4886	+ if (ret)
3689	4887	goto out;
3690		- }
3691	4888
3692		- memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
	4889	+ memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz);
3693	4890
3694	4891	out:
3695	4892	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
..	..	@@ -3702,15 +4899,11 @@
3702	4899	{
3703	4900	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3704	4901	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3705		- struct hns_roce_v2_qp_context *context;
3706		- struct device *dev = hr_dev->dev;
	4902	+ struct hns_roce_v2_qp_context context = {};
	4903	+ struct ib_device *ibdev = &hr_dev->ib_dev;
3707	4904	int tmp_qp_state;
3708	4905	int state;
3709	4906	int ret;
3710		-
3711		- context = kzalloc(sizeof(*context), GFP_KERNEL);
3712		- if (!context)
3713		- return -ENOMEM;
3714	4907
3715	4908	memset(qp_attr, 0, sizeof(*qp_attr));
3716	4909	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
..	..	@@ -3723,89 +4916,92 @@
3723	4916	goto done;
3724	4917	}
3725	4918
3726		- ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, context);
	4919	+ ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
3727	4920	if (ret) {
3728		- dev_err(dev, "query qpc error\n");
	4921	+ ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);
3729	4922	ret = -EINVAL;
3730	4923	goto out;
3731	4924	}
3732	4925
3733		- state = roce_get_field(context->byte_60_qpst_mapid,
	4926	+ state = roce_get_field(context.byte_60_qpst_tempid,
3734	4927	V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
3735	4928	tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
3736	4929	if (tmp_qp_state == -1) {
3737		- dev_err(dev, "Illegal ib_qp_state\n");
	4930	+ ibdev_err(ibdev, "Illegal ib_qp_state\n");
3738	4931	ret = -EINVAL;
3739	4932	goto out;
3740	4933	}
3741	4934	hr_qp->state = (u8)tmp_qp_state;
3742	4935	qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
3743		- qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->byte_24_mtu_tc,
	4936	+ qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context.byte_24_mtu_tc,
3744	4937	V2_QPC_BYTE_24_MTU_M,
3745	4938	V2_QPC_BYTE_24_MTU_S);
3746	4939	qp_attr->path_mig_state = IB_MIG_ARMED;
3747	4940	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
3748	4941	if (hr_qp->ibqp.qp_type == IB_QPT_UD)
3749		- qp_attr->qkey = V2_QKEY_VAL;
	4942	+ qp_attr->qkey = le32_to_cpu(context.qkey_xrcd);
3750	4943
3751		- qp_attr->rq_psn = roce_get_field(context->byte_108_rx_reqepsn,
	4944	+ qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn,
3752	4945	V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3753	4946	V2_QPC_BYTE_108_RX_REQ_EPSN_S);
3754		- qp_attr->sq_psn = (u32)roce_get_field(context->byte_172_sq_psn,
	4947	+ qp_attr->sq_psn = (u32)roce_get_field(context.byte_172_sq_psn,
3755	4948	V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3756	4949	V2_QPC_BYTE_172_SQ_CUR_PSN_S);
3757		- qp_attr->dest_qp_num = (u8)roce_get_field(context->byte_56_dqpn_err,
	4950	+ qp_attr->dest_qp_num = (u8)roce_get_field(context.byte_56_dqpn_err,
3758	4951	V2_QPC_BYTE_56_DQPN_M,
3759	4952	V2_QPC_BYTE_56_DQPN_S);
3760		- qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en,
3761		- V2_QPC_BYTE_76_RRE_S)) << 2) \|
3762		- ((roce_get_bit(context->byte_76_srqn_op_en,
3763		- V2_QPC_BYTE_76_RWE_S)) << 1) \|
3764		- ((roce_get_bit(context->byte_76_srqn_op_en,
3765		- V2_QPC_BYTE_76_ATE_S)) << 3);
	4953	+ qp_attr->qp_access_flags = ((roce_get_bit(context.byte_76_srqn_op_en,
	4954	+ V2_QPC_BYTE_76_RRE_S)) << V2_QP_RRE_S) \|
	4955	+ ((roce_get_bit(context.byte_76_srqn_op_en,
	4956	+ V2_QPC_BYTE_76_RWE_S)) << V2_QP_RWE_S) \|
	4957	+ ((roce_get_bit(context.byte_76_srqn_op_en,
	4958	+ V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
	4959	+
3766	4960	if (hr_qp->ibqp.qp_type == IB_QPT_RC \|\|
3767	4961	hr_qp->ibqp.qp_type == IB_QPT_UC) {
3768	4962	struct ib_global_route *grh =
3769	4963	rdma_ah_retrieve_grh(&qp_attr->ah_attr);
3770	4964
3771	4965	rdma_ah_set_sl(&qp_attr->ah_attr,
3772		- roce_get_field(context->byte_28_at_fl,
	4966	+ roce_get_field(context.byte_28_at_fl,
3773	4967	V2_QPC_BYTE_28_SL_M,
3774	4968	V2_QPC_BYTE_28_SL_S));
3775		- grh->flow_label = roce_get_field(context->byte_28_at_fl,
	4969	+ grh->flow_label = roce_get_field(context.byte_28_at_fl,
3776	4970	V2_QPC_BYTE_28_FL_M,
3777	4971	V2_QPC_BYTE_28_FL_S);
3778		- grh->sgid_index = roce_get_field(context->byte_20_smac_sgid_idx,
	4972	+ grh->sgid_index = roce_get_field(context.byte_20_smac_sgid_idx,
3779	4973	V2_QPC_BYTE_20_SGID_IDX_M,
3780	4974	V2_QPC_BYTE_20_SGID_IDX_S);
3781		- grh->hop_limit = roce_get_field(context->byte_24_mtu_tc,
	4975	+ grh->hop_limit = roce_get_field(context.byte_24_mtu_tc,
3782	4976	V2_QPC_BYTE_24_HOP_LIMIT_M,
3783	4977	V2_QPC_BYTE_24_HOP_LIMIT_S);
3784		- grh->traffic_class = roce_get_field(context->byte_24_mtu_tc,
	4978	+ grh->traffic_class = roce_get_field(context.byte_24_mtu_tc,
3785	4979	V2_QPC_BYTE_24_TC_M,
3786	4980	V2_QPC_BYTE_24_TC_S);
3787	4981
3788		- memcpy(grh->dgid.raw, context->dgid, sizeof(grh->dgid.raw));
	4982	+ memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
3789	4983	}
3790	4984
3791	4985	qp_attr->port_num = hr_qp->port + 1;
3792	4986	qp_attr->sq_draining = 0;
3793		- qp_attr->max_rd_atomic = 1 << roce_get_field(context->byte_208_irrl,
	4987	+ qp_attr->max_rd_atomic = 1 << roce_get_field(context.byte_208_irrl,
3794	4988	V2_QPC_BYTE_208_SR_MAX_M,
3795	4989	V2_QPC_BYTE_208_SR_MAX_S);
3796		- qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->byte_140_raq,
	4990	+ qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq,
3797	4991	V2_QPC_BYTE_140_RR_MAX_M,
3798	4992	V2_QPC_BYTE_140_RR_MAX_S);
3799		- qp_attr->min_rnr_timer = (u8)roce_get_field(context->byte_80_rnr_rx_cqn,
	4993	+ qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn,
3800	4994	V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3801	4995	V2_QPC_BYTE_80_MIN_RNR_TIME_S);
3802		- qp_attr->timeout = (u8)roce_get_field(context->byte_28_at_fl,
	4996	+ qp_attr->timeout = (u8)roce_get_field(context.byte_28_at_fl,
3803	4997	V2_QPC_BYTE_28_AT_M,
3804	4998	V2_QPC_BYTE_28_AT_S);
3805		- qp_attr->retry_cnt = roce_get_field(context->byte_212_lsn,
3806		- V2_QPC_BYTE_212_RETRY_CNT_M,
3807		- V2_QPC_BYTE_212_RETRY_CNT_S);
3808		- qp_attr->rnr_retry = context->rq_rnr_timer;
	4999	+ qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn,
	5000	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
	5001	+ V2_QPC_BYTE_212_RETRY_NUM_INIT_S);
	5002	+ qp_attr->rnr_retry = roce_get_field(context.byte_244_rnr_rxack,
	5003	+ V2_QPC_BYTE_244_RNR_NUM_INIT_M,
	5004	+ V2_QPC_BYTE_244_RNR_NUM_INIT_S);
3809	5005
3810	5006	done:
3811	5007	qp_attr->cur_qp_state = qp_attr->qp_state;
..	..	@@ -3825,100 +5021,306 @@
3825	5021
3826	5022	out:
3827	5023	mutex_unlock(&hr_qp->mutex);
3828		- kfree(context);
3829	5024	return ret;
3830	5025	}
3831	5026
3832	5027	static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
3833	5028	struct hns_roce_qp *hr_qp,
3834		- int is_user)
	5029	+ struct ib_udata *udata)
3835	5030	{
	5031	+ struct ib_device *ibdev = &hr_dev->ib_dev;
3836	5032	struct hns_roce_cq send_cq, recv_cq;
3837		- struct device *dev = hr_dev->dev;
3838		- int ret;
	5033	+ unsigned long flags;
	5034	+ int ret = 0;
3839	5035
3840	5036	if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
3841	5037	/* Modify qp to reset before destroying qp */
3842	5038	ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
3843	5039	hr_qp->state, IB_QPS_RESET);
3844		- if (ret) {
3845		- dev_err(dev, "modify QP %06lx to ERR failed.\n",
3846		- hr_qp->qpn);
3847		- return ret;
3848		- }
	5040	+ if (ret)
	5041	+ ibdev_err(ibdev,
	5042	+ "failed to modify QP to RST, ret = %d.\n",
	5043	+ ret);
3849	5044	}
3850	5045
3851		- send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
3852		- recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
	5046	+ send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;
	5047	+ recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;
3853	5048
	5049	+ spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
3854	5050	hns_roce_lock_cqs(send_cq, recv_cq);
3855	5051
3856		- if (!is_user) {
3857		- __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
3858		- to_hr_srq(hr_qp->ibqp.srq) : NULL);
3859		- if (send_cq != recv_cq)
	5052	+ if (!udata) {
	5053	+ if (recv_cq)
	5054	+ __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn,
	5055	+ (hr_qp->ibqp.srq ?
	5056	+ to_hr_srq(hr_qp->ibqp.srq) :
	5057	+ NULL));
	5058	+
	5059	+ if (send_cq && send_cq != recv_cq)
3860	5060	__hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
	5061	+
3861	5062	}
3862	5063
3863	5064	hns_roce_qp_remove(hr_dev, hr_qp);
3864	5065
3865	5066	hns_roce_unlock_cqs(send_cq, recv_cq);
	5067	+ spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
3866	5068
3867		- hns_roce_qp_free(hr_dev, hr_qp);
3868		-
3869		- /* Not special_QP, free their QPN */
3870		- if ((hr_qp->ibqp.qp_type == IB_QPT_RC) \|\|
3871		- (hr_qp->ibqp.qp_type == IB_QPT_UC) \|\|
3872		- (hr_qp->ibqp.qp_type == IB_QPT_UD))
3873		- hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
3874		-
3875		- hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
3876		-
3877		- if (is_user) {
3878		- if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
3879		- hns_roce_db_unmap_user(
3880		- to_hr_ucontext(hr_qp->ibqp.uobject->context),
3881		- &hr_qp->sdb);
3882		-
3883		- if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
3884		- hns_roce_db_unmap_user(
3885		- to_hr_ucontext(hr_qp->ibqp.uobject->context),
3886		- &hr_qp->rdb);
3887		- ib_umem_release(hr_qp->umem);
3888		- } else {
3889		- kfree(hr_qp->sq.wrid);
3890		- kfree(hr_qp->rq.wrid);
3891		- hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
3892		- if (hr_qp->rq.wqe_cnt)
3893		- hns_roce_free_db(hr_dev, &hr_qp->rdb);
3894		- }
3895		-
3896		- if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
3897		- kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
3898		- kfree(hr_qp->rq_inl_buf.wqe_list);
3899		- }
3900		-
3901		- return 0;
	5069	+ return ret;
3902	5070	}
3903	5071
3904		-static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp)
	5072	+static int hns_roce_v2_destroy_qp(struct ib_qp ibqp, struct ib_udata udata)
3905	5073	{
3906	5074	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3907	5075	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3908	5076	int ret;
3909	5077
3910		- ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject);
3911		- if (ret) {
3912		- dev_err(hr_dev->dev, "Destroy qp failed(%d)\n", ret);
3913		- return ret;
3914		- }
	5078	+ ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
	5079	+ if (ret)
	5080	+ ibdev_err(&hr_dev->ib_dev,
	5081	+ "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",
	5082	+ hr_qp->qpn, ret);
3915	5083
3916		- if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
3917		- kfree(hr_to_hr_sqp(hr_qp));
3918		- else
3919		- kfree(hr_qp);
	5084	+ hns_roce_qp_destroy(hr_dev, hr_qp, udata);
3920	5085
3921	5086	return 0;
	5087	+}
	5088	+
	5089	+static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
	5090	+ struct hns_roce_qp *hr_qp)
	5091	+{
	5092	+ struct ib_device *ibdev = &hr_dev->ib_dev;
	5093	+ struct hns_roce_sccc_clr_done *resp;
	5094	+ struct hns_roce_sccc_clr *clr;
	5095	+ struct hns_roce_cmq_desc desc;
	5096	+ int ret, i;
	5097	+
	5098	+ mutex_lock(&hr_dev->qp_table.scc_mutex);
	5099	+
	5100	+ /* set scc ctx clear done flag */
	5101	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
	5102	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	5103	+ if (ret) {
	5104	+ ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);
	5105	+ goto out;
	5106	+ }
	5107	+
	5108	+ /* clear scc context */
	5109	+ hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
	5110	+ clr = (struct hns_roce_sccc_clr *)desc.data;
	5111	+ clr->qpn = cpu_to_le32(hr_qp->qpn);
	5112	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	5113	+ if (ret) {
	5114	+ ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);
	5115	+ goto out;
	5116	+ }
	5117	+
	5118	+ /* query scc context clear is done or not */
	5119	+ resp = (struct hns_roce_sccc_clr_done *)desc.data;
	5120	+ for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
	5121	+ hns_roce_cmq_setup_basic_desc(&desc,
	5122	+ HNS_ROCE_OPC_QUERY_SCCC, true);
	5123	+ ret = hns_roce_cmq_send(hr_dev, &desc, 1);
	5124	+ if (ret) {
	5125	+ ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n",
	5126	+ ret);
	5127	+ goto out;
	5128	+ }
	5129	+
	5130	+ if (resp->clr_done)
	5131	+ goto out;
	5132	+
	5133	+ msleep(20);
	5134	+ }
	5135	+
	5136	+ ibdev_err(ibdev, "Query SCC clr done flag overtime.\n");
	5137	+ ret = -ETIMEDOUT;
	5138	+
	5139	+out:
	5140	+ mutex_unlock(&hr_dev->qp_table.scc_mutex);
	5141	+ return ret;
	5142	+}
	5143	+
	5144	+static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
	5145	+ struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
	5146	+ u32 cqn, void mb_buf, u64 mtts_wqe,
	5147	+ u64 *mtts_idx, dma_addr_t dma_handle_wqe,
	5148	+ dma_addr_t dma_handle_idx)
	5149	+{
	5150	+ struct hns_roce_srq_context *srq_context;
	5151	+
	5152	+ srq_context = mb_buf;
	5153	+ memset(srq_context, 0, sizeof(*srq_context));
	5154	+
	5155	+ roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
	5156	+ SRQC_BYTE_4_SRQ_ST_S, 1);
	5157	+
	5158	+ roce_set_field(srq_context->byte_4_srqn_srqst,
	5159	+ SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
	5160	+ SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
	5161	+ to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num,
	5162	+ srq->wqe_cnt));
	5163	+ roce_set_field(srq_context->byte_4_srqn_srqst,
	5164	+ SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
	5165	+ ilog2(srq->wqe_cnt));
	5166	+
	5167	+ roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
	5168	+ SRQC_BYTE_4_SRQN_S, srq->srqn);
	5169	+
	5170	+ roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5171	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
	5172	+
	5173	+ roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
	5174	+ SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
	5175	+
	5176	+ srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
	5177	+
	5178	+ roce_set_field(srq_context->byte_24_wqe_bt_ba,
	5179	+ SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
	5180	+ SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
	5181	+ dma_handle_wqe >> 35);
	5182	+
	5183	+ roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
	5184	+ SRQC_BYTE_28_PD_S, pdn);
	5185	+ roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
	5186	+ SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
	5187	+ fls(srq->max_gs - 1));
	5188	+
	5189	+ srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3);
	5190	+ roce_set_field(srq_context->rsv_idx_bt_ba,
	5191	+ SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
	5192	+ SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
	5193	+ dma_handle_idx >> 35);
	5194	+
	5195	+ srq_context->idx_cur_blk_addr =
	5196	+ cpu_to_le32(to_hr_hw_page_addr(mtts_idx[0]));
	5197	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5198	+ SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
	5199	+ SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
	5200	+ upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));
	5201	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5202	+ SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
	5203	+ SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
	5204	+ to_hr_hem_hopnum(hr_dev->caps.idx_hop_num,
	5205	+ srq->wqe_cnt));
	5206	+
	5207	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5208	+ SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
	5209	+ SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
	5210	+ to_hr_hw_page_shift(srq->idx_que.mtr.hem_cfg.ba_pg_shift));
	5211	+ roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
	5212	+ SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
	5213	+ SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
	5214	+ to_hr_hw_page_shift(srq->idx_que.mtr.hem_cfg.buf_pg_shift));
	5215	+
	5216	+ srq_context->idx_nxt_blk_addr =
	5217	+ cpu_to_le32(to_hr_hw_page_addr(mtts_idx[1]));
	5218	+ roce_set_field(srq_context->rsv_idxnxtblkaddr,
	5219	+ SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
	5220	+ SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
	5221	+ upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));
	5222	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5223	+ SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
	5224	+ cqn);
	5225	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5226	+ SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
	5227	+ SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
	5228	+ to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift));
	5229	+ roce_set_field(srq_context->byte_56_xrc_cqn,
	5230	+ SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
	5231	+ SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
	5232	+ to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift));
	5233	+
	5234	+ roce_set_bit(srq_context->db_record_addr_record_en,
	5235	+ SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
	5236	+}
	5237	+
	5238	+static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
	5239	+ struct ib_srq_attr *srq_attr,
	5240	+ enum ib_srq_attr_mask srq_attr_mask,
	5241	+ struct ib_udata *udata)
	5242	+{
	5243	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	5244	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	5245	+ struct hns_roce_srq_context *srq_context;
	5246	+ struct hns_roce_srq_context *srqc_mask;
	5247	+ struct hns_roce_cmd_mailbox *mailbox;
	5248	+ int ret;
	5249	+
	5250	+ /* Resizing SRQs is not supported yet */
	5251	+ if (srq_attr_mask & IB_SRQ_MAX_WR)
	5252	+ return -EINVAL;
	5253	+
	5254	+ if (srq_attr_mask & IB_SRQ_LIMIT) {
	5255	+ if (srq_attr->srq_limit >= srq->wqe_cnt)
	5256	+ return -EINVAL;
	5257	+
	5258	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	5259	+ if (IS_ERR(mailbox))
	5260	+ return PTR_ERR(mailbox);
	5261	+
	5262	+ srq_context = mailbox->buf;
	5263	+ srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
	5264	+
	5265	+ memset(srqc_mask, 0xff, sizeof(*srqc_mask));
	5266	+
	5267	+ roce_set_field(srq_context->byte_8_limit_wl,
	5268	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5269	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
	5270	+ roce_set_field(srqc_mask->byte_8_limit_wl,
	5271	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5272	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
	5273	+
	5274	+ ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
	5275	+ HNS_ROCE_CMD_MODIFY_SRQC,
	5276	+ HNS_ROCE_CMD_TIMEOUT_MSECS);
	5277	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	5278	+ if (ret) {
	5279	+ ibdev_err(&hr_dev->ib_dev,
	5280	+ "failed to handle cmd of modifying SRQ, ret = %d.\n",
	5281	+ ret);
	5282	+ return ret;
	5283	+ }
	5284	+ }
	5285	+
	5286	+ return 0;
	5287	+}
	5288	+
	5289	+static int hns_roce_v2_query_srq(struct ib_srq ibsrq, struct ib_srq_attr attr)
	5290	+{
	5291	+ struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
	5292	+ struct hns_roce_srq *srq = to_hr_srq(ibsrq);
	5293	+ struct hns_roce_srq_context *srq_context;
	5294	+ struct hns_roce_cmd_mailbox *mailbox;
	5295	+ int limit_wl;
	5296	+ int ret;
	5297	+
	5298	+ mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
	5299	+ if (IS_ERR(mailbox))
	5300	+ return PTR_ERR(mailbox);
	5301	+
	5302	+ srq_context = mailbox->buf;
	5303	+ ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
	5304	+ HNS_ROCE_CMD_QUERY_SRQC,
	5305	+ HNS_ROCE_CMD_TIMEOUT_MSECS);
	5306	+ if (ret) {
	5307	+ ibdev_err(&hr_dev->ib_dev,
	5308	+ "failed to process cmd of querying SRQ, ret = %d.\n",
	5309	+ ret);
	5310	+ goto out;
	5311	+ }
	5312	+
	5313	+ limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
	5314	+ SRQC_BYTE_8_SRQ_LIMIT_WL_M,
	5315	+ SRQC_BYTE_8_SRQ_LIMIT_WL_S);
	5316	+
	5317	+ attr->srq_limit = limit_wl;
	5318	+ attr->max_wr = srq->wqe_cnt - 1;
	5319	+ attr->max_sge = srq->max_gs;
	5320	+
	5321	+out:
	5322	+ hns_roce_free_cmd_mailbox(hr_dev, mailbox);
	5323	+ return ret;
3922	5324	}
3923	5325
3924	5326	static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
..	..	@@ -3957,55 +5359,65 @@
3957	5359	HNS_ROCE_CMD_TIMEOUT_MSECS);
3958	5360	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3959	5361	if (ret)
3960		- dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n");
	5362	+ ibdev_err(&hr_dev->ib_dev,
	5363	+ "failed to process cmd when modifying CQ, ret = %d.\n",
	5364	+ ret);
3961	5365
3962	5366	return ret;
3963		-}
3964		-
3965		-static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)
3966		-{
3967		- struct hns_roce_qp *hr_qp;
3968		- struct ib_qp_attr attr;
3969		- int attr_mask;
3970		- int ret;
3971		-
3972		- hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
3973		- if (!hr_qp) {
3974		- dev_warn(hr_dev->dev, "no hr_qp can be found!\n");
3975		- return;
3976		- }
3977		-
3978		- if (hr_qp->ibqp.uobject) {
3979		- if (hr_qp->sdb_en == 1) {
3980		- hr_qp->sq.head = (int )(hr_qp->sdb.virt_addr);
3981		- if (hr_qp->rdb_en == 1)
3982		- hr_qp->rq.head = (int )(hr_qp->rdb.virt_addr);
3983		- } else {
3984		- dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n");
3985		- return;
3986		- }
3987		- }
3988		-
3989		- attr_mask = IB_QP_STATE;
3990		- attr.qp_state = IB_QPS_ERR;
3991		- ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask,
3992		- hr_qp->state, IB_QPS_ERR);
3993		- if (ret)
3994		- dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n",
3995		- qpn);
3996	5367	}
3997	5368
3998	5369	static void hns_roce_irq_work_handle(struct work_struct *work)
3999	5370	{
4000	5371	struct hns_roce_work *irq_work =
4001	5372	container_of(work, struct hns_roce_work, work);
	5373	+ struct ib_device *ibdev = &irq_work->hr_dev->ib_dev;
4002	5374	u32 qpn = irq_work->qpn;
	5375	+ u32 cqn = irq_work->cqn;
4003	5376
4004	5377	switch (irq_work->event_type) {
	5378	+ case HNS_ROCE_EVENT_TYPE_PATH_MIG:
	5379	+ ibdev_info(ibdev, "Path migrated succeeded.\n");
	5380	+ break;
	5381	+ case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
	5382	+ ibdev_warn(ibdev, "Path migration failed.\n");
	5383	+ break;
	5384	+ case HNS_ROCE_EVENT_TYPE_COMM_EST:
	5385	+ break;
	5386	+ case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
	5387	+ ibdev_warn(ibdev, "Send queue drained.\n");
	5388	+ break;
4005	5389	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
	5390	+ ibdev_err(ibdev, "Local work queue 0x%x catast error, sub_event type is: %d\n",
	5391	+ qpn, irq_work->sub_type);
	5392	+ break;
4006	5393	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
	5394	+ ibdev_err(ibdev, "Invalid request local work queue 0x%x error.\n",
	5395	+ qpn);
	5396	+ break;
4007	5397	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4008		- hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
	5398	+ ibdev_err(ibdev, "Local access violation work queue 0x%x error, sub_event type is: %d\n",
	5399	+ qpn, irq_work->sub_type);
	5400	+ break;
	5401	+ case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
	5402	+ ibdev_warn(ibdev, "SRQ limit reach.\n");
	5403	+ break;
	5404	+ case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
	5405	+ ibdev_warn(ibdev, "SRQ last wqe reach.\n");
	5406	+ break;
	5407	+ case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
	5408	+ ibdev_err(ibdev, "SRQ catas error.\n");
	5409	+ break;
	5410	+ case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
	5411	+ ibdev_err(ibdev, "CQ 0x%x access err.\n", cqn);
	5412	+ break;
	5413	+ case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
	5414	+ ibdev_warn(ibdev, "CQ 0x%x overflow\n", cqn);
	5415	+ break;
	5416	+ case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
	5417	+ ibdev_warn(ibdev, "DB overflow.\n");
	5418	+ break;
	5419	+ case HNS_ROCE_EVENT_TYPE_FLR:
	5420	+ ibdev_warn(ibdev, "Function level reset.\n");
4009	5421	break;
4010	5422	default:
4011	5423	break;
..	..	@@ -4015,7 +5427,8 @@
4015	5427	}
4016	5428
4017	5429	static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
4018		- struct hns_roce_eq *eq, u32 qpn)
	5430	+ struct hns_roce_eq *eq,
	5431	+ u32 qpn, u32 cqn)
4019	5432	{
4020	5433	struct hns_roce_work *irq_work;
4021	5434
..	..	@@ -4026,6 +5439,7 @@
4026	5439	INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
4027	5440	irq_work->hr_dev = hr_dev;
4028	5441	irq_work->qpn = qpn;
	5442	+ irq_work->cqn = cqn;
4029	5443	irq_work->event_type = eq->event_type;
4030	5444	irq_work->sub_type = eq->sub_type;
4031	5445	queue_work(hr_dev->irq_workq, &(irq_work->work));
..	..	@@ -4033,10 +5447,8 @@
4033	5447
4034	5448	static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
4035	5449	{
4036		- u32 doorbell[2];
4037		-
4038		- doorbell[0] = 0;
4039		- doorbell[1] = 0;
	5450	+ struct hns_roce_dev *hr_dev = eq->hr_dev;
	5451	+ __le32 doorbell[2] = {};
4040	5452
4041	5453	if (eq->type_flag == HNS_ROCE_AEQ) {
4042	5454	roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
..	..	@@ -4059,164 +5471,16 @@
4059	5471	HNS_ROCE_V2_EQ_DB_PARA_S,
4060	5472	(eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
4061	5473
4062		- hns_roce_write64_k(doorbell, eq->doorbell);
4063		-}
4064		-
4065		-static void hns_roce_v2_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
4066		- struct hns_roce_aeqe *aeqe,
4067		- u32 qpn)
4068		-{
4069		- struct device *dev = hr_dev->dev;
4070		- int sub_type;
4071		-
4072		- dev_warn(dev, "Local work queue catastrophic error.\n");
4073		- sub_type = roce_get_field(aeqe->asyn, HNS_ROCE_V2_AEQE_SUB_TYPE_M,
4074		- HNS_ROCE_V2_AEQE_SUB_TYPE_S);
4075		- switch (sub_type) {
4076		- case HNS_ROCE_LWQCE_QPC_ERROR:
4077		- dev_warn(dev, "QP %d, QPC error.\n", qpn);
4078		- break;
4079		- case HNS_ROCE_LWQCE_MTU_ERROR:
4080		- dev_warn(dev, "QP %d, MTU error.\n", qpn);
4081		- break;
4082		- case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
4083		- dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
4084		- break;
4085		- case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
4086		- dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
4087		- break;
4088		- case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
4089		- dev_warn(dev, "QP %d, WQE shift error.\n", qpn);
4090		- break;
4091		- default:
4092		- dev_err(dev, "Unhandled sub_event type %d.\n", sub_type);
4093		- break;
4094		- }
4095		-}
4096		-
4097		-static void hns_roce_v2_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
4098		- struct hns_roce_aeqe *aeqe, u32 qpn)
4099		-{
4100		- struct device *dev = hr_dev->dev;
4101		- int sub_type;
4102		-
4103		- dev_warn(dev, "Local access violation work queue error.\n");
4104		- sub_type = roce_get_field(aeqe->asyn, HNS_ROCE_V2_AEQE_SUB_TYPE_M,
4105		- HNS_ROCE_V2_AEQE_SUB_TYPE_S);
4106		- switch (sub_type) {
4107		- case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
4108		- dev_warn(dev, "QP %d, R_key violation.\n", qpn);
4109		- break;
4110		- case HNS_ROCE_LAVWQE_LENGTH_ERROR:
4111		- dev_warn(dev, "QP %d, length error.\n", qpn);
4112		- break;
4113		- case HNS_ROCE_LAVWQE_VA_ERROR:
4114		- dev_warn(dev, "QP %d, VA error.\n", qpn);
4115		- break;
4116		- case HNS_ROCE_LAVWQE_PD_ERROR:
4117		- dev_err(dev, "QP %d, PD error.\n", qpn);
4118		- break;
4119		- case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
4120		- dev_warn(dev, "QP %d, rw acc error.\n", qpn);
4121		- break;
4122		- case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
4123		- dev_warn(dev, "QP %d, key state error.\n", qpn);
4124		- break;
4125		- case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
4126		- dev_warn(dev, "QP %d, MR operation error.\n", qpn);
4127		- break;
4128		- default:
4129		- dev_err(dev, "Unhandled sub_event type %d.\n", sub_type);
4130		- break;
4131		- }
4132		-}
4133		-
4134		-static void hns_roce_v2_qp_err_handle(struct hns_roce_dev *hr_dev,
4135		- struct hns_roce_aeqe *aeqe,
4136		- int event_type, u32 qpn)
4137		-{
4138		- struct device *dev = hr_dev->dev;
4139		-
4140		- switch (event_type) {
4141		- case HNS_ROCE_EVENT_TYPE_COMM_EST:
4142		- dev_warn(dev, "Communication established.\n");
4143		- break;
4144		- case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4145		- dev_warn(dev, "Send queue drained.\n");
4146		- break;
4147		- case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
4148		- hns_roce_v2_wq_catas_err_handle(hr_dev, aeqe, qpn);
4149		- break;
4150		- case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4151		- dev_warn(dev, "Invalid request local work queue error.\n");
4152		- break;
4153		- case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4154		- hns_roce_v2_local_wq_access_err_handle(hr_dev, aeqe, qpn);
4155		- break;
4156		- default:
4157		- break;
4158		- }
4159		-
4160		- hns_roce_qp_event(hr_dev, qpn, event_type);
4161		-}
4162		-
4163		-static void hns_roce_v2_cq_err_handle(struct hns_roce_dev *hr_dev,
4164		- struct hns_roce_aeqe *aeqe,
4165		- int event_type, u32 cqn)
4166		-{
4167		- struct device *dev = hr_dev->dev;
4168		-
4169		- switch (event_type) {
4170		- case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4171		- dev_warn(dev, "CQ 0x%x access err.\n", cqn);
4172		- break;
4173		- case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4174		- dev_warn(dev, "CQ 0x%x overflow\n", cqn);
4175		- break;
4176		- default:
4177		- break;
4178		- }
4179		-
4180		- hns_roce_cq_event(hr_dev, cqn, event_type);
4181		-}
4182		-
4183		-static struct hns_roce_aeqe get_aeqe_v2(struct hns_roce_eq eq, u32 entry)
4184		-{
4185		- u32 buf_chk_sz;
4186		- unsigned long off;
4187		-
4188		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4189		- off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4190		-
4191		- return (struct hns_roce_aeqe )((char )(eq->buf_list->buf) +
4192		- off % buf_chk_sz);
4193		-}
4194		-
4195		-static struct hns_roce_aeqe mhop_get_aeqe(struct hns_roce_eq eq, u32 entry)
4196		-{
4197		- u32 buf_chk_sz;
4198		- unsigned long off;
4199		-
4200		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4201		-
4202		- off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4203		-
4204		- if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4205		- return (struct hns_roce_aeqe )((u8 )(eq->bt_l0) +
4206		- off % buf_chk_sz);
4207		- else
4208		- return (struct hns_roce_aeqe )((u8 )
4209		- (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
	5474	+ hns_roce_write64(hr_dev, doorbell, eq->doorbell);
4210	5475	}
4211	5476
4212	5477	static struct hns_roce_aeqe next_aeqe_sw_v2(struct hns_roce_eq eq)
4213	5478	{
4214	5479	struct hns_roce_aeqe *aeqe;
4215	5480
4216		- if (!eq->hop_num)
4217		- aeqe = get_aeqe_v2(eq, eq->cons_index);
4218		- else
4219		- aeqe = mhop_get_aeqe(eq, eq->cons_index);
	5481	+ aeqe = hns_roce_buf_offset(eq->mtr.kmem,
	5482	+ (eq->cons_index & (eq->entries - 1)) *
	5483	+ eq->eqe_size);
4220	5484
4221	5485	return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
4222	5486	!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
..	..	@@ -4226,15 +5490,15 @@
4226	5490	struct hns_roce_eq *eq)
4227	5491	{
4228	5492	struct device *dev = hr_dev->dev;
4229		- struct hns_roce_aeqe *aeqe;
	5493	+ struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
4230	5494	int aeqe_found = 0;
4231	5495	int event_type;
4232	5496	int sub_type;
	5497	+ u32 srqn;
4233	5498	u32 qpn;
4234	5499	u32 cqn;
4235	5500
4236		- while ((aeqe = next_aeqe_sw_v2(eq))) {
4237		-
	5501	+ while (aeqe) {
4238	5502	/* Make sure we read AEQ entry after we have checked the
4239	5503	* ownership bit
4240	5504	*/
..	..	@@ -4252,34 +5516,28 @@
4252	5516	cqn = roce_get_field(aeqe->event.cq_event.cq,
4253	5517	HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4254	5518	HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
	5519	+ srqn = roce_get_field(aeqe->event.srq_event.srq,
	5520	+ HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
	5521	+ HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
4255	5522
4256	5523	switch (event_type) {
4257	5524	case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4258		- dev_warn(dev, "Path migrated succeeded.\n");
4259		- break;
4260	5525	case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4261		- dev_warn(dev, "Path migration failed.\n");
4262		- break;
4263	5526	case HNS_ROCE_EVENT_TYPE_COMM_EST:
4264	5527	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4265	5528	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
	5529	+ case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4266	5530	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4267	5531	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4268		- hns_roce_v2_qp_err_handle(hr_dev, aeqe, event_type,
4269		- qpn);
	5532	+ hns_roce_qp_event(hr_dev, qpn, event_type);
4270	5533	break;
4271	5534	case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4272		- case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4273	5535	case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4274		- dev_warn(dev, "SRQ not support.\n");
	5536	+ hns_roce_srq_event(hr_dev, srqn, event_type);
4275	5537	break;
4276	5538	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4277	5539	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4278		- hns_roce_v2_cq_err_handle(hr_dev, aeqe, event_type,
4279		- cqn);
4280		- break;
4281		- case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4282		- dev_warn(dev, "DB overflow.\n");
	5540	+ hns_roce_cq_event(hr_dev, cqn, event_type);
4283	5541	break;
4284	5542	case HNS_ROCE_EVENT_TYPE_MB:
4285	5543	hns_roce_cmd_event(hr_dev,
..	..	@@ -4287,71 +5545,40 @@
4287	5545	aeqe->event.cmd.status,
4288	5546	le64_to_cpu(aeqe->event.cmd.out_param));
4289	5547	break;
	5548	+ case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4290	5549	case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
4291		- dev_warn(dev, "CEQ overflow.\n");
4292		- break;
4293	5550	case HNS_ROCE_EVENT_TYPE_FLR:
4294		- dev_warn(dev, "Function level reset.\n");
4295	5551	break;
4296	5552	default:
4297	5553	dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
4298	5554	event_type, eq->eqn, eq->cons_index);
4299	5555	break;
4300		- };
	5556	+ }
4301	5557
4302	5558	eq->event_type = event_type;
4303	5559	eq->sub_type = sub_type;
4304	5560	++eq->cons_index;
4305	5561	aeqe_found = 1;
4306	5562
4307		- if (eq->cons_index > (2 * eq->entries - 1)) {
4308		- dev_warn(dev, "cons_index overflow, set back to 0.\n");
	5563	+ if (eq->cons_index > (2 * eq->entries - 1))
4309	5564	eq->cons_index = 0;
4310		- }
4311		- hns_roce_v2_init_irq_work(hr_dev, eq, qpn);
	5565	+
	5566	+ hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
	5567	+
	5568	+ aeqe = next_aeqe_sw_v2(eq);
4312	5569	}
4313	5570
4314	5571	set_eq_cons_index_v2(eq);
4315	5572	return aeqe_found;
4316	5573	}
4317	5574
4318		-static struct hns_roce_ceqe get_ceqe_v2(struct hns_roce_eq eq, u32 entry)
4319		-{
4320		- u32 buf_chk_sz;
4321		- unsigned long off;
4322		-
4323		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4324		- off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
4325		-
4326		- return (struct hns_roce_ceqe )((char )(eq->buf_list->buf) +
4327		- off % buf_chk_sz);
4328		-}
4329		-
4330		-static struct hns_roce_ceqe mhop_get_ceqe(struct hns_roce_eq eq, u32 entry)
4331		-{
4332		- u32 buf_chk_sz;
4333		- unsigned long off;
4334		-
4335		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4336		-
4337		- off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
4338		-
4339		- if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4340		- return (struct hns_roce_ceqe )((u8 )(eq->bt_l0) +
4341		- off % buf_chk_sz);
4342		- else
4343		- return (struct hns_roce_ceqe )((u8 )(eq->buf[off /
4344		- buf_chk_sz]) + off % buf_chk_sz);
4345		-}
4346		-
4347	5575	static struct hns_roce_ceqe next_ceqe_sw_v2(struct hns_roce_eq eq)
4348	5576	{
4349	5577	struct hns_roce_ceqe *ceqe;
4350	5578
4351		- if (!eq->hop_num)
4352		- ceqe = get_ceqe_v2(eq, eq->cons_index);
4353		- else
4354		- ceqe = mhop_get_ceqe(eq, eq->cons_index);
	5579	+ ceqe = hns_roce_buf_offset(eq->mtr.kmem,
	5580	+ (eq->cons_index & (eq->entries - 1)) *
	5581	+ eq->eqe_size);
4355	5582
4356	5583	return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
4357	5584	(!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
..	..	@@ -4360,20 +5587,17 @@
4360	5587	static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
4361	5588	struct hns_roce_eq *eq)
4362	5589	{
4363		- struct device *dev = hr_dev->dev;
4364		- struct hns_roce_ceqe *ceqe;
	5590	+ struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
4365	5591	int ceqe_found = 0;
4366	5592	u32 cqn;
4367	5593
4368		- while ((ceqe = next_ceqe_sw_v2(eq))) {
4369		-
	5594	+ while (ceqe) {
4370	5595	/* Make sure we read CEQ entry after we have checked the
4371	5596	* ownership bit
4372	5597	*/
4373	5598	dma_rmb();
4374	5599
4375		- cqn = roce_get_field(ceqe->comp,
4376		- HNS_ROCE_V2_CEQE_COMP_CQN_M,
	5600	+ cqn = roce_get_field(ceqe->comp, HNS_ROCE_V2_CEQE_COMP_CQN_M,
4377	5601	HNS_ROCE_V2_CEQE_COMP_CQN_S);
4378	5602
4379	5603	hns_roce_cq_completion(hr_dev, cqn);
..	..	@@ -4381,10 +5605,10 @@
4381	5605	++eq->cons_index;
4382	5606	ceqe_found = 1;
4383	5607
4384		- if (eq->cons_index > (2 * eq->entries - 1)) {
4385		- dev_warn(dev, "cons_index overflow, set back to 0.\n");
	5608	+ if (eq->cons_index > (EQ_DEPTH_COEFF * eq->entries - 1))
4386	5609	eq->cons_index = 0;
4387		- }
	5610	+
	5611	+ ceqe = next_ceqe_sw_v2(eq);
4388	5612	}
4389	5613
4390	5614	set_eq_cons_index_v2(eq);
..	..	@@ -4396,7 +5620,7 @@
4396	5620	{
4397	5621	struct hns_roce_eq *eq = eq_ptr;
4398	5622	struct hns_roce_dev *hr_dev = eq->hr_dev;
4399		- int int_work = 0;
	5623	+ int int_work;
4400	5624
4401	5625	if (eq->type_flag == HNS_ROCE_CEQ)
4402	5626	/* Completion event interrupt */
..	..	@@ -4420,33 +5644,44 @@
4420	5644	int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
4421	5645	int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
4422	5646
4423		- if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
	5647	+ if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
	5648	+ struct pci_dev *pdev = hr_dev->pci_dev;
	5649	+ struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
	5650	+ const struct hnae3_ae_ops *ops = ae_dev->ops;
	5651	+
4424	5652	dev_err(dev, "AEQ overflow!\n");
4425	5653
4426		- roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1);
4427		- roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
	5654	+ roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG,
	5655	+ 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S);
4428	5656
4429		- roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
	5657	+ /* Set reset level for reset_event() */
	5658	+ if (ops->set_default_reset_request)
	5659	+ ops->set_default_reset_request(ae_dev,
	5660	+ HNAE3_FUNC_RESET);
	5661	+ if (ops->reset_event)
	5662	+ ops->reset_event(pdev, NULL);
	5663	+
	5664	+ int_en \|= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
4430	5665	roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4431	5666
4432	5667	int_work = 1;
4433		- } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
	5668	+ } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
4434	5669	dev_err(dev, "BUS ERR!\n");
4435	5670
4436		- roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1);
	5671	+ int_st \|= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S;
4437	5672	roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
4438	5673
4439		- roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
	5674	+ int_en \|= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
4440	5675	roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4441	5676
4442	5677	int_work = 1;
4443		- } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
	5678	+ } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
4444	5679	dev_err(dev, "OTHER ERR!\n");
4445	5680
4446		- roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1);
	5681	+ int_st \|= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S;
4447	5682	roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
4448	5683
4449		- roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
	5684	+ int_en \|= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
4450	5685	roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4451	5686
4452	5687	int_work = 1;
..	..	@@ -4501,502 +5736,187 @@
4501	5736	dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
4502	5737	}
4503	5738
4504		-static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
4505		- struct hns_roce_eq *eq)
	5739	+static void free_eq_buf(struct hns_roce_dev hr_dev, struct hns_roce_eq eq)
4506	5740	{
4507		- struct device *dev = hr_dev->dev;
4508		- u64 idx;
4509		- u64 size;
4510		- u32 buf_chk_sz;
4511		- u32 bt_chk_sz;
4512		- u32 mhop_num;
4513		- int eqe_alloc;
4514		- int i = 0;
4515		- int j = 0;
4516		-
4517		- mhop_num = hr_dev->caps.eqe_hop_num;
4518		- buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
4519		- bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
4520		-
4521		- /* hop_num = 0 */
4522		- if (mhop_num == HNS_ROCE_HOP_NUM_0) {
4523		- dma_free_coherent(dev, (unsigned int)(eq->entries *
4524		- eq->eqe_size), eq->bt_l0, eq->l0_dma);
4525		- return;
4526		- }
4527		-
4528		- /* hop_num = 1 or hop = 2 */
4529		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
4530		- if (mhop_num == 1) {
4531		- for (i = 0; i < eq->l0_last_num; i++) {
4532		- if (i == eq->l0_last_num - 1) {
4533		- eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
4534		- size = (eq->entries - eqe_alloc) * eq->eqe_size;
4535		- dma_free_coherent(dev, size, eq->buf[i],
4536		- eq->buf_dma[i]);
4537		- break;
4538		- }
4539		- dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
4540		- eq->buf_dma[i]);
4541		- }
4542		- } else if (mhop_num == 2) {
4543		- for (i = 0; i < eq->l0_last_num; i++) {
4544		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
4545		- eq->l1_dma[i]);
4546		-
4547		- for (j = 0; j < bt_chk_sz / 8; j++) {
4548		- idx = i * (bt_chk_sz / 8) + j;
4549		- if ((i == eq->l0_last_num - 1)
4550		- && j == eq->l1_last_num - 1) {
4551		- eqe_alloc = (buf_chk_sz / eq->eqe_size)
4552		- * idx;
4553		- size = (eq->entries - eqe_alloc)
4554		- * eq->eqe_size;
4555		- dma_free_coherent(dev, size,
4556		- eq->buf[idx],
4557		- eq->buf_dma[idx]);
4558		- break;
4559		- }
4560		- dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
4561		- eq->buf_dma[idx]);
4562		- }
4563		- }
4564		- }
4565		- kfree(eq->buf_dma);
4566		- kfree(eq->buf);
4567		- kfree(eq->l1_dma);
4568		- kfree(eq->bt_l1);
4569		- eq->buf_dma = NULL;
4570		- eq->buf = NULL;
4571		- eq->l1_dma = NULL;
4572		- eq->bt_l1 = NULL;
	5741	+ hns_roce_mtr_destroy(hr_dev, &eq->mtr);
4573	5742	}
4574	5743
4575		-static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
4576		- struct hns_roce_eq *eq)
	5744	+static int config_eqc(struct hns_roce_dev hr_dev, struct hns_roce_eq eq,
	5745	+ void *mb_buf)
4577	5746	{
4578		- u32 buf_chk_sz;
4579		-
4580		- buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4581		-
4582		- if (hr_dev->caps.eqe_hop_num) {
4583		- hns_roce_mhop_free_eq(hr_dev, eq);
4584		- return;
4585		- }
4586		-
4587		- dma_free_coherent(hr_dev->dev, buf_chk_sz, eq->buf_list->buf,
4588		- eq->buf_list->map);
4589		- kfree(eq->buf_list);
4590		-}
4591		-
4592		-static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
4593		- struct hns_roce_eq *eq,
4594		- void *mb_buf)
4595		-{
	5747	+ u64 eqe_ba[MTT_MIN_COUNT] = { 0 };
4596	5748	struct hns_roce_eq_context *eqc;
	5749	+ u64 bt_ba = 0;
	5750	+ int count;
4597	5751
4598	5752	eqc = mb_buf;
4599	5753	memset(eqc, 0, sizeof(struct hns_roce_eq_context));
4600	5754
4601	5755	/* init eqc */
4602	5756	eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
4603		- eq->hop_num = hr_dev->caps.eqe_hop_num;
4604	5757	eq->cons_index = 0;
4605	5758	eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
4606	5759	eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
4607	5760	eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
4608		- eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
4609		- eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
4610	5761	eq->shift = ilog2((unsigned int)eq->entries);
4611	5762
4612		- if (!eq->hop_num)
4613		- eq->eqe_ba = eq->buf_list->map;
4614		- else
4615		- eq->eqe_ba = eq->l0_dma;
	5763	+ /* if not multi-hop, eqe buffer only use one trunk */
	5764	+ count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT,
	5765	+ &bt_ba);
	5766	+ if (count < 1) {
	5767	+ dev_err(hr_dev->dev, "failed to find EQE mtr\n");
	5768	+ return -ENOBUFS;
	5769	+ }
4616	5770
4617	5771	/* set eqc state */
4618		- roce_set_field(eqc->byte_4,
4619		- HNS_ROCE_EQC_EQ_ST_M,
4620		- HNS_ROCE_EQC_EQ_ST_S,
	5772	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQ_ST_M, HNS_ROCE_EQC_EQ_ST_S,
4621	5773	HNS_ROCE_V2_EQ_STATE_VALID);
4622	5774
4623	5775	/* set eqe hop num */
4624		- roce_set_field(eqc->byte_4,
4625		- HNS_ROCE_EQC_HOP_NUM_M,
	5776	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_HOP_NUM_M,
4626	5777	HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
4627	5778
4628	5779	/* set eqc over_ignore */
4629		- roce_set_field(eqc->byte_4,
4630		- HNS_ROCE_EQC_OVER_IGNORE_M,
	5780	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_OVER_IGNORE_M,
4631	5781	HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
4632	5782
4633	5783	/* set eqc coalesce */
4634		- roce_set_field(eqc->byte_4,
4635		- HNS_ROCE_EQC_COALESCE_M,
	5784	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_COALESCE_M,
4636	5785	HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
4637	5786
4638	5787	/* set eqc arm_state */
4639		- roce_set_field(eqc->byte_4,
4640		- HNS_ROCE_EQC_ARM_ST_M,
	5788	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_ARM_ST_M,
4641	5789	HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
4642	5790
4643	5791	/* set eqn */
4644		- roce_set_field(eqc->byte_4,
4645		- HNS_ROCE_EQC_EQN_M,
4646		- HNS_ROCE_EQC_EQN_S, eq->eqn);
	5792	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQN_M, HNS_ROCE_EQC_EQN_S,
	5793	+ eq->eqn);
4647	5794
4648	5795	/* set eqe_cnt */
4649		- roce_set_field(eqc->byte_4,
4650		- HNS_ROCE_EQC_EQE_CNT_M,
4651		- HNS_ROCE_EQC_EQE_CNT_S,
4652		- HNS_ROCE_EQ_INIT_EQE_CNT);
	5796	+ roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQE_CNT_M,
	5797	+ HNS_ROCE_EQC_EQE_CNT_S, HNS_ROCE_EQ_INIT_EQE_CNT);
4653	5798
4654	5799	/* set eqe_ba_pg_sz */
4655		- roce_set_field(eqc->byte_8,
4656		- HNS_ROCE_EQC_BA_PG_SZ_M,
	5800	+ roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M,
4657	5801	HNS_ROCE_EQC_BA_PG_SZ_S,
4658		- eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
	5802	+ to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift));
4659	5803
4660	5804	/* set eqe_buf_pg_sz */
4661		- roce_set_field(eqc->byte_8,
4662		- HNS_ROCE_EQC_BUF_PG_SZ_M,
	5805	+ roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M,
4663	5806	HNS_ROCE_EQC_BUF_PG_SZ_S,
4664		- eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
	5807	+ to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift));
4665	5808
4666	5809	/* set eq_producer_idx */
4667		- roce_set_field(eqc->byte_8,
4668		- HNS_ROCE_EQC_PROD_INDX_M,
4669		- HNS_ROCE_EQC_PROD_INDX_S,
4670		- HNS_ROCE_EQ_INIT_PROD_IDX);
	5810	+ roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M,
	5811	+ HNS_ROCE_EQC_PROD_INDX_S, HNS_ROCE_EQ_INIT_PROD_IDX);
4671	5812
4672	5813	/* set eq_max_cnt */
4673		- roce_set_field(eqc->byte_12,
4674		- HNS_ROCE_EQC_MAX_CNT_M,
	5814	+ roce_set_field(eqc->byte_12, HNS_ROCE_EQC_MAX_CNT_M,
4675	5815	HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
4676	5816
4677	5817	/* set eq_period */
4678		- roce_set_field(eqc->byte_12,
4679		- HNS_ROCE_EQC_PERIOD_M,
	5818	+ roce_set_field(eqc->byte_12, HNS_ROCE_EQC_PERIOD_M,
4680	5819	HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
4681	5820
4682	5821	/* set eqe_report_timer */
4683		- roce_set_field(eqc->eqe_report_timer,
4684		- HNS_ROCE_EQC_REPORT_TIMER_M,
	5822	+ roce_set_field(eqc->eqe_report_timer, HNS_ROCE_EQC_REPORT_TIMER_M,
4685	5823	HNS_ROCE_EQC_REPORT_TIMER_S,
4686	5824	HNS_ROCE_EQ_INIT_REPORT_TIMER);
4687	5825
4688		- /* set eqe_ba [34:3] */
4689		- roce_set_field(eqc->eqe_ba0,
4690		- HNS_ROCE_EQC_EQE_BA_L_M,
4691		- HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
	5826	+ /* set bt_ba [34:3] */
	5827	+ roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M,
	5828	+ HNS_ROCE_EQC_EQE_BA_L_S, bt_ba >> 3);
4692	5829
4693		- /* set eqe_ba [64:35] */
4694		- roce_set_field(eqc->eqe_ba1,
4695		- HNS_ROCE_EQC_EQE_BA_H_M,
4696		- HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
	5830	+ /* set bt_ba [64:35] */
	5831	+ roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M,
	5832	+ HNS_ROCE_EQC_EQE_BA_H_S, bt_ba >> 35);
4697	5833
4698	5834	/* set eq shift */
4699		- roce_set_field(eqc->byte_28,
4700		- HNS_ROCE_EQC_SHIFT_M,
4701		- HNS_ROCE_EQC_SHIFT_S, eq->shift);
	5835	+ roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S,
	5836	+ eq->shift);
4702	5837
4703	5838	/* set eq MSI_IDX */
4704		- roce_set_field(eqc->byte_28,
4705		- HNS_ROCE_EQC_MSI_INDX_M,
4706		- HNS_ROCE_EQC_MSI_INDX_S,
4707		- HNS_ROCE_EQ_INIT_MSI_IDX);
	5839	+ roce_set_field(eqc->byte_28, HNS_ROCE_EQC_MSI_INDX_M,
	5840	+ HNS_ROCE_EQC_MSI_INDX_S, HNS_ROCE_EQ_INIT_MSI_IDX);
4708	5841
4709	5842	/* set cur_eqe_ba [27:12] */
4710		- roce_set_field(eqc->byte_28,
4711		- HNS_ROCE_EQC_CUR_EQE_BA_L_M,
4712		- HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
	5843	+ roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M,
	5844	+ HNS_ROCE_EQC_CUR_EQE_BA_L_S, eqe_ba[0] >> 12);
4713	5845
4714	5846	/* set cur_eqe_ba [59:28] */
4715		- roce_set_field(eqc->byte_32,
4716		- HNS_ROCE_EQC_CUR_EQE_BA_M_M,
4717		- HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
	5847	+ roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M,
	5848	+ HNS_ROCE_EQC_CUR_EQE_BA_M_S, eqe_ba[0] >> 28);
4718	5849
4719	5850	/* set cur_eqe_ba [63:60] */
4720		- roce_set_field(eqc->byte_36,
4721		- HNS_ROCE_EQC_CUR_EQE_BA_H_M,
4722		- HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
	5851	+ roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M,
	5852	+ HNS_ROCE_EQC_CUR_EQE_BA_H_S, eqe_ba[0] >> 60);
4723	5853
4724	5854	/* set eq consumer idx */
4725		- roce_set_field(eqc->byte_36,
4726		- HNS_ROCE_EQC_CONS_INDX_M,
4727		- HNS_ROCE_EQC_CONS_INDX_S,
4728		- HNS_ROCE_EQ_INIT_CONS_IDX);
	5855	+ roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M,
	5856	+ HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX);
4729	5857
4730		- /* set nex_eqe_ba[43:12] */
4731		- roce_set_field(eqc->nxt_eqe_ba0,
4732		- HNS_ROCE_EQC_NXT_EQE_BA_L_M,
4733		- HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
	5858	+ roce_set_field(eqc->byte_40, HNS_ROCE_EQC_NXT_EQE_BA_L_M,
	5859	+ HNS_ROCE_EQC_NXT_EQE_BA_L_S, eqe_ba[1] >> 12);
4734	5860
4735		- /* set nex_eqe_ba[63:44] */
4736		- roce_set_field(eqc->nxt_eqe_ba1,
4737		- HNS_ROCE_EQC_NXT_EQE_BA_H_M,
4738		- HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
4739		-}
	5861	+ roce_set_field(eqc->byte_44, HNS_ROCE_EQC_NXT_EQE_BA_H_M,
	5862	+ HNS_ROCE_EQC_NXT_EQE_BA_H_S, eqe_ba[1] >> 44);
4740	5863
4741		-static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
4742		- struct hns_roce_eq *eq)
4743		-{
4744		- struct device *dev = hr_dev->dev;
4745		- int eq_alloc_done = 0;
4746		- int eq_buf_cnt = 0;
4747		- int eqe_alloc;
4748		- u32 buf_chk_sz;
4749		- u32 bt_chk_sz;
4750		- u32 mhop_num;
4751		- u64 size;
4752		- u64 idx;
4753		- int ba_num;
4754		- int bt_num;
4755		- int record_i;
4756		- int record_j;
4757		- int i = 0;
4758		- int j = 0;
4759		-
4760		- mhop_num = hr_dev->caps.eqe_hop_num;
4761		- buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
4762		- bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
4763		-
4764		- ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1)
4765		- / buf_chk_sz;
4766		- bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8);
4767		-
4768		- /* hop_num = 0 */
4769		- if (mhop_num == HNS_ROCE_HOP_NUM_0) {
4770		- if (eq->entries > buf_chk_sz / eq->eqe_size) {
4771		- dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
4772		- eq->entries);
4773		- return -EINVAL;
4774		- }
4775		- eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
4776		- &(eq->l0_dma), GFP_KERNEL);
4777		- if (!eq->bt_l0)
4778		- return -ENOMEM;
4779		-
4780		- eq->cur_eqe_ba = eq->l0_dma;
4781		- eq->nxt_eqe_ba = 0;
4782		-
4783		- memset(eq->bt_l0, 0, eq->entries * eq->eqe_size);
4784		-
4785		- return 0;
4786		- }
4787		-
4788		- eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
4789		- if (!eq->buf_dma)
4790		- return -ENOMEM;
4791		- eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
4792		- if (!eq->buf)
4793		- goto err_kcalloc_buf;
4794		-
4795		- if (mhop_num == 2) {
4796		- eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
4797		- if (!eq->l1_dma)
4798		- goto err_kcalloc_l1_dma;
4799		-
4800		- eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
4801		- if (!eq->bt_l1)
4802		- goto err_kcalloc_bt_l1;
4803		- }
4804		-
4805		- /* alloc L0 BT */
4806		- eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
4807		- if (!eq->bt_l0)
4808		- goto err_dma_alloc_l0;
4809		-
4810		- if (mhop_num == 1) {
4811		- if (ba_num > (bt_chk_sz / 8))
4812		- dev_err(dev, "ba_num %d is too large for 1 hop\n",
4813		- ba_num);
4814		-
4815		- /* alloc buf */
4816		- for (i = 0; i < bt_chk_sz / 8; i++) {
4817		- if (eq_buf_cnt + 1 < ba_num) {
4818		- size = buf_chk_sz;
4819		- } else {
4820		- eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
4821		- size = (eq->entries - eqe_alloc) * eq->eqe_size;
4822		- }
4823		- eq->buf[i] = dma_alloc_coherent(dev, size,
4824		- &(eq->buf_dma[i]),
4825		- GFP_KERNEL);
4826		- if (!eq->buf[i])
4827		- goto err_dma_alloc_buf;
4828		-
4829		- memset(eq->buf[i], 0, size);
4830		- *(eq->bt_l0 + i) = eq->buf_dma[i];
4831		-
4832		- eq_buf_cnt++;
4833		- if (eq_buf_cnt >= ba_num)
4834		- break;
4835		- }
4836		- eq->cur_eqe_ba = eq->buf_dma[0];
4837		- if (ba_num > 1)
4838		- eq->nxt_eqe_ba = eq->buf_dma[1];
4839		-
4840		- } else if (mhop_num == 2) {
4841		- /* alloc L1 BT and buf */
4842		- for (i = 0; i < bt_chk_sz / 8; i++) {
4843		- eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
4844		- &(eq->l1_dma[i]),
4845		- GFP_KERNEL);
4846		- if (!eq->bt_l1[i])
4847		- goto err_dma_alloc_l1;
4848		- *(eq->bt_l0 + i) = eq->l1_dma[i];
4849		-
4850		- for (j = 0; j < bt_chk_sz / 8; j++) {
4851		- idx = i * bt_chk_sz / 8 + j;
4852		- if (eq_buf_cnt + 1 < ba_num) {
4853		- size = buf_chk_sz;
4854		- } else {
4855		- eqe_alloc = (buf_chk_sz / eq->eqe_size)
4856		- * idx;
4857		- size = (eq->entries - eqe_alloc)
4858		- * eq->eqe_size;
4859		- }
4860		- eq->buf[idx] = dma_alloc_coherent(dev, size,
4861		- &(eq->buf_dma[idx]),
4862		- GFP_KERNEL);
4863		- if (!eq->buf[idx])
4864		- goto err_dma_alloc_buf;
4865		-
4866		- memset(eq->buf[idx], 0, size);
4867		- *(eq->bt_l1[i] + j) = eq->buf_dma[idx];
4868		-
4869		- eq_buf_cnt++;
4870		- if (eq_buf_cnt >= ba_num) {
4871		- eq_alloc_done = 1;
4872		- break;
4873		- }
4874		- }
4875		-
4876		- if (eq_alloc_done)
4877		- break;
4878		- }
4879		- eq->cur_eqe_ba = eq->buf_dma[0];
4880		- if (ba_num > 1)
4881		- eq->nxt_eqe_ba = eq->buf_dma[1];
4882		- }
4883		-
4884		- eq->l0_last_num = i + 1;
4885		- if (mhop_num == 2)
4886		- eq->l1_last_num = j + 1;
	5864	+ roce_set_field(eqc->byte_44, HNS_ROCE_EQC_EQE_SIZE_M,
	5865	+ HNS_ROCE_EQC_EQE_SIZE_S,
	5866	+ eq->eqe_size == HNS_ROCE_V3_EQE_SIZE ? 1 : 0);
4887	5867
4888	5868	return 0;
	5869	+}
4889	5870
4890		-err_dma_alloc_l1:
4891		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
4892		- eq->bt_l0 = NULL;
4893		- eq->l0_dma = 0;
4894		- for (i -= 1; i >= 0; i--) {
4895		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
4896		- eq->l1_dma[i]);
	5871	+static int alloc_eq_buf(struct hns_roce_dev hr_dev, struct hns_roce_eq eq)
	5872	+{
	5873	+ struct hns_roce_buf_attr buf_attr = {};
	5874	+ int err;
4897	5875
4898		- for (j = 0; j < bt_chk_sz / 8; j++) {
4899		- idx = i * bt_chk_sz / 8 + j;
4900		- dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
4901		- eq->buf_dma[idx]);
4902		- }
4903		- }
4904		- goto err_dma_alloc_l0;
	5876	+ if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)
	5877	+ eq->hop_num = 0;
	5878	+ else
	5879	+ eq->hop_num = hr_dev->caps.eqe_hop_num;
4905	5880
4906		-err_dma_alloc_buf:
4907		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
4908		- eq->bt_l0 = NULL;
4909		- eq->l0_dma = 0;
	5881	+ buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + HNS_HW_PAGE_SHIFT;
	5882	+ buf_attr.region[0].size = eq->entries * eq->eqe_size;
	5883	+ buf_attr.region[0].hopnum = eq->hop_num;
	5884	+ buf_attr.region_count = 1;
	5885	+ buf_attr.fixed_page = true;
4910	5886
4911		- if (mhop_num == 1)
4912		- for (i -= 1; i >= 0; i--)
4913		- dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
4914		- eq->buf_dma[i]);
4915		- else if (mhop_num == 2) {
4916		- record_i = i;
4917		- record_j = j;
4918		- for (; i >= 0; i--) {
4919		- dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
4920		- eq->l1_dma[i]);
	5887	+ err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
	5888	+ hr_dev->caps.eqe_ba_pg_sz +
	5889	+ HNS_HW_PAGE_SHIFT, NULL, 0);
	5890	+ if (err)
	5891	+ dev_err(hr_dev->dev, "Failed to alloc EQE mtr, err %d\n", err);
4921	5892
4922		- for (j = 0; j < bt_chk_sz / 8; j++) {
4923		- if (i == record_i && j >= record_j)
4924		- break;
4925		-
4926		- idx = i * bt_chk_sz / 8 + j;
4927		- dma_free_coherent(dev, buf_chk_sz,
4928		- eq->buf[idx],
4929		- eq->buf_dma[idx]);
4930		- }
4931		- }
4932		- }
4933		-
4934		-err_dma_alloc_l0:
4935		- kfree(eq->bt_l1);
4936		- eq->bt_l1 = NULL;
4937		-
4938		-err_kcalloc_bt_l1:
4939		- kfree(eq->l1_dma);
4940		- eq->l1_dma = NULL;
4941		-
4942		-err_kcalloc_l1_dma:
4943		- kfree(eq->buf);
4944		- eq->buf = NULL;
4945		-
4946		-err_kcalloc_buf:
4947		- kfree(eq->buf_dma);
4948		- eq->buf_dma = NULL;
4949		-
4950		- return -ENOMEM;
	5893	+ return err;
4951	5894	}
4952	5895
4953	5896	static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
4954	5897	struct hns_roce_eq *eq,
4955	5898	unsigned int eq_cmd)
4956	5899	{
4957		- struct device *dev = hr_dev->dev;
4958	5900	struct hns_roce_cmd_mailbox *mailbox;
4959		- u32 buf_chk_sz = 0;
4960	5901	int ret;
4961	5902
4962	5903	/* Allocate mailbox memory */
4963	5904	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4964		- if (IS_ERR(mailbox))
4965		- return PTR_ERR(mailbox);
	5905	+ if (IS_ERR_OR_NULL(mailbox))
	5906	+ return -ENOMEM;
4966	5907
4967		- if (!hr_dev->caps.eqe_hop_num) {
4968		- buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
	5908	+ ret = alloc_eq_buf(hr_dev, eq);
	5909	+ if (ret)
	5910	+ goto free_cmd_mbox;
4969	5911
4970		- eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
4971		- GFP_KERNEL);
4972		- if (!eq->buf_list) {
4973		- ret = -ENOMEM;
4974		- goto free_cmd_mbox;
4975		- }
4976		-
4977		- eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
4978		- &(eq->buf_list->map),
4979		- GFP_KERNEL);
4980		- if (!eq->buf_list->buf) {
4981		- ret = -ENOMEM;
4982		- goto err_alloc_buf;
4983		- }
4984		-
4985		- memset(eq->buf_list->buf, 0, buf_chk_sz);
4986		- } else {
4987		- ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
4988		- if (ret) {
4989		- ret = -ENOMEM;
4990		- goto free_cmd_mbox;
4991		- }
4992		- }
4993		-
4994		- hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
	5912	+ ret = config_eqc(hr_dev, eq, mailbox->buf);
	5913	+ if (ret)
	5914	+ goto err_cmd_mbox;
4995	5915
4996	5916	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
4997	5917	eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
4998	5918	if (ret) {
4999		- dev_err(dev, "[mailbox cmd] create eqc failed.\n");
	5919	+ dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n");
5000	5920	goto err_cmd_mbox;
5001	5921	}
5002	5922
..	..	@@ -5005,21 +5925,99 @@
5005	5925	return 0;
5006	5926
5007	5927	err_cmd_mbox:
5008		- if (!hr_dev->caps.eqe_hop_num)
5009		- dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
5010		- eq->buf_list->map);
5011		- else {
5012		- hns_roce_mhop_free_eq(hr_dev, eq);
5013		- goto free_cmd_mbox;
5014		- }
5015		-
5016		-err_alloc_buf:
5017		- kfree(eq->buf_list);
	5928	+ free_eq_buf(hr_dev, eq);
5018	5929
5019	5930	free_cmd_mbox:
5020	5931	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5021	5932
5022	5933	return ret;
	5934	+}
	5935	+
	5936	+static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
	5937	+ int comp_num, int aeq_num, int other_num)
	5938	+{
	5939	+ struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
	5940	+ int i, j;
	5941	+ int ret;
	5942	+
	5943	+ for (i = 0; i < irq_num; i++) {
	5944	+ hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
	5945	+ GFP_KERNEL);
	5946	+ if (!hr_dev->irq_names[i]) {
	5947	+ ret = -ENOMEM;
	5948	+ goto err_kzalloc_failed;
	5949	+ }
	5950	+ }
	5951	+
	5952	+ /* irq contains: abnormal + AEQ + CEQ */
	5953	+ for (j = 0; j < other_num; j++)
	5954	+ snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
	5955	+ "hns-abn-%d", j);
	5956	+
	5957	+ for (j = other_num; j < (other_num + aeq_num); j++)
	5958	+ snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
	5959	+ "hns-aeq-%d", j - other_num);
	5960	+
	5961	+ for (j = (other_num + aeq_num); j < irq_num; j++)
	5962	+ snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
	5963	+ "hns-ceq-%d", j - other_num - aeq_num);
	5964	+
	5965	+ for (j = 0; j < irq_num; j++) {
	5966	+ if (j < other_num)
	5967	+ ret = request_irq(hr_dev->irq[j],
	5968	+ hns_roce_v2_msix_interrupt_abn,
	5969	+ 0, hr_dev->irq_names[j], hr_dev);
	5970	+
	5971	+ else if (j < (other_num + comp_num))
	5972	+ ret = request_irq(eq_table->eq[j - other_num].irq,
	5973	+ hns_roce_v2_msix_interrupt_eq,
	5974	+ 0, hr_dev->irq_names[j + aeq_num],
	5975	+ &eq_table->eq[j - other_num]);
	5976	+ else
	5977	+ ret = request_irq(eq_table->eq[j - other_num].irq,
	5978	+ hns_roce_v2_msix_interrupt_eq,
	5979	+ 0, hr_dev->irq_names[j - comp_num],
	5980	+ &eq_table->eq[j - other_num]);
	5981	+ if (ret) {
	5982	+ dev_err(hr_dev->dev, "Request irq error!\n");
	5983	+ goto err_request_failed;
	5984	+ }
	5985	+ }
	5986	+
	5987	+ return 0;
	5988	+
	5989	+err_request_failed:
	5990	+ for (j -= 1; j >= 0; j--)
	5991	+ if (j < other_num)
	5992	+ free_irq(hr_dev->irq[j], hr_dev);
	5993	+ else
	5994	+ free_irq(eq_table->eq[j - other_num].irq,
	5995	+ &eq_table->eq[j - other_num]);
	5996	+
	5997	+err_kzalloc_failed:
	5998	+ for (i -= 1; i >= 0; i--)
	5999	+ kfree(hr_dev->irq_names[i]);
	6000	+
	6001	+ return ret;
	6002	+}
	6003	+
	6004	+static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
	6005	+{
	6006	+ int irq_num;
	6007	+ int eq_num;
	6008	+ int i;
	6009	+
	6010	+ eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
	6011	+ irq_num = eq_num + hr_dev->caps.num_other_vectors;
	6012	+
	6013	+ for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
	6014	+ free_irq(hr_dev->irq[i], hr_dev);
	6015	+
	6016	+ for (i = 0; i < eq_num; i++)
	6017	+ free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
	6018	+
	6019	+ for (i = 0; i < irq_num; i++)
	6020	+ kfree(hr_dev->irq_names[i]);
5023	6021	}
5024	6022
5025	6023	static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
..	..	@@ -5033,7 +6031,7 @@
5033	6031	int other_num;
5034	6032	int comp_num;
5035	6033	int aeq_num;
5036		- int i, j, k;
	6034	+ int i;
5037	6035	int ret;
5038	6036
5039	6037	other_num = hr_dev->caps.num_other_vectors;
..	..	@@ -5047,27 +6045,18 @@
5047	6045	if (!eq_table->eq)
5048	6046	return -ENOMEM;
5049	6047
5050		- for (i = 0; i < irq_num; i++) {
5051		- hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5052		- GFP_KERNEL);
5053		- if (!hr_dev->irq_names[i]) {
5054		- ret = -ENOMEM;
5055		- goto err_failed_kzalloc;
5056		- }
5057		- }
5058		-
5059	6048	/* create eq */
5060		- for (j = 0; j < eq_num; j++) {
5061		- eq = &eq_table->eq[j];
	6049	+ for (i = 0; i < eq_num; i++) {
	6050	+ eq = &eq_table->eq[i];
5062	6051	eq->hr_dev = hr_dev;
5063		- eq->eqn = j;
5064		- if (j < comp_num) {
	6052	+ eq->eqn = i;
	6053	+ if (i < comp_num) {
5065	6054	/* CEQ */
5066	6055	eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
5067	6056	eq->type_flag = HNS_ROCE_CEQ;
5068	6057	eq->entries = hr_dev->caps.ceqe_depth;
5069		- eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
5070		- eq->irq = hr_dev->irq[j + other_num + aeq_num];
	6058	+ eq->eqe_size = hr_dev->caps.ceqe_size;
	6059	+ eq->irq = hr_dev->irq[i + other_num + aeq_num];
5071	6060	eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
5072	6061	eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
5073	6062	} else {
..	..	@@ -5075,8 +6064,8 @@
5075	6064	eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
5076	6065	eq->type_flag = HNS_ROCE_AEQ;
5077	6066	eq->entries = hr_dev->caps.aeqe_depth;
5078		- eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
5079		- eq->irq = hr_dev->irq[j - comp_num + other_num];
	6067	+ eq->eqe_size = hr_dev->caps.aeqe_size;
	6068	+ eq->irq = hr_dev->irq[i - comp_num + other_num];
5080	6069	eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
5081	6070	eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
5082	6071	}
..	..	@@ -5091,67 +6080,31 @@
5091	6080	/* enable irq */
5092	6081	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
5093	6082
5094		- /* irq contains: abnormal + AEQ + CEQ*/
5095		- for (k = 0; k < irq_num; k++)
5096		- if (k < other_num)
5097		- snprintf((char *)hr_dev->irq_names[k],
5098		- HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k);
5099		- else if (k < (other_num + aeq_num))
5100		- snprintf((char *)hr_dev->irq_names[k],
5101		- HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",
5102		- k - other_num);
5103		- else
5104		- snprintf((char *)hr_dev->irq_names[k],
5105		- HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",
5106		- k - other_num - aeq_num);
5107		-
5108		- for (k = 0; k < irq_num; k++) {
5109		- if (k < other_num)
5110		- ret = request_irq(hr_dev->irq[k],
5111		- hns_roce_v2_msix_interrupt_abn,
5112		- 0, hr_dev->irq_names[k], hr_dev);
5113		-
5114		- else if (k < (other_num + comp_num))
5115		- ret = request_irq(eq_table->eq[k - other_num].irq,
5116		- hns_roce_v2_msix_interrupt_eq,
5117		- 0, hr_dev->irq_names[k + aeq_num],
5118		- &eq_table->eq[k - other_num]);
5119		- else
5120		- ret = request_irq(eq_table->eq[k - other_num].irq,
5121		- hns_roce_v2_msix_interrupt_eq,
5122		- 0, hr_dev->irq_names[k - comp_num],
5123		- &eq_table->eq[k - other_num]);
5124		- if (ret) {
5125		- dev_err(dev, "Request irq error!\n");
5126		- goto err_request_irq_fail;
5127		- }
	6083	+ ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num,
	6084	+ aeq_num, other_num);
	6085	+ if (ret) {
	6086	+ dev_err(dev, "Request irq failed.\n");
	6087	+ goto err_request_irq_fail;
5128	6088	}
5129	6089
5130		- hr_dev->irq_workq =
5131		- create_singlethread_workqueue("hns_roce_irq_workqueue");
	6090	+ hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0);
5132	6091	if (!hr_dev->irq_workq) {
5133	6092	dev_err(dev, "Create irq workqueue failed!\n");
5134	6093	ret = -ENOMEM;
5135		- goto err_request_irq_fail;
	6094	+ goto err_create_wq_fail;
5136	6095	}
5137	6096
5138	6097	return 0;
5139	6098
	6099	+err_create_wq_fail:
	6100	+ __hns_roce_free_irq(hr_dev);
	6101	+
5140	6102	err_request_irq_fail:
5141		- for (k -= 1; k >= 0; k--)
5142		- if (k < other_num)
5143		- free_irq(hr_dev->irq[k], hr_dev);
5144		- else
5145		- free_irq(eq_table->eq[k - other_num].irq,
5146		- &eq_table->eq[k - other_num]);
	6103	+ hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5147	6104
5148	6105	err_create_eq_fail:
5149		- for (j -= 1; j >= 0; j--)
5150		- hns_roce_v2_free_eq(hr_dev, &eq_table->eq[j]);
5151		-
5152		-err_failed_kzalloc:
5153	6106	for (i -= 1; i >= 0; i--)
5154		- kfree(hr_dev->irq_names[i]);
	6107	+ free_eq_buf(hr_dev, &eq_table->eq[i]);
5155	6108	kfree(eq_table->eq);
5156	6109
5157	6110	return ret;
..	..	@@ -5160,35 +6113,47 @@
5160	6113	static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
5161	6114	{
5162	6115	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5163		- int irq_num;
5164	6116	int eq_num;
5165	6117	int i;
5166	6118
5167	6119	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
5168		- irq_num = eq_num + hr_dev->caps.num_other_vectors;
5169	6120
5170	6121	/* Disable irq */
5171	6122	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5172	6123
5173		- for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
5174		- free_irq(hr_dev->irq[i], hr_dev);
	6124	+ __hns_roce_free_irq(hr_dev);
5175	6125
5176	6126	for (i = 0; i < eq_num; i++) {
5177	6127	hns_roce_v2_destroy_eqc(hr_dev, i);
5178	6128
5179		- free_irq(eq_table->eq[i].irq, &eq_table->eq[i]);
5180		-
5181		- hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
	6129	+ free_eq_buf(hr_dev, &eq_table->eq[i]);
5182	6130	}
5183		-
5184		- for (i = 0; i < irq_num; i++)
5185		- kfree(hr_dev->irq_names[i]);
5186	6131
5187	6132	kfree(eq_table->eq);
5188	6133
5189	6134	flush_workqueue(hr_dev->irq_workq);
5190	6135	destroy_workqueue(hr_dev->irq_workq);
5191	6136	}
	6137	+
	6138	+static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
	6139	+ .query_cqc_info = hns_roce_v2_query_cqc_info,
	6140	+};
	6141	+
	6142	+static const struct ib_device_ops hns_roce_v2_dev_ops = {
	6143	+ .destroy_qp = hns_roce_v2_destroy_qp,
	6144	+ .modify_cq = hns_roce_v2_modify_cq,
	6145	+ .poll_cq = hns_roce_v2_poll_cq,
	6146	+ .post_recv = hns_roce_v2_post_recv,
	6147	+ .post_send = hns_roce_v2_post_send,
	6148	+ .query_qp = hns_roce_v2_query_qp,
	6149	+ .req_notify_cq = hns_roce_v2_req_notify_cq,
	6150	+};
	6151	+
	6152	+static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
	6153	+ .modify_srq = hns_roce_v2_modify_srq,
	6154	+ .post_srq_recv = hns_roce_v2_post_srq_recv,
	6155	+ .query_srq = hns_roce_v2_query_srq,
	6156	+};
5192	6157
5193	6158	static const struct hns_roce_hw hns_roce_hw_v2 = {
5194	6159	.cmq_init = hns_roce_v2_cmq_init,
..	..	@@ -5198,16 +6163,20 @@
5198	6163	.hw_exit = hns_roce_v2_exit,
5199	6164	.post_mbox = hns_roce_v2_post_mbox,
5200	6165	.chk_mbox = hns_roce_v2_chk_mbox,
	6166	+ .rst_prc_mbox = hns_roce_v2_rst_process_cmd,
5201	6167	.set_gid = hns_roce_v2_set_gid,
5202	6168	.set_mac = hns_roce_v2_set_mac,
5203	6169	.write_mtpt = hns_roce_v2_write_mtpt,
5204	6170	.rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
	6171	+ .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
	6172	+ .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
5205	6173	.write_cqc = hns_roce_v2_write_cqc,
5206	6174	.set_hem = hns_roce_v2_set_hem,
5207	6175	.clear_hem = hns_roce_v2_clear_hem,
5208	6176	.modify_qp = hns_roce_v2_modify_qp,
5209	6177	.query_qp = hns_roce_v2_query_qp,
5210	6178	.destroy_qp = hns_roce_v2_destroy_qp,
	6179	+ .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
5211	6180	.modify_cq = hns_roce_v2_modify_cq,
5212	6181	.post_send = hns_roce_v2_post_send,
5213	6182	.post_recv = hns_roce_v2_post_recv,
..	..	@@ -5215,6 +6184,12 @@
5215	6184	.poll_cq = hns_roce_v2_poll_cq,
5216	6185	.init_eq = hns_roce_v2_init_eq_table,
5217	6186	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
	6187	+ .write_srqc = hns_roce_v2_write_srqc,
	6188	+ .modify_srq = hns_roce_v2_modify_srq,
	6189	+ .query_srq = hns_roce_v2_query_srq,
	6190	+ .post_srq_recv = hns_roce_v2_post_srq_recv,
	6191	+ .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
	6192	+ .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
5218	6193	};
5219	6194
5220	6195	static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
..	..	@@ -5229,19 +6204,16 @@
5229	6204
5230	6205	MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
5231	6206
5232		-static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
	6207	+static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
5233	6208	struct hnae3_handle *handle)
5234	6209	{
5235		- const struct pci_device_id *id;
	6210	+ struct hns_roce_v2_priv *priv = hr_dev->priv;
5236	6211	int i;
5237	6212
5238		- id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);
5239		- if (!id) {
5240		- dev_err(hr_dev->dev, "device is not compatible!\n");
5241		- return -ENXIO;
5242		- }
5243		-
	6213	+ hr_dev->pci_dev = handle->pdev;
	6214	+ hr_dev->dev = &handle->pdev->dev;
5244	6215	hr_dev->hw = &hns_roce_hw_v2;
	6216	+ hr_dev->dfx = &hns_roce_dfx_hw_v2;
5245	6217	hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
5246	6218	hr_dev->odb_offset = hr_dev->sdb_offset;
5247	6219
..	..	@@ -5262,15 +6234,16 @@
5262	6234	hr_dev->cmd_mod = 1;
5263	6235	hr_dev->loop_idc = 0;
5264	6236
5265		- return 0;
	6237	+ hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
	6238	+ priv->handle = handle;
5266	6239	}
5267	6240
5268		-static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
	6241	+static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
5269	6242	{
5270	6243	struct hns_roce_dev *hr_dev;
5271	6244	int ret;
5272	6245
5273		- hr_dev = (struct hns_roce_dev )ib_alloc_device(sizeof(hr_dev));
	6246	+ hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
5274	6247	if (!hr_dev)
5275	6248	return -ENOMEM;
5276	6249
..	..	@@ -5280,21 +6253,15 @@
5280	6253	goto error_failed_kzalloc;
5281	6254	}
5282	6255
5283		- hr_dev->pci_dev = handle->pdev;
5284		- hr_dev->dev = &handle->pdev->dev;
5285		- handle->priv = hr_dev;
5286		-
5287		- ret = hns_roce_hw_v2_get_cfg(hr_dev, handle);
5288		- if (ret) {
5289		- dev_err(hr_dev->dev, "Get Configuration failed!\n");
5290		- goto error_failed_get_cfg;
5291		- }
	6256	+ hns_roce_hw_v2_get_cfg(hr_dev, handle);
5292	6257
5293	6258	ret = hns_roce_init(hr_dev);
5294	6259	if (ret) {
5295	6260	dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
5296	6261	goto error_failed_get_cfg;
5297	6262	}
	6263	+
	6264	+ handle->priv = hr_dev;
5298	6265
5299	6266	return 0;
5300	6267
..	..	@@ -5307,54 +6274,125 @@
5307	6274	return ret;
5308	6275	}
5309	6276
5310		-static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
	6277	+static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
5311	6278	bool reset)
5312	6279	{
5313		- struct hns_roce_dev hr_dev = (struct hns_roce_dev )handle->priv;
	6280	+ struct hns_roce_dev *hr_dev = handle->priv;
5314	6281
5315	6282	if (!hr_dev)
5316	6283	return;
	6284	+
	6285	+ handle->priv = NULL;
	6286	+
	6287	+ hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;
	6288	+ hns_roce_handle_device_err(hr_dev);
5317	6289
5318	6290	hns_roce_exit(hr_dev);
5319	6291	kfree(hr_dev->priv);
5320	6292	ib_dealloc_device(&hr_dev->ib_dev);
5321	6293	}
5322	6294
5323		-static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
	6295	+static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
5324	6296	{
5325		- struct hns_roce_dev hr_dev = (struct hns_roce_dev )handle->priv;
5326		- struct ib_event event;
	6297	+ const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
	6298	+ const struct pci_device_id *id;
	6299	+ struct device *dev = &handle->pdev->dev;
	6300	+ int ret;
5327	6301
5328		- if (!hr_dev) {
5329		- dev_err(&handle->pdev->dev,
5330		- "Input parameter handle->priv is NULL!\n");
5331		- return -EINVAL;
	6302	+ handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
	6303	+
	6304	+ if (ops->ae_dev_resetting(handle) \|\| ops->get_hw_reset_stat(handle)) {
	6305	+ handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
	6306	+ goto reset_chk_err;
5332	6307	}
5333	6308
5334		- hr_dev->active = false;
5335		- hr_dev->is_reset = true;
	6309	+ id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
	6310	+ if (!id)
	6311	+ return 0;
5336	6312
5337		- event.event = IB_EVENT_DEVICE_FATAL;
5338		- event.device = &hr_dev->ib_dev;
5339		- event.element.port_num = 1;
5340		- ib_dispatch_event(&event);
	6313	+ ret = __hns_roce_hw_v2_init_instance(handle);
	6314	+ if (ret) {
	6315	+ handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
	6316	+ dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
	6317	+ if (ops->ae_dev_resetting(handle) \|\|
	6318	+ ops->get_hw_reset_stat(handle))
	6319	+ goto reset_chk_err;
	6320	+ else
	6321	+ return ret;
	6322	+ }
	6323	+
	6324	+ handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
	6325	+
	6326	+
	6327	+ return 0;
	6328	+
	6329	+reset_chk_err:
	6330	+ dev_err(dev, "Device is busy in resetting state.\n"
	6331	+ "please retry later.\n");
	6332	+
	6333	+ return -EBUSY;
	6334	+}
	6335	+
	6336	+static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
	6337	+ bool reset)
	6338	+{
	6339	+ if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
	6340	+ return;
	6341	+
	6342	+ handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
	6343	+
	6344	+ __hns_roce_hw_v2_uninit_instance(handle, reset);
	6345	+
	6346	+ handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
	6347	+}
	6348	+static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
	6349	+{
	6350	+ struct hns_roce_dev *hr_dev;
	6351	+
	6352	+ if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
	6353	+ set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
	6354	+ return 0;
	6355	+ }
	6356	+
	6357	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
	6358	+ clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
	6359	+
	6360	+ hr_dev = handle->priv;
	6361	+ if (!hr_dev)
	6362	+ return 0;
	6363	+
	6364	+ hr_dev->active = false;
	6365	+ hr_dev->dis_db = true;
	6366	+ hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
5341	6367
5342	6368	return 0;
5343	6369	}
5344	6370
5345	6371	static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
5346	6372	{
	6373	+ struct device *dev = &handle->pdev->dev;
5347	6374	int ret;
5348	6375
5349		- ret = hns_roce_hw_v2_init_instance(handle);
	6376	+ if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
	6377	+ &handle->rinfo.state)) {
	6378	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
	6379	+ return 0;
	6380	+ }
	6381	+
	6382	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
	6383	+
	6384	+ dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
	6385	+ ret = __hns_roce_hw_v2_init_instance(handle);
5350	6386	if (ret) {
5351	6387	/* when reset notify type is HNAE3_INIT_CLIENT In reset notify
5352	6388	* callback function, RoCE Engine reinitialize. If RoCE reinit
5353	6389	* failed, we should inform NIC driver.
5354	6390	*/
5355	6391	handle->priv = NULL;
5356		- dev_err(&handle->pdev->dev,
5357		- "In reset process RoCE reinit failed %d.\n", ret);
	6392	+ dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
	6393	+ } else {
	6394	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
	6395	+ dev_info(dev, "Reset done, RoCE client reinit finished.\n");
5358	6396	}
5359	6397
5360	6398	return ret;
..	..	@@ -5362,8 +6400,14 @@
5362	6400
5363	6401	static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
5364	6402	{
5365		- msleep(100);
5366		- hns_roce_hw_v2_uninit_instance(handle, false);
	6403	+ if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
	6404	+ return 0;
	6405	+
	6406	+ handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
	6407	+ dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
	6408	+ msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
	6409	+ __hns_roce_hw_v2_uninit_instance(handle, false);
	6410	+
5367	6411	return 0;
5368	6412	}
5369	6413