~hc/RK356X_SDK_RELEASE.git

..	..	@@ -37,16 +37,18 @@
37	37
38	38	#define DRV_NAME "hns_roce"
39	39
	40	+#define PCI_REVISION_ID_HIP08 0x21
	41	+#define PCI_REVISION_ID_HIP09 0x30
	42	+
40	43	#define HNS_ROCE_HW_VER1 ('h' << 24 \| 'i' << 16 \| '0' << 8 \| '6')
41	44
42		-#define MAC_ADDR_OCTET_NUM 6
43	45	#define HNS_ROCE_MAX_MSG_LEN 0x80000000
44		-
45		-#define HNS_ROCE_ALOGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b))
46	46
47	47	#define HNS_ROCE_IB_MIN_SQ_STRIDE 6
48	48
49	49	#define HNS_ROCE_BA_SIZE (32 * 4096)
	50	+
	51	+#define BA_BYTE_LEN 8
50	52
51	53	/* Hardware specification only for v1 engine */
52	54	#define HNS_ROCE_MIN_CQE_NUM 0x40
..	..	@@ -64,23 +66,29 @@
64	66
65	67	#define HNS_ROCE_MAX_IRQ_NUM 128
66	68
	69	+#define HNS_ROCE_SGE_IN_WQE 2
	70	+#define HNS_ROCE_SGE_SHIFT 4
	71	+
67	72	#define EQ_ENABLE 1
68	73	#define EQ_DISABLE 0
69	74
70	75	#define HNS_ROCE_CEQ 0
71	76	#define HNS_ROCE_AEQ 1
72	77
73		-#define HNS_ROCE_CEQ_ENTRY_SIZE 0x4
74		-#define HNS_ROCE_AEQ_ENTRY_SIZE 0x10
	78	+#define HNS_ROCE_CEQE_SIZE 0x4
	79	+#define HNS_ROCE_AEQE_SIZE 0x10
75	80
76		-/* 4G/4K = 1M */
77		-#define HNS_ROCE_SL_SHIFT 28
78		-#define HNS_ROCE_TCLASS_SHIFT 20
79		-#define HNS_ROCE_FLOW_LABEL_MASK 0xfffff
	81	+#define HNS_ROCE_V3_EQE_SIZE 0x40
	82	+
	83	+#define HNS_ROCE_V2_CQE_SIZE 32
	84	+#define HNS_ROCE_V3_CQE_SIZE 64
	85	+
	86	+#define HNS_ROCE_V2_QPC_SZ 256
	87	+#define HNS_ROCE_V3_QPC_SZ 512
80	88
81	89	#define HNS_ROCE_MAX_PORTS 6
82		-#define HNS_ROCE_MAX_GID_NUM 16
83	90	#define HNS_ROCE_GID_SIZE 16
	91	+#define HNS_ROCE_SGE_SIZE 16
84	92
85	93	#define HNS_ROCE_HOP_NUM_0 0xff
86	94
..	..	@@ -88,17 +96,15 @@
88	96	#define BITMAP_RR 1
89	97
90	98	#define MR_TYPE_MR 0x00
	99	+#define MR_TYPE_FRMR 0x01
91	100	#define MR_TYPE_DMA 0x03
	101	+
	102	+#define HNS_ROCE_FRMR_MAX_PA 512
92	103
93	104	#define PKEY_ID 0xffff
94	105	#define GUID_LEN 8
95	106	#define NODE_DESC_SIZE 64
96	107	#define DB_REG_OFFSET 0x1000
97		-
98		-#define SERV_TYPE_RC 0
99		-#define SERV_TYPE_RD 1
100		-#define SERV_TYPE_UC 2
101		-#define SERV_TYPE_UD 3
102	108
103	109	/* Configure to HW for PAGE_SIZE larger than 4KB */
104	110	#define PG_SHIFT_OFFSET (PAGE_SHIFT - 12)
..	..	@@ -108,13 +114,28 @@
108	114	#define PAGES_SHIFT_24 24
109	115	#define PAGES_SHIFT_32 32
110	116
	117	+#define HNS_ROCE_IDX_QUE_ENTRY_SZ 4
	118	+#define SRQ_DB_REG 0x230
	119	+
	120	+/* The chip implementation of the consumer index is calculated
	121	+ * according to twice the actual EQ depth
	122	+ */
	123	+#define EQ_DEPTH_COEFF 2
	124	+
111	125	enum {
112		- HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0,
113		- HNS_ROCE_SUPPORT_SQ_RECORD_DB = 1 << 1,
	126	+ SERV_TYPE_RC,
	127	+ SERV_TYPE_UC,
	128	+ SERV_TYPE_RD,
	129	+ SERV_TYPE_UD,
114	130	};
115	131
116	132	enum {
117		- HNS_ROCE_SUPPORT_CQ_RECORD_DB = 1 << 0,
	133	+ HNS_ROCE_QP_CAP_RQ_RECORD_DB = BIT(0),
	134	+ HNS_ROCE_QP_CAP_SQ_RECORD_DB = BIT(1),
	135	+};
	136	+
	137	+enum hns_roce_cq_flags {
	138	+ HNS_ROCE_CQ_FLAG_RECORD_DB = BIT(0),
118	139	};
119	140
120	141	enum hns_roce_qp_state {
..	..	@@ -187,21 +208,52 @@
187	208	HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE = 0x07,
188	209	};
189	210
	211	+#define HNS_ROCE_CAP_FLAGS_EX_SHIFT 12
	212	+
190	213	enum {
191	214	HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0),
192	215	HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1),
193	216	HNS_ROCE_CAP_FLAG_RQ_INLINE = BIT(2),
194	217	HNS_ROCE_CAP_FLAG_RECORD_DB = BIT(3),
195	218	HNS_ROCE_CAP_FLAG_SQ_RECORD_DB = BIT(4),
	219	+ HNS_ROCE_CAP_FLAG_SRQ = BIT(5),
	220	+ HNS_ROCE_CAP_FLAG_MW = BIT(7),
	221	+ HNS_ROCE_CAP_FLAG_FRMR = BIT(8),
	222	+ HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),
	223	+ HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10),
196	224	};
197	225
198		-enum hns_roce_mtt_type {
199		- MTT_TYPE_WQE,
200		- MTT_TYPE_CQE,
201		-};
	226	+#define HNS_ROCE_DB_TYPE_COUNT 2
	227	+#define HNS_ROCE_DB_UNIT_SIZE 4
202	228
203	229	enum {
204	230	HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4
	231	+};
	232	+
	233	+enum hns_roce_reset_stage {
	234	+ HNS_ROCE_STATE_NON_RST,
	235	+ HNS_ROCE_STATE_RST_BEF_DOWN,
	236	+ HNS_ROCE_STATE_RST_DOWN,
	237	+ HNS_ROCE_STATE_RST_UNINIT,
	238	+ HNS_ROCE_STATE_RST_INIT,
	239	+ HNS_ROCE_STATE_RST_INITED,
	240	+};
	241	+
	242	+enum hns_roce_instance_state {
	243	+ HNS_ROCE_STATE_NON_INIT,
	244	+ HNS_ROCE_STATE_INIT,
	245	+ HNS_ROCE_STATE_INITED,
	246	+ HNS_ROCE_STATE_UNINIT,
	247	+};
	248	+
	249	+enum {
	250	+ HNS_ROCE_RST_DIRECT_RETURN = 0,
	251	+};
	252	+
	253	+enum {
	254	+ CMD_RST_PRC_OTHERS,
	255	+ CMD_RST_PRC_SUCCESS,
	256	+ CMD_RST_PRC_EBUSY,
205	257	};
206	258
207	259	#define HNS_ROCE_CMD_SUCCESS 1
..	..	@@ -209,9 +261,12 @@
209	261	#define HNS_ROCE_PORT_DOWN 0
210	262	#define HNS_ROCE_PORT_UP 1
211	263
212		-#define HNS_ROCE_MTT_ENTRY_PER_SEG 8
	264	+/* The minimum page size is 4K for hardware */
	265	+#define HNS_HW_PAGE_SHIFT 12
	266	+#define HNS_HW_PAGE_SIZE (1 << HNS_HW_PAGE_SHIFT)
213	267
214		-#define PAGE_ADDR_SHIFT 12
	268	+/* The minimum page count for hardware access page directly. */
	269	+#define HNS_HW_DIRECT_PAGE_COUNT 2
215	270
216	271	struct hns_roce_uar {
217	272	u64 pfn;
..	..	@@ -219,19 +274,11 @@
219	274	unsigned long logic_idx;
220	275	};
221	276
222		-struct hns_roce_vma_data {
223		- struct list_head list;
224		- struct vm_area_struct *vma;
225		- struct mutex *vma_list_mutex;
226		-};
227		-
228	277	struct hns_roce_ucontext {
229	278	struct ib_ucontext ibucontext;
230	279	struct hns_roce_uar uar;
231	280	struct list_head page_list;
232	281	struct mutex page_mutex;
233		- struct list_head vma_list;
234		- struct mutex vma_list_mutex;
235	282	};
236	283
237	284	struct hns_roce_pd {
..	..	@@ -250,22 +297,6 @@
250	297	unsigned long *table;
251	298	};
252	299
253		-/* Order bitmap length -- bit num compute formula: 1 << (max_order - order) */
254		-/* Order = 0: bitmap is biggest, order = max bitmap is least (only a bit) */
255		-/* Every bit repesent to a partner free/used status in bitmap */
256		-/*
257		- * Initial, bits of other bitmap are all 0 except that a bit of max_order is 1
258		- * Bit = 1 represent to idle and available; bit = 0: not available
259		- */
260		-struct hns_roce_buddy {
261		- /* Members point to every order level bitmap */
262		- unsigned long **bits;
263		- /* Represent to avail bits of the order level bitmap */
264		- u32 *num_free;
265		- int max_order;
266		- spinlock_t lock;
267		-};
268		-
269	300	/* For Hardware Entry Memory */
270	301	struct hns_roce_hem_table {
271	302	/* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */
..	..	@@ -274,7 +305,7 @@
274	305	unsigned long num_hem;
275	306	/* HEM entry record obj total num */
276	307	unsigned long num_obj;
277		- /Single obj size /
	308	+ /* Single obj size */
278	309	unsigned long obj_size;
279	310	unsigned long table_chunk_size;
280	311	int lowmem;
..	..	@@ -286,11 +317,60 @@
286	317	dma_addr_t *bt_l0_dma_addr;
287	318	};
288	319
289		-struct hns_roce_mtt {
290		- unsigned long first_seg;
291		- int order;
292		- int page_shift;
293		- enum hns_roce_mtt_type mtt_type;
	320	+struct hns_roce_buf_region {
	321	+ int offset; /* page offset */
	322	+ u32 count; /* page count */
	323	+ int hopnum; /* addressing hop num */
	324	+};
	325	+
	326	+#define HNS_ROCE_MAX_BT_REGION 3
	327	+#define HNS_ROCE_MAX_BT_LEVEL 3
	328	+struct hns_roce_hem_list {
	329	+ struct list_head root_bt;
	330	+ /* link all bt dma mem by hop config */
	331	+ struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL];
	332	+ struct list_head btm_bt; /* link all bottom bt in @mid_bt */
	333	+ dma_addr_t root_ba; /* pointer to the root ba table */
	334	+};
	335	+
	336	+struct hns_roce_buf_attr {
	337	+ struct {
	338	+ size_t size; /* region size */
	339	+ int hopnum; /* multi-hop addressing hop num */
	340	+ } region[HNS_ROCE_MAX_BT_REGION];
	341	+ int region_count; /* valid region count */
	342	+ unsigned int page_shift; /* buffer page shift */
	343	+ bool fixed_page; /* decide page shift is fixed-size or maximum size */
	344	+ int user_access; /* umem access flag */
	345	+ bool mtt_only; /* only alloc buffer-required MTT memory */
	346	+};
	347	+
	348	+struct hns_roce_hem_cfg {
	349	+ dma_addr_t root_ba; /* root BA table's address */
	350	+ bool is_direct; /* addressing without BA table */
	351	+ unsigned int ba_pg_shift; /* BA table page shift */
	352	+ unsigned int buf_pg_shift; /* buffer page shift */
	353	+ unsigned int buf_pg_count; /* buffer page count */
	354	+ struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
	355	+ int region_count;
	356	+};
	357	+
	358	+/* memory translate region */
	359	+struct hns_roce_mtr {
	360	+ struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */
	361	+ struct ib_umem umem; / user space buffer */
	362	+ struct hns_roce_buf kmem; / kernel space buffer */
	363	+ struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */
	364	+};
	365	+
	366	+struct hns_roce_mw {
	367	+ struct ib_mw ibmw;
	368	+ u32 pdn;
	369	+ u32 rkey;
	370	+ int enabled; /* MW's active status */
	371	+ u32 pbl_hop_num;
	372	+ u32 pbl_ba_pg_sz;
	373	+ u32 pbl_buf_pg_sz;
294	374	};
295	375
296	376	/* Only support 4K page size for mr register */
..	..	@@ -298,57 +378,40 @@
298	378
299	379	struct hns_roce_mr {
300	380	struct ib_mr ibmr;
301		- struct ib_umem *umem;
302	381	u64 iova; /* MR's virtual orignal addr */
303	382	u64 size; /* Address range of MR */
304	383	u32 key; /* Key of MR */
305	384	u32 pd; /* PD num of MR */
306		- u32 access;/* Access permission of MR */
	385	+ u32 access; /* Access permission of MR */
307	386	int enabled; /* MR's active status */
308	387	int type; /* MR's register type */
309		- u64 pbl_buf;/ MR's PBL space */
310		- dma_addr_t pbl_dma_addr; /* MR's PBL space PA */
311		- u32 pbl_size;/* PA number in the PBL */
312		- u64 pbl_ba;/* page table address */
313		- u32 l0_chunk_last_num;/* L0 last number */
314		- u32 l1_chunk_last_num;/* L1 last number */
315		- u64 *pbl_bt_l2;/ PBL BT L2 */
316		- u64 *pbl_bt_l1;/ PBL BT L1 */
317		- u64 pbl_bt_l0;/ PBL BT L0 */
318		- dma_addr_t pbl_l2_dma_addr;/ PBL BT L2 dma addr */
319		- dma_addr_t pbl_l1_dma_addr;/ PBL BT L1 dma addr */
320		- dma_addr_t pbl_l0_dma_addr;/* PBL BT L0 dma addr */
321		- u32 pbl_ba_pg_sz;/* BT chunk page size */
322		- u32 pbl_buf_pg_sz;/* buf chunk page size */
323		- u32 pbl_hop_num;/* multi-hop number */
	388	+ u32 pbl_hop_num; /* multi-hop number */
	389	+ struct hns_roce_mtr pbl_mtr;
	390	+ u32 npages;
	391	+ dma_addr_t *page_list;
324	392	};
325	393
326	394	struct hns_roce_mr_table {
327	395	struct hns_roce_bitmap mtpt_bitmap;
328		- struct hns_roce_buddy mtt_buddy;
329		- struct hns_roce_hem_table mtt_table;
330	396	struct hns_roce_hem_table mtpt_table;
331		- struct hns_roce_buddy mtt_cqe_buddy;
332		- struct hns_roce_hem_table mtt_cqe_table;
333	397	};
334	398
335	399	struct hns_roce_wq {
336	400	u64 wrid; / Work request ID */
337	401	spinlock_t lock;
338		- int wqe_cnt; /* WQE num */
339		- u32 max_post;
	402	+ u32 wqe_cnt; /* WQE num */
340	403	int max_gs;
341	404	int offset;
342		- int wqe_shift;/* WQE size */
	405	+ int wqe_shift; /* WQE size */
343	406	u32 head;
344	407	u32 tail;
345	408	void __iomem *db_reg_l;
346	409	};
347	410
348	411	struct hns_roce_sge {
349		- int sge_cnt; /* SGE num */
	412	+ unsigned int sge_cnt; /* SGE num */
350	413	int offset;
351		- int sge_shift;/* SGE size */
	414	+ int sge_shift; /* SGE size */
352	415	};
353	416
354	417	struct hns_roce_buf_list {
..	..	@@ -359,16 +422,16 @@
359	422	struct hns_roce_buf {
360	423	struct hns_roce_buf_list direct;
361	424	struct hns_roce_buf_list *page_list;
362		- int nbufs;
363	425	u32 npages;
364		- int page_shift;
	426	+ u32 size;
	427	+ unsigned int page_shift;
365	428	};
366	429
367	430	struct hns_roce_db_pgdir {
368	431	struct list_head list;
369	432	DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE);
370		- DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / 2);
371		- unsigned long *bits[2];
	433	+ DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
	434	+ unsigned long *bits[HNS_ROCE_DB_TYPE_COUNT];
372	435	u32 *page;
373	436	dma_addr_t db_dma;
374	437	};
..	..	@@ -392,37 +455,55 @@
392	455	int order;
393	456	};
394	457
395		-struct hns_roce_cq_buf {
396		- struct hns_roce_buf hr_buf;
397		- struct hns_roce_mtt hr_mtt;
398		-};
399		-
400	458	struct hns_roce_cq {
401	459	struct ib_cq ib_cq;
402		- struct hns_roce_cq_buf hr_buf;
	460	+ struct hns_roce_mtr mtr;
403	461	struct hns_roce_db db;
404		- u8 db_en;
	462	+ u32 flags;
405	463	spinlock_t lock;
406		- struct ib_umem *umem;
407		- void (comp)(struct hns_roce_cq cq);
408		- void (event)(struct hns_roce_cq cq, enum hns_roce_event event_type);
409		-
410		- struct hns_roce_uar *uar;
411	464	u32 cq_depth;
412	465	u32 cons_index;
413	466	u32 *set_ci_db;
414	467	void __iomem *cq_db_l;
415	468	u16 *tptr_addr;
416	469	int arm_sn;
	470	+ int cqe_size;
417	471	unsigned long cqn;
418	472	u32 vector;
419	473	atomic_t refcount;
420	474	struct completion free;
	475	+ struct list_head sq_list; /* all qps on this send cq */
	476	+ struct list_head rq_list; /* all qps on this recv cq */
	477	+ int is_armed; /* cq is armed */
	478	+ struct list_head node; /* all armed cqs are on a list */
	479	+};
	480	+
	481	+struct hns_roce_idx_que {
	482	+ struct hns_roce_mtr mtr;
	483	+ int entry_shift;
	484	+ unsigned long *bitmap;
421	485	};
422	486
423	487	struct hns_roce_srq {
424	488	struct ib_srq ibsrq;
425		- int srqn;
	489	+ unsigned long srqn;
	490	+ u32 wqe_cnt;
	491	+ int max_gs;
	492	+ int wqe_shift;
	493	+ void __iomem *db_reg_l;
	494	+
	495	+ atomic_t refcount;
	496	+ struct completion free;
	497	+
	498	+ struct hns_roce_mtr buf_mtr;
	499	+
	500	+ u64 *wrid;
	501	+ struct hns_roce_idx_que idx_que;
	502	+ spinlock_t lock;
	503	+ int head;
	504	+ int tail;
	505	+ struct mutex mutex;
	506	+ void (event)(struct hns_roce_srq srq, enum hns_roce_event event);
426	507	};
427	508
428	509	struct hns_roce_uar_table {
..	..	@@ -431,16 +512,22 @@
431	512
432	513	struct hns_roce_qp_table {
433	514	struct hns_roce_bitmap bitmap;
434		- spinlock_t lock;
435	515	struct hns_roce_hem_table qp_table;
436	516	struct hns_roce_hem_table irrl_table;
437	517	struct hns_roce_hem_table trrl_table;
	518	+ struct hns_roce_hem_table sccc_table;
	519	+ struct mutex scc_mutex;
438	520	};
439	521
440	522	struct hns_roce_cq_table {
441	523	struct hns_roce_bitmap bitmap;
442		- spinlock_t lock;
443		- struct radix_tree_root tree;
	524	+ struct xarray array;
	525	+ struct hns_roce_hem_table table;
	526	+};
	527	+
	528	+struct hns_roce_srq_table {
	529	+ struct hns_roce_bitmap bitmap;
	530	+ struct xarray xa;
444	531	struct hns_roce_hem_table table;
445	532	};
446	533
..	..	@@ -449,14 +536,18 @@
449	536	};
450	537
451	538	struct hns_roce_av {
452		- __le32 port_pd;
453		- u8 gid_index;
454		- u8 stat_rate;
455		- u8 hop_limit;
456		- __le32 sl_tclass_flowlabel;
457		- u8 dgid[HNS_ROCE_GID_SIZE];
458		- u8 mac[6];
459		- __le16 vlan;
	539	+ u8 port;
	540	+ u8 gid_index;
	541	+ u8 stat_rate;
	542	+ u8 hop_limit;
	543	+ u32 flowlabel;
	544	+ u16 udp_sport;
	545	+ u8 sl;
	546	+ u8 tclass;
	547	+ u8 dgid[HNS_ROCE_GID_SIZE];
	548	+ u8 mac[ETH_ALEN];
	549	+ u16 vlan_id;
	550	+ u8 vlan_en;
460	551	};
461	552
462	553	struct hns_roce_ah {
..	..	@@ -497,7 +588,6 @@
497	588	* close device, switch into poll mode(non event mode)
498	589	*/
499	590	u8 use_events;
500		- u8 toggle;
501	591	};
502	592
503	593	struct hns_roce_cmd_mailbox {
..	..	@@ -522,23 +612,31 @@
522	612	u32 wqe_cnt;
523	613	};
524	614
	615	+enum {
	616	+ HNS_ROCE_FLUSH_FLAG = 0,
	617	+};
	618	+
	619	+struct hns_roce_work {
	620	+ struct hns_roce_dev *hr_dev;
	621	+ struct work_struct work;
	622	+ u32 qpn;
	623	+ u32 cqn;
	624	+ int event_type;
	625	+ int sub_type;
	626	+};
	627	+
525	628	struct hns_roce_qp {
526	629	struct ib_qp ibqp;
527		- struct hns_roce_buf hr_buf;
528	630	struct hns_roce_wq rq;
529	631	struct hns_roce_db rdb;
530	632	struct hns_roce_db sdb;
531		- u8 rdb_en;
532		- u8 sdb_en;
	633	+ unsigned long en_flags;
533	634	u32 doorbell_qpn;
534		- __le32 sq_signal_bits;
535		- u32 sq_next_wqe;
536		- int sq_max_wqes_per_wr;
537		- int sq_spare_wqes;
	635	+ enum ib_sig_type sq_signal_bits;
538	636	struct hns_roce_wq sq;
539	637
540		- struct ib_umem *umem;
541		- struct hns_roce_mtt mtt;
	638	+ struct hns_roce_mtr mtr;
	639	+
542	640	u32 buff_size;
543	641	struct mutex mutex;
544	642	u8 port;
..	..	@@ -559,12 +657,16 @@
559	657
560	658	struct hns_roce_sge sge;
561	659	u32 next_sge;
	660	+ enum ib_mtu path_mtu;
	661	+ u32 max_inline_data;
562	662
	663	+ /* 0: flush needed, 1: unneeded */
	664	+ unsigned long flush_flag;
	665	+ struct hns_roce_work flush_work;
563	666	struct hns_roce_rinl_buf rq_inl_buf;
564		-};
565		-
566		-struct hns_roce_sqp {
567		- struct hns_roce_qp hr_qp;
	667	+ struct list_head node; /* all qps are on a list */
	668	+ struct list_head rq_node; /* all recv qps are on a list */
	669	+ struct list_head sq_node; /* all send qps are on a list */
568	670	};
569	671
570	672	struct hns_roce_ib_iboe {
..	..	@@ -580,7 +682,8 @@
580	682	};
581	683
582	684	struct hns_roce_ceqe {
583		- u32 comp;
	685	+ __le32 comp;
	686	+ __le32 rsv[15];
584	687	};
585	688
586	689	struct hns_roce_aeqe {
..	..	@@ -591,6 +694,12 @@
591	694	u32 rsv0;
592	695	u32 rsv1;
593	696	} qp_event;
	697	+
	698	+ struct {
	699	+ __le32 srq;
	700	+ u32 rsv0;
	701	+ u32 rsv1;
	702	+ } srq_event;
594	703
595	704	struct {
596	705	__le32 cq;
..	..	@@ -611,13 +720,14 @@
611	720	u8 rsv0;
612	721	} __packed cmd;
613	722	} event;
	723	+ __le32 rsv[12];
614	724	};
615	725
616	726	struct hns_roce_eq {
617	727	struct hns_roce_dev *hr_dev;
618	728	void __iomem *doorbell;
619	729
620		- int type_flag;/* Aeq:1 ceq:0 */
	730	+ int type_flag; /* Aeq:1 ceq:0 */
621	731	int eqn;
622	732	u32 entries;
623	733	int log_entries;
..	..	@@ -629,23 +739,11 @@
629	739	int over_ignore;
630	740	int coalesce;
631	741	int arm_st;
632		- u64 eqe_ba;
633		- int eqe_ba_pg_sz;
634		- int eqe_buf_pg_sz;
635	742	int hop_num;
636		- u64 bt_l0; / Base address table for L0 */
637		- u64 *bt_l1; / Base address table for L1 */
638		- u64 **buf;
639		- dma_addr_t l0_dma;
640		- dma_addr_t *l1_dma;
641		- dma_addr_t *buf_dma;
642		- u32 l0_last_num; /* L0 last chunk num */
643		- u32 l1_last_num; /* L1 last chunk num */
644		- int eq_max_cnt;
	743	+ struct hns_roce_mtr mtr;
	744	+ u16 eq_max_cnt;
645	745	int eq_period;
646	746	int shift;
647		- dma_addr_t cur_eqe_ba;
648		- dma_addr_t nxt_eqe_ba;
649	747	int event_type;
650	748	int sub_type;
651	749	};
..	..	@@ -656,58 +754,77 @@
656	754	};
657	755
658	756	struct hns_roce_caps {
	757	+ u64 fw_ver;
659	758	u8 num_ports;
660	759	int gid_table_len[HNS_ROCE_MAX_PORTS];
661	760	int pkey_table_len[HNS_ROCE_MAX_PORTS];
662	761	int local_ca_ack_delay;
663	762	int num_uars;
664	763	u32 phy_num_uars;
665		- u32 max_sq_sg; /* 2 */
666		- u32 max_sq_inline; /* 32 */
667		- u32 max_rq_sg; /* 2 */
	764	+ u32 max_sq_sg;
	765	+ u32 max_sq_inline;
	766	+ u32 max_rq_sg;
668	767	u32 max_extend_sg;
669		- int num_qps; /* 256k */
	768	+ int num_qps;
670	769	int reserved_qps;
671		- u32 max_wqes; /* 16k */
672		- u32 max_sq_desc_sz; /* 64 */
673		- u32 max_rq_desc_sz; /* 64 */
	770	+ int num_qpc_timer;
	771	+ int num_cqc_timer;
	772	+ int num_srqs;
	773	+ u32 max_wqes;
	774	+ u32 max_srq_wrs;
	775	+ u32 max_srq_sges;
	776	+ u32 max_sq_desc_sz;
	777	+ u32 max_rq_desc_sz;
674	778	u32 max_srq_desc_sz;
675	779	int max_qp_init_rdma;
676	780	int max_qp_dest_rdma;
677	781	int num_cqs;
678		- int max_cqes;
679		- int min_cqes;
	782	+ u32 max_cqes;
	783	+ u32 min_cqes;
680	784	u32 min_wqes;
681	785	int reserved_cqs;
682		- int num_aeq_vectors; /* 1 */
	786	+ int reserved_srqs;
	787	+ int num_aeq_vectors;
683	788	int num_comp_vectors;
684	789	int num_other_vectors;
685	790	int num_mtpts;
686	791	u32 num_mtt_segs;
687	792	u32 num_cqe_segs;
	793	+ u32 num_srqwqe_segs;
	794	+ u32 num_idx_segs;
688	795	int reserved_mrws;
689	796	int reserved_uars;
690	797	int num_pds;
691	798	int reserved_pds;
692	799	u32 mtt_entry_sz;
693		- u32 cq_entry_sz;
	800	+ u32 cqe_sz;
694	801	u32 page_size_cap;
695	802	u32 reserved_lkey;
696	803	int mtpt_entry_sz;
697		- int qpc_entry_sz;
	804	+ int qpc_sz;
698	805	int irrl_entry_sz;
699	806	int trrl_entry_sz;
700	807	int cqc_entry_sz;
	808	+ int sccc_sz;
	809	+ int qpc_timer_entry_sz;
	810	+ int cqc_timer_entry_sz;
	811	+ int srqc_entry_sz;
	812	+ int idx_entry_sz;
701	813	u32 pbl_ba_pg_sz;
702	814	u32 pbl_buf_pg_sz;
703	815	u32 pbl_hop_num;
704	816	int aeqe_depth;
705	817	int ceqe_depth;
	818	+ u32 aeqe_size;
	819	+ u32 ceqe_size;
706	820	enum ib_mtu max_mtu;
707	821	u32 qpc_bt_num;
	822	+ u32 qpc_timer_bt_num;
708	823	u32 srqc_bt_num;
709	824	u32 cqc_bt_num;
	825	+ u32 cqc_timer_bt_num;
710	826	u32 mpt_bt_num;
	827	+ u32 sccc_bt_num;
711	828	u32 qpc_ba_pg_sz;
712	829	u32 qpc_buf_pg_sz;
713	830	u32 qpc_hop_num;
..	..	@@ -723,25 +840,52 @@
723	840	u32 mtt_ba_pg_sz;
724	841	u32 mtt_buf_pg_sz;
725	842	u32 mtt_hop_num;
726		- u32 cqe_ba_pg_sz;
	843	+ u32 wqe_sq_hop_num;
	844	+ u32 wqe_sge_hop_num;
	845	+ u32 wqe_rq_hop_num;
	846	+ u32 sccc_ba_pg_sz;
	847	+ u32 sccc_buf_pg_sz;
	848	+ u32 sccc_hop_num;
	849	+ u32 qpc_timer_ba_pg_sz;
	850	+ u32 qpc_timer_buf_pg_sz;
	851	+ u32 qpc_timer_hop_num;
	852	+ u32 cqc_timer_ba_pg_sz;
	853	+ u32 cqc_timer_buf_pg_sz;
	854	+ u32 cqc_timer_hop_num;
	855	+ u32 cqe_ba_pg_sz; /* page_size = 4K(2^cqe_ba_pg_sz) /
727	856	u32 cqe_buf_pg_sz;
728	857	u32 cqe_hop_num;
	858	+ u32 srqwqe_ba_pg_sz;
	859	+ u32 srqwqe_buf_pg_sz;
	860	+ u32 srqwqe_hop_num;
	861	+ u32 idx_ba_pg_sz;
	862	+ u32 idx_buf_pg_sz;
	863	+ u32 idx_hop_num;
729	864	u32 eqe_ba_pg_sz;
730	865	u32 eqe_buf_pg_sz;
731	866	u32 eqe_hop_num;
732	867	u32 sl_num;
733	868	u32 tsq_buf_pg_sz;
734	869	u32 tpq_buf_pg_sz;
735		- u32 chunk_sz; /* chunk size in non multihop mode*/
	870	+ u32 chunk_sz; /* chunk size in non multihop mode */
736	871	u64 flags;
	872	+ u16 default_ceq_max_cnt;
	873	+ u16 default_ceq_period;
	874	+ u16 default_aeq_max_cnt;
	875	+ u16 default_aeq_period;
	876	+ u16 default_aeq_arm_st;
	877	+ u16 default_ceq_arm_st;
737	878	};
738	879
739		-struct hns_roce_work {
740		- struct hns_roce_dev *hr_dev;
741		- struct work_struct work;
742		- u32 qpn;
743		- int event_type;
744		- int sub_type;
	880	+struct hns_roce_dfx_hw {
	881	+ int (query_cqc_info)(struct hns_roce_dev hr_dev, u32 cqn,
	882	+ int *buffer);
	883	+};
	884	+
	885	+enum hns_roce_device_state {
	886	+ HNS_ROCE_DEVICE_STATE_INITED,
	887	+ HNS_ROCE_DEVICE_STATE_RST_DOWN,
	888	+ HNS_ROCE_DEVICE_STATE_UNINIT,
745	889	};
746	890
747	891	struct hns_roce_hw {
..	..	@@ -755,20 +899,24 @@
755	899	u64 out_param, u32 in_modifier, u8 op_modifier, u16 op,
756	900	u16 token, int event);
757	901	int (chk_mbox)(struct hns_roce_dev hr_dev, unsigned long timeout);
	902	+ int (rst_prc_mbox)(struct hns_roce_dev hr_dev);
758	903	int (set_gid)(struct hns_roce_dev hr_dev, u8 port, int gid_index,
759	904	const union ib_gid gid, const struct ib_gid_attr attr);
760	905	int (set_mac)(struct hns_roce_dev hr_dev, u8 phy_port, u8 *addr);
761	906	void (set_mtu)(struct hns_roce_dev hr_dev, u8 phy_port,
762	907	enum ib_mtu mtu);
763		- int (write_mtpt)(void mb_buf, struct hns_roce_mr *mr,
764		- unsigned long mtpt_idx);
	908	+ int (write_mtpt)(struct hns_roce_dev hr_dev, void *mb_buf,
	909	+ struct hns_roce_mr *mr, unsigned long mtpt_idx);
765	910	int (rereg_write_mtpt)(struct hns_roce_dev hr_dev,
766	911	struct hns_roce_mr *mr, int flags, u32 pdn,
767	912	int mr_access_flags, u64 iova, u64 size,
768	913	void *mb_buf);
	914	+ int (frmr_write_mtpt)(struct hns_roce_dev hr_dev, void *mb_buf,
	915	+ struct hns_roce_mr *mr);
	916	+ int (mw_write_mtpt)(void mb_buf, struct hns_roce_mw *mw);
769	917	void (write_cqc)(struct hns_roce_dev hr_dev,
770	918	struct hns_roce_cq hr_cq, void mb_buf, u64 *mtts,
771		- dma_addr_t dma_handle, int nent, u32 vector);
	919	+ dma_addr_t dma_handle);
772	920	int (set_hem)(struct hns_roce_dev hr_dev,
773	921	struct hns_roce_hem_table *table, int obj, int step_idx);
774	922	int (clear_hem)(struct hns_roce_dev hr_dev,
..	..	@@ -779,18 +927,34 @@
779	927	int (modify_qp)(struct ib_qp ibqp, const struct ib_qp_attr *attr,
780	928	int attr_mask, enum ib_qp_state cur_state,
781	929	enum ib_qp_state new_state);
782		- int (destroy_qp)(struct ib_qp ibqp);
	930	+ int (destroy_qp)(struct ib_qp ibqp, struct ib_udata *udata);
	931	+ int (qp_flow_control_init)(struct hns_roce_dev hr_dev,
	932	+ struct hns_roce_qp *hr_qp);
783	933	int (post_send)(struct ib_qp ibqp, const struct ib_send_wr *wr,
784	934	const struct ib_send_wr **bad_wr);
785	935	int (post_recv)(struct ib_qp qp, const struct ib_recv_wr *recv_wr,
786	936	const struct ib_recv_wr **bad_recv_wr);
787	937	int (req_notify_cq)(struct ib_cq ibcq, enum ib_cq_notify_flags flags);
788	938	int (poll_cq)(struct ib_cq ibcq, int num_entries, struct ib_wc *wc);
789		- int (dereg_mr)(struct hns_roce_dev hr_dev, struct hns_roce_mr *mr);
790		- int (destroy_cq)(struct ib_cq ibcq);
	939	+ int (dereg_mr)(struct hns_roce_dev hr_dev, struct hns_roce_mr *mr,
	940	+ struct ib_udata *udata);
	941	+ int (destroy_cq)(struct ib_cq ibcq, struct ib_udata *udata);
791	942	int (modify_cq)(struct ib_cq cq, u16 cq_count, u16 cq_period);
792	943	int (init_eq)(struct hns_roce_dev hr_dev);
793	944	void (cleanup_eq)(struct hns_roce_dev hr_dev);
	945	+ void (write_srqc)(struct hns_roce_dev hr_dev,
	946	+ struct hns_roce_srq *srq, u32 pdn, u16 xrcd, u32 cqn,
	947	+ void mb_buf, u64 mtts_wqe, u64 *mtts_idx,
	948	+ dma_addr_t dma_handle_wqe,
	949	+ dma_addr_t dma_handle_idx);
	950	+ int (modify_srq)(struct ib_srq ibsrq, struct ib_srq_attr *srq_attr,
	951	+ enum ib_srq_attr_mask srq_attr_mask,
	952	+ struct ib_udata *udata);
	953	+ int (query_srq)(struct ib_srq ibsrq, struct ib_srq_attr *attr);
	954	+ int (post_srq_recv)(struct ib_srq ibsrq, const struct ib_recv_wr *wr,
	955	+ const struct ib_recv_wr **bad_wr);
	956	+ const struct ib_device_ops *hns_roce_dev_ops;
	957	+ const struct ib_device_ops *hns_roce_dev_srq_ops;
794	958	};
795	959
796	960	struct hns_roce_dev {
..	..	@@ -804,16 +968,21 @@
804	968	spinlock_t bt_cmd_lock;
805	969	bool active;
806	970	bool is_reset;
	971	+ bool dis_db;
	972	+ unsigned long reset_cnt;
807	973	struct hns_roce_ib_iboe iboe;
	974	+ enum hns_roce_device_state state;
	975	+ struct list_head qp_list; /* list of all qps on this dev */
	976	+ spinlock_t qp_list_lock; /* protect qp_list */
808	977
809	978	struct list_head pgdir_list;
810	979	struct mutex pgdir_mutex;
811	980	int irq[HNS_ROCE_MAX_IRQ_NUM];
812	981	u8 __iomem *reg_base;
813	982	struct hns_roce_caps caps;
814		- struct radix_tree_root qp_table_tree;
	983	+ struct xarray qp_table_xa;
815	984
816		- unsigned char dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM];
	985	+ unsigned char dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN];
817	986	u64 sys_image_guid;
818	987	u32 vendor_id;
819	988	u32 vendor_part_id;
..	..	@@ -825,18 +994,22 @@
825	994	struct hns_roce_uar_table uar_table;
826	995	struct hns_roce_mr_table mr_table;
827	996	struct hns_roce_cq_table cq_table;
	997	+ struct hns_roce_srq_table srq_table;
828	998	struct hns_roce_qp_table qp_table;
829	999	struct hns_roce_eq_table eq_table;
	1000	+ struct hns_roce_hem_table qpc_timer_table;
	1001	+ struct hns_roce_hem_table cqc_timer_table;
830	1002
831	1003	int cmd_mod;
832	1004	int loop_idc;
833	1005	u32 sdb_offset;
834	1006	u32 odb_offset;
835		- dma_addr_t tptr_dma_addr; /only for hw v1/
836		- u32 tptr_size; /only for hw v1/
	1007	+ dma_addr_t tptr_dma_addr; /* only for hw v1 */
	1008	+ u32 tptr_size; /* only for hw v1 */
837	1009	const struct hns_roce_hw *hw;
838	1010	void *priv;
839	1011	struct workqueue_struct *irq_workq;
	1012	+ const struct hns_roce_dfx_hw *dfx;
840	1013	};
841	1014
842	1015	static inline struct hns_roce_dev to_hr_dev(struct ib_device ib_dev)
..	..	@@ -865,6 +1038,11 @@
865	1038	return container_of(ibmr, struct hns_roce_mr, ibmr);
866	1039	}
867	1040
	1041	+static inline struct hns_roce_mw to_hr_mw(struct ib_mw ibmw)
	1042	+{
	1043	+ return container_of(ibmw, struct hns_roce_mw, ibmw);
	1044	+}
	1045	+
868	1046	static inline struct hns_roce_qp to_hr_qp(struct ib_qp ibqp)
869	1047	{
870	1048	return container_of(ibqp, struct hns_roce_qp, ibqp);
..	..	@@ -880,11 +1058,6 @@
880	1058	return container_of(ibsrq, struct hns_roce_srq, ibsrq);
881	1059	}
882	1060
883		-static inline struct hns_roce_sqp hr_to_hr_sqp(struct hns_roce_qp hr_qp)
884		-{
885		- return container_of(hr_qp, struct hns_roce_sqp, hr_qp);
886		-}
887		-
888	1061	static inline void hns_roce_write64_k(__le32 val[2], void __iomem *dest)
889	1062	{
890	1063	__raw_writeq((u64 ) val, dest);
..	..	@@ -893,19 +1066,78 @@
893	1066	static inline struct hns_roce_qp
894	1067	__hns_roce_qp_lookup(struct hns_roce_dev hr_dev, u32 qpn)
895	1068	{
896		- return radix_tree_lookup(&hr_dev->qp_table_tree,
897		- qpn & (hr_dev->caps.num_qps - 1));
	1069	+ return xa_load(&hr_dev->qp_table_xa, qpn & (hr_dev->caps.num_qps - 1));
	1070	+}
	1071	+
	1072	+static inline bool hns_roce_buf_is_direct(struct hns_roce_buf *buf)
	1073	+{
	1074	+ if (buf->page_list)
	1075	+ return false;
	1076	+
	1077	+ return true;
898	1078	}
899	1079
900	1080	static inline void hns_roce_buf_offset(struct hns_roce_buf buf, int offset)
901	1081	{
902		- u32 page_size = 1 << buf->page_shift;
	1082	+ if (hns_roce_buf_is_direct(buf))
	1083	+ return (char *)(buf->direct.buf) + (offset & (buf->size - 1));
903	1084
904		- if (buf->nbufs == 1)
905		- return (char *)(buf->direct.buf) + offset;
	1085	+ return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
	1086	+ (offset & ((1 << buf->page_shift) - 1));
	1087	+}
	1088	+
	1089	+static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, int idx)
	1090	+{
	1091	+ if (hns_roce_buf_is_direct(buf))
	1092	+ return buf->direct.map + ((dma_addr_t)idx << buf->page_shift);
906	1093	else
907		- return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
908		- (offset & (page_size - 1));
	1094	+ return buf->page_list[idx].map;
	1095	+}
	1096	+
	1097	+#define hr_hw_page_align(x) ALIGN(x, 1 << HNS_HW_PAGE_SHIFT)
	1098	+
	1099	+static inline u64 to_hr_hw_page_addr(u64 addr)
	1100	+{
	1101	+ return addr >> HNS_HW_PAGE_SHIFT;
	1102	+}
	1103	+
	1104	+static inline u32 to_hr_hw_page_shift(u32 page_shift)
	1105	+{
	1106	+ return page_shift - HNS_HW_PAGE_SHIFT;
	1107	+}
	1108	+
	1109	+static inline u32 to_hr_hem_hopnum(u32 hopnum, u32 count)
	1110	+{
	1111	+ if (count > 0)
	1112	+ return hopnum == HNS_ROCE_HOP_NUM_0 ? 0 : hopnum;
	1113	+
	1114	+ return 0;
	1115	+}
	1116	+
	1117	+static inline u32 to_hr_hem_entries_size(u32 count, u32 buf_shift)
	1118	+{
	1119	+ return hr_hw_page_align(count << buf_shift);
	1120	+}
	1121	+
	1122	+static inline u32 to_hr_hem_entries_count(u32 count, u32 buf_shift)
	1123	+{
	1124	+ return hr_hw_page_align(count << buf_shift) >> buf_shift;
	1125	+}
	1126	+
	1127	+static inline u32 to_hr_hem_entries_shift(u32 count, u32 buf_shift)
	1128	+{
	1129	+ if (!count)
	1130	+ return 0;
	1131	+
	1132	+ return ilog2(to_hr_hem_entries_count(count, buf_shift));
	1133	+}
	1134	+
	1135	+#define DSCP_SHIFT 2
	1136	+
	1137	+static inline u8 get_tclass(const struct ib_global_route *grh)
	1138	+{
	1139	+ return grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP ?
	1140	+ grh->traffic_class >> DSCP_SHIFT : grh->traffic_class;
909	1141	}
910	1142
911	1143	int hns_roce_init_uar_table(struct hns_roce_dev *dev);
..	..	@@ -920,24 +1152,31 @@
920	1152	int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev);
921	1153	void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);
922	1154
923		-int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
924		- struct hns_roce_mtt *mtt);
925		-void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev,
926		- struct hns_roce_mtt *mtt);
927		-int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
928		- struct hns_roce_mtt mtt, struct hns_roce_buf buf);
	1155	+/* hns roce hw need current block and next block addr from mtt */
	1156	+#define MTT_MIN_COUNT 2
	1157	+int hns_roce_mtr_find(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,
	1158	+ int offset, u64 mtt_buf, int mtt_max, u64 base_addr);
	1159	+int hns_roce_mtr_create(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,
	1160	+ struct hns_roce_buf_attr *buf_attr,
	1161	+ unsigned int page_shift, struct ib_udata *udata,
	1162	+ unsigned long user_addr);
	1163	+void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
	1164	+ struct hns_roce_mtr *mtr);
	1165	+int hns_roce_mtr_map(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,
	1166	+ dma_addr_t *pages, int page_cnt);
929	1167
930	1168	int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);
931	1169	int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
932		-int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev);
933	1170	int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);
934	1171	int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev);
	1172	+int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev);
935	1173
936	1174	void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev);
937	1175	void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev);
938	1176	void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev);
939	1177	void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);
940	1178	void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);
	1179	+void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev);
941	1180
942	1181	int hns_roce_bitmap_alloc(struct hns_roce_bitmap bitmap, unsigned long obj);
943	1182	void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,
..	..	@@ -952,16 +1191,16 @@
952	1191	unsigned long obj, int cnt,
953	1192	int rr);
954	1193
955		-struct ib_ah hns_roce_create_ah(struct ib_pd pd,
956		- struct rdma_ah_attr *ah_attr,
957		- struct ib_udata *udata);
	1194	+int hns_roce_create_ah(struct ib_ah ah, struct rdma_ah_init_attr init_attr,
	1195	+ struct ib_udata *udata);
958	1196	int hns_roce_query_ah(struct ib_ah ibah, struct rdma_ah_attr ah_attr);
959		-int hns_roce_destroy_ah(struct ib_ah *ah);
	1197	+static inline int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags)
	1198	+{
	1199	+ return 0;
	1200	+}
960	1201
961		-struct ib_pd hns_roce_alloc_pd(struct ib_device ib_dev,
962		- struct ib_ucontext *context,
963		- struct ib_udata *udata);
964		-int hns_roce_dealloc_pd(struct ib_pd *pd);
	1202	+int hns_roce_alloc_pd(struct ib_pd pd, struct ib_udata udata);
	1203	+int hns_roce_dealloc_pd(struct ib_pd pd, struct ib_udata udata);
965	1204
966	1205	struct ib_mr hns_roce_get_dma_mr(struct ib_pd pd, int acc);
967	1206	struct ib_mr hns_roce_reg_user_mr(struct ib_pd pd, u64 start, u64 length,
..	..	@@ -970,28 +1209,46 @@
970	1209	int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length,
971	1210	u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
972	1211	struct ib_udata *udata);
973		-int hns_roce_dereg_mr(struct ib_mr *ibmr);
974		-int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
975		- struct hns_roce_cmd_mailbox *mailbox,
976		- unsigned long mpt_index);
	1212	+struct ib_mr hns_roce_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
	1213	+ u32 max_num_sg);
	1214	+int hns_roce_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg, int sg_nents,
	1215	+ unsigned int *sg_offset);
	1216	+int hns_roce_dereg_mr(struct ib_mr ibmr, struct ib_udata udata);
	1217	+int hns_roce_hw_destroy_mpt(struct hns_roce_dev *hr_dev,
	1218	+ struct hns_roce_cmd_mailbox *mailbox,
	1219	+ unsigned long mpt_index);
977	1220	unsigned long key_to_hw_index(u32 key);
978	1221
979		-void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
980		- struct hns_roce_buf *buf);
	1222	+int hns_roce_alloc_mw(struct ib_mw mw, struct ib_udata udata);
	1223	+int hns_roce_dealloc_mw(struct ib_mw *ibmw);
	1224	+
	1225	+void hns_roce_buf_free(struct hns_roce_dev hr_dev, struct hns_roce_buf buf);
981	1226	int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
982	1227	struct hns_roce_buf *buf, u32 page_shift);
983	1228
984		-int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
985		- struct hns_roce_mtt mtt, struct ib_umem umem);
	1229	+int hns_roce_get_kmem_bufs(struct hns_roce_dev hr_dev, dma_addr_t bufs,
	1230	+ int buf_cnt, int start, struct hns_roce_buf *buf);
	1231	+int hns_roce_get_umem_bufs(struct hns_roce_dev hr_dev, dma_addr_t bufs,
	1232	+ int buf_cnt, int start, struct ib_umem *umem,
	1233	+ unsigned int page_shift);
	1234	+
	1235	+int hns_roce_create_srq(struct ib_srq *srq,
	1236	+ struct ib_srq_init_attr *srq_init_attr,
	1237	+ struct ib_udata *udata);
	1238	+int hns_roce_modify_srq(struct ib_srq ibsrq, struct ib_srq_attr srq_attr,
	1239	+ enum ib_srq_attr_mask srq_attr_mask,
	1240	+ struct ib_udata *udata);
	1241	+int hns_roce_destroy_srq(struct ib_srq ibsrq, struct ib_udata udata);
986	1242
987	1243	struct ib_qp hns_roce_create_qp(struct ib_pd ib_pd,
988	1244	struct ib_qp_init_attr *init_attr,
989	1245	struct ib_udata *udata);
990	1246	int hns_roce_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
991	1247	int attr_mask, struct ib_udata *udata);
992		-void get_recv_wqe(struct hns_roce_qp hr_qp, int n);
993		-void get_send_wqe(struct hns_roce_qp hr_qp, int n);
994		-void get_send_extend_sge(struct hns_roce_qp hr_qp, int n);
	1248	+void init_flush_work(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp);
	1249	+void hns_roce_get_recv_wqe(struct hns_roce_qp hr_qp, int n);
	1250	+void hns_roce_get_send_wqe(struct hns_roce_qp hr_qp, int n);
	1251	+void hns_roce_get_extend_sge(struct hns_roce_qp hr_qp, int n);
995	1252	bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
996	1253	struct ib_cq *ib_cq);
997	1254	enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state);
..	..	@@ -1000,21 +1257,17 @@
1000	1257	void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1001	1258	struct hns_roce_cq *recv_cq);
1002	1259	void hns_roce_qp_remove(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp);
1003		-void hns_roce_qp_free(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp);
1004		-void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
1005		- int cnt);
	1260	+void hns_roce_qp_destroy(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp,
	1261	+ struct ib_udata *udata);
1006	1262	__be32 send_ieth(const struct ib_send_wr *wr);
1007	1263	int to_hr_qp_type(int qp_type);
1008	1264
1009		-struct ib_cq hns_roce_ib_create_cq(struct ib_device ib_dev,
1010		- const struct ib_cq_init_attr *attr,
1011		- struct ib_ucontext *context,
1012		- struct ib_udata *udata);
	1265	+int hns_roce_create_cq(struct ib_cq ib_cq, const struct ib_cq_init_attr attr,
	1266	+ struct ib_udata *udata);
1013	1267
1014		-int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq);
1015		-void hns_roce_free_cq(struct hns_roce_dev hr_dev, struct hns_roce_cq hr_cq);
1016		-
1017		-int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
	1268	+int hns_roce_destroy_cq(struct ib_cq ib_cq, struct ib_udata udata);
	1269	+int hns_roce_db_map_user(struct hns_roce_ucontext *context,
	1270	+ struct ib_udata *udata, unsigned long virt,
1018	1271	struct hns_roce_db *db);
1019	1272	void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
1020	1273	struct hns_roce_db *db);
..	..	@@ -1025,8 +1278,12 @@
1025	1278	void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);
1026	1279	void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
1027	1280	void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);
	1281	+void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type);
1028	1282	int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
	1283	+void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
1029	1284	int hns_roce_init(struct hns_roce_dev *hr_dev);
1030	1285	void hns_roce_exit(struct hns_roce_dev *hr_dev);
1031	1286
	1287	+int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
	1288	+ struct ib_cq *ib_cq);
1032	1289	#endif /* _HNS_ROCE_DEVICE_H */