~hc/RK356X_SDK_RELEASE.git

..	..	@@ -37,16 +37,16 @@
37	37
38	38	#define DRV_NAME "hns_roce"
39	39
	40	+#define PCI_REVISION_ID_HIP08 0x21
	41	+#define PCI_REVISION_ID_HIP09 0x30
	42	+
40	43	#define HNS_ROCE_HW_VER1 ('h' << 24 \| 'i' << 16 \| '0' << 8 \| '6')
41	44
42		-#define MAC_ADDR_OCTET_NUM 6
43	45	#define HNS_ROCE_MAX_MSG_LEN 0x80000000
44		-
45		-#define HNS_ROCE_ALOGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b))
46	46
47	47	#define HNS_ROCE_IB_MIN_SQ_STRIDE 6
48	48
49		-#define HNS_ROCE_BA_SIZE (32 * 4096)
	49	+#define BA_BYTE_LEN 8
50	50
51	51	/* Hardware specification only for v1 engine */
52	52	#define HNS_ROCE_MIN_CQE_NUM 0x40
..	..	@@ -64,23 +64,29 @@
64	64
65	65	#define HNS_ROCE_MAX_IRQ_NUM 128
66	66
	67	+#define HNS_ROCE_SGE_IN_WQE 2
	68	+#define HNS_ROCE_SGE_SHIFT 4
	69	+
67	70	#define EQ_ENABLE 1
68	71	#define EQ_DISABLE 0
69	72
70	73	#define HNS_ROCE_CEQ 0
71	74	#define HNS_ROCE_AEQ 1
72	75
73		-#define HNS_ROCE_CEQ_ENTRY_SIZE 0x4
74		-#define HNS_ROCE_AEQ_ENTRY_SIZE 0x10
	76	+#define HNS_ROCE_CEQE_SIZE 0x4
	77	+#define HNS_ROCE_AEQE_SIZE 0x10
75	78
76		-/* 4G/4K = 1M */
77		-#define HNS_ROCE_SL_SHIFT 28
78		-#define HNS_ROCE_TCLASS_SHIFT 20
79		-#define HNS_ROCE_FLOW_LABEL_MASK 0xfffff
	79	+#define HNS_ROCE_V3_EQE_SIZE 0x40
	80	+
	81	+#define HNS_ROCE_V2_CQE_SIZE 32
	82	+#define HNS_ROCE_V3_CQE_SIZE 64
	83	+
	84	+#define HNS_ROCE_V2_QPC_SZ 256
	85	+#define HNS_ROCE_V3_QPC_SZ 512
80	86
81	87	#define HNS_ROCE_MAX_PORTS 6
82		-#define HNS_ROCE_MAX_GID_NUM 16
83	88	#define HNS_ROCE_GID_SIZE 16
	89	+#define HNS_ROCE_SGE_SIZE 16
84	90
85	91	#define HNS_ROCE_HOP_NUM_0 0xff
86	92
..	..	@@ -88,17 +94,15 @@
88	94	#define BITMAP_RR 1
89	95
90	96	#define MR_TYPE_MR 0x00
	97	+#define MR_TYPE_FRMR 0x01
91	98	#define MR_TYPE_DMA 0x03
	99	+
	100	+#define HNS_ROCE_FRMR_MAX_PA 512
92	101
93	102	#define PKEY_ID 0xffff
94	103	#define GUID_LEN 8
95	104	#define NODE_DESC_SIZE 64
96	105	#define DB_REG_OFFSET 0x1000
97		-
98		-#define SERV_TYPE_RC 0
99		-#define SERV_TYPE_RD 1
100		-#define SERV_TYPE_UC 2
101		-#define SERV_TYPE_UD 3
102	106
103	107	/* Configure to HW for PAGE_SIZE larger than 4KB */
104	108	#define PG_SHIFT_OFFSET (PAGE_SHIFT - 12)
..	..	@@ -108,13 +112,28 @@
108	112	#define PAGES_SHIFT_24 24
109	113	#define PAGES_SHIFT_32 32
110	114
	115	+#define HNS_ROCE_IDX_QUE_ENTRY_SZ 4
	116	+#define SRQ_DB_REG 0x230
	117	+
	118	+/* The chip implementation of the consumer index is calculated
	119	+ * according to twice the actual EQ depth
	120	+ */
	121	+#define EQ_DEPTH_COEFF 2
	122	+
111	123	enum {
112		- HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0,
113		- HNS_ROCE_SUPPORT_SQ_RECORD_DB = 1 << 1,
	124	+ SERV_TYPE_RC,
	125	+ SERV_TYPE_UC,
	126	+ SERV_TYPE_RD,
	127	+ SERV_TYPE_UD,
114	128	};
115	129
116	130	enum {
117		- HNS_ROCE_SUPPORT_CQ_RECORD_DB = 1 << 0,
	131	+ HNS_ROCE_QP_CAP_RQ_RECORD_DB = BIT(0),
	132	+ HNS_ROCE_QP_CAP_SQ_RECORD_DB = BIT(1),
	133	+};
	134	+
	135	+enum hns_roce_cq_flags {
	136	+ HNS_ROCE_CQ_FLAG_RECORD_DB = BIT(0),
118	137	};
119	138
120	139	enum hns_roce_qp_state {
..	..	@@ -187,21 +206,52 @@
187	206	HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE = 0x07,
188	207	};
189	208
	209	+#define HNS_ROCE_CAP_FLAGS_EX_SHIFT 12
	210	+
190	211	enum {
191	212	HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0),
192	213	HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1),
193	214	HNS_ROCE_CAP_FLAG_RQ_INLINE = BIT(2),
194	215	HNS_ROCE_CAP_FLAG_RECORD_DB = BIT(3),
195	216	HNS_ROCE_CAP_FLAG_SQ_RECORD_DB = BIT(4),
	217	+ HNS_ROCE_CAP_FLAG_SRQ = BIT(5),
	218	+ HNS_ROCE_CAP_FLAG_MW = BIT(7),
	219	+ HNS_ROCE_CAP_FLAG_FRMR = BIT(8),
	220	+ HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),
	221	+ HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10),
196	222	};
197	223
198		-enum hns_roce_mtt_type {
199		- MTT_TYPE_WQE,
200		- MTT_TYPE_CQE,
201		-};
	224	+#define HNS_ROCE_DB_TYPE_COUNT 2
	225	+#define HNS_ROCE_DB_UNIT_SIZE 4
202	226
203	227	enum {
204	228	HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4
	229	+};
	230	+
	231	+enum hns_roce_reset_stage {
	232	+ HNS_ROCE_STATE_NON_RST,
	233	+ HNS_ROCE_STATE_RST_BEF_DOWN,
	234	+ HNS_ROCE_STATE_RST_DOWN,
	235	+ HNS_ROCE_STATE_RST_UNINIT,
	236	+ HNS_ROCE_STATE_RST_INIT,
	237	+ HNS_ROCE_STATE_RST_INITED,
	238	+};
	239	+
	240	+enum hns_roce_instance_state {
	241	+ HNS_ROCE_STATE_NON_INIT,
	242	+ HNS_ROCE_STATE_INIT,
	243	+ HNS_ROCE_STATE_INITED,
	244	+ HNS_ROCE_STATE_UNINIT,
	245	+};
	246	+
	247	+enum {
	248	+ HNS_ROCE_RST_DIRECT_RETURN = 0,
	249	+};
	250	+
	251	+enum {
	252	+ CMD_RST_PRC_OTHERS,
	253	+ CMD_RST_PRC_SUCCESS,
	254	+ CMD_RST_PRC_EBUSY,
205	255	};
206	256
207	257	#define HNS_ROCE_CMD_SUCCESS 1
..	..	@@ -209,9 +259,12 @@
209	259	#define HNS_ROCE_PORT_DOWN 0
210	260	#define HNS_ROCE_PORT_UP 1
211	261
212		-#define HNS_ROCE_MTT_ENTRY_PER_SEG 8
	262	+/* The minimum page size is 4K for hardware */
	263	+#define HNS_HW_PAGE_SHIFT 12
	264	+#define HNS_HW_PAGE_SIZE (1 << HNS_HW_PAGE_SHIFT)
213	265
214		-#define PAGE_ADDR_SHIFT 12
	266	+/* The minimum page count for hardware access page directly. */
	267	+#define HNS_HW_DIRECT_PAGE_COUNT 2
215	268
216	269	struct hns_roce_uar {
217	270	u64 pfn;
..	..	@@ -219,19 +272,11 @@
219	272	unsigned long logic_idx;
220	273	};
221	274
222		-struct hns_roce_vma_data {
223		- struct list_head list;
224		- struct vm_area_struct *vma;
225		- struct mutex *vma_list_mutex;
226		-};
227		-
228	275	struct hns_roce_ucontext {
229	276	struct ib_ucontext ibucontext;
230	277	struct hns_roce_uar uar;
231	278	struct list_head page_list;
232	279	struct mutex page_mutex;
233		- struct list_head vma_list;
234		- struct mutex vma_list_mutex;
235	280	};
236	281
237	282	struct hns_roce_pd {
..	..	@@ -250,22 +295,6 @@
250	295	unsigned long *table;
251	296	};
252	297
253		-/* Order bitmap length -- bit num compute formula: 1 << (max_order - order) */
254		-/* Order = 0: bitmap is biggest, order = max bitmap is least (only a bit) */
255		-/* Every bit repesent to a partner free/used status in bitmap */
256		-/*
257		- * Initial, bits of other bitmap are all 0 except that a bit of max_order is 1
258		- * Bit = 1 represent to idle and available; bit = 0: not available
259		- */
260		-struct hns_roce_buddy {
261		- /* Members point to every order level bitmap */
262		- unsigned long **bits;
263		- /* Represent to avail bits of the order level bitmap */
264		- u32 *num_free;
265		- int max_order;
266		- spinlock_t lock;
267		-};
268		-
269	298	/* For Hardware Entry Memory */
270	299	struct hns_roce_hem_table {
271	300	/* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */
..	..	@@ -274,7 +303,7 @@
274	303	unsigned long num_hem;
275	304	/* HEM entry record obj total num */
276	305	unsigned long num_obj;
277		- /Single obj size /
	306	+ /* Single obj size */
278	307	unsigned long obj_size;
279	308	unsigned long table_chunk_size;
280	309	int lowmem;
..	..	@@ -286,11 +315,60 @@
286	315	dma_addr_t *bt_l0_dma_addr;
287	316	};
288	317
289		-struct hns_roce_mtt {
290		- unsigned long first_seg;
291		- int order;
292		- int page_shift;
293		- enum hns_roce_mtt_type mtt_type;
	318	+struct hns_roce_buf_region {
	319	+ int offset; /* page offset */
	320	+ u32 count; /* page count */
	321	+ int hopnum; /* addressing hop num */
	322	+};
	323	+
	324	+#define HNS_ROCE_MAX_BT_REGION 3
	325	+#define HNS_ROCE_MAX_BT_LEVEL 3
	326	+struct hns_roce_hem_list {
	327	+ struct list_head root_bt;
	328	+ /* link all bt dma mem by hop config */
	329	+ struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL];
	330	+ struct list_head btm_bt; /* link all bottom bt in @mid_bt */
	331	+ dma_addr_t root_ba; /* pointer to the root ba table */
	332	+};
	333	+
	334	+struct hns_roce_buf_attr {
	335	+ struct {
	336	+ size_t size; /* region size */
	337	+ int hopnum; /* multi-hop addressing hop num */
	338	+ } region[HNS_ROCE_MAX_BT_REGION];
	339	+ int region_count; /* valid region count */
	340	+ unsigned int page_shift; /* buffer page shift */
	341	+ bool fixed_page; /* decide page shift is fixed-size or maximum size */
	342	+ int user_access; /* umem access flag */
	343	+ bool mtt_only; /* only alloc buffer-required MTT memory */
	344	+};
	345	+
	346	+struct hns_roce_hem_cfg {
	347	+ dma_addr_t root_ba; /* root BA table's address */
	348	+ bool is_direct; /* addressing without BA table */
	349	+ unsigned int ba_pg_shift; /* BA table page shift */
	350	+ unsigned int buf_pg_shift; /* buffer page shift */
	351	+ unsigned int buf_pg_count; /* buffer page count */
	352	+ struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
	353	+ int region_count;
	354	+};
	355	+
	356	+/* memory translate region */
	357	+struct hns_roce_mtr {
	358	+ struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */
	359	+ struct ib_umem umem; / user space buffer */
	360	+ struct hns_roce_buf kmem; / kernel space buffer */
	361	+ struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */
	362	+};
	363	+
	364	+struct hns_roce_mw {
	365	+ struct ib_mw ibmw;
	366	+ u32 pdn;
	367	+ u32 rkey;
	368	+ int enabled; /* MW's active status */
	369	+ u32 pbl_hop_num;
	370	+ u32 pbl_ba_pg_sz;
	371	+ u32 pbl_buf_pg_sz;
294	372	};
295	373
296	374	/* Only support 4K page size for mr register */
..	..	@@ -298,57 +376,40 @@
298	376
299	377	struct hns_roce_mr {
300	378	struct ib_mr ibmr;
301		- struct ib_umem *umem;
302	379	u64 iova; /* MR's virtual orignal addr */
303	380	u64 size; /* Address range of MR */
304	381	u32 key; /* Key of MR */
305	382	u32 pd; /* PD num of MR */
306		- u32 access;/* Access permission of MR */
	383	+ u32 access; /* Access permission of MR */
307	384	int enabled; /* MR's active status */
308	385	int type; /* MR's register type */
309		- u64 pbl_buf;/ MR's PBL space */
310		- dma_addr_t pbl_dma_addr; /* MR's PBL space PA */
311		- u32 pbl_size;/* PA number in the PBL */
312		- u64 pbl_ba;/* page table address */
313		- u32 l0_chunk_last_num;/* L0 last number */
314		- u32 l1_chunk_last_num;/* L1 last number */
315		- u64 *pbl_bt_l2;/ PBL BT L2 */
316		- u64 *pbl_bt_l1;/ PBL BT L1 */
317		- u64 pbl_bt_l0;/ PBL BT L0 */
318		- dma_addr_t pbl_l2_dma_addr;/ PBL BT L2 dma addr */
319		- dma_addr_t pbl_l1_dma_addr;/ PBL BT L1 dma addr */
320		- dma_addr_t pbl_l0_dma_addr;/* PBL BT L0 dma addr */
321		- u32 pbl_ba_pg_sz;/* BT chunk page size */
322		- u32 pbl_buf_pg_sz;/* buf chunk page size */
323		- u32 pbl_hop_num;/* multi-hop number */
	386	+ u32 pbl_hop_num; /* multi-hop number */
	387	+ struct hns_roce_mtr pbl_mtr;
	388	+ u32 npages;
	389	+ dma_addr_t *page_list;
324	390	};
325	391
326	392	struct hns_roce_mr_table {
327	393	struct hns_roce_bitmap mtpt_bitmap;
328		- struct hns_roce_buddy mtt_buddy;
329		- struct hns_roce_hem_table mtt_table;
330	394	struct hns_roce_hem_table mtpt_table;
331		- struct hns_roce_buddy mtt_cqe_buddy;
332		- struct hns_roce_hem_table mtt_cqe_table;
333	395	};
334	396
335	397	struct hns_roce_wq {
336	398	u64 wrid; / Work request ID */
337	399	spinlock_t lock;
338		- int wqe_cnt; /* WQE num */
339		- u32 max_post;
	400	+ u32 wqe_cnt; /* WQE num */
340	401	int max_gs;
341	402	int offset;
342		- int wqe_shift;/* WQE size */
	403	+ int wqe_shift; /* WQE size */
343	404	u32 head;
344	405	u32 tail;
345	406	void __iomem *db_reg_l;
346	407	};
347	408
348	409	struct hns_roce_sge {
349		- int sge_cnt; /* SGE num */
	410	+ unsigned int sge_cnt; /* SGE num */
350	411	int offset;
351		- int sge_shift;/* SGE size */
	412	+ int sge_shift; /* SGE size */
352	413	};
353	414
354	415	struct hns_roce_buf_list {
..	..	@@ -359,16 +420,16 @@
359	420	struct hns_roce_buf {
360	421	struct hns_roce_buf_list direct;
361	422	struct hns_roce_buf_list *page_list;
362		- int nbufs;
363	423	u32 npages;
364		- int page_shift;
	424	+ u32 size;
	425	+ unsigned int page_shift;
365	426	};
366	427
367	428	struct hns_roce_db_pgdir {
368	429	struct list_head list;
369	430	DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE);
370		- DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / 2);
371		- unsigned long *bits[2];
	431	+ DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
	432	+ unsigned long *bits[HNS_ROCE_DB_TYPE_COUNT];
372	433	u32 *page;
373	434	dma_addr_t db_dma;
374	435	};
..	..	@@ -392,37 +453,55 @@
392	453	int order;
393	454	};
394	455
395		-struct hns_roce_cq_buf {
396		- struct hns_roce_buf hr_buf;
397		- struct hns_roce_mtt hr_mtt;
398		-};
399		-
400	456	struct hns_roce_cq {
401	457	struct ib_cq ib_cq;
402		- struct hns_roce_cq_buf hr_buf;
	458	+ struct hns_roce_mtr mtr;
403	459	struct hns_roce_db db;
404		- u8 db_en;
	460	+ u32 flags;
405	461	spinlock_t lock;
406		- struct ib_umem *umem;
407		- void (comp)(struct hns_roce_cq cq);
408		- void (event)(struct hns_roce_cq cq, enum hns_roce_event event_type);
409		-
410		- struct hns_roce_uar *uar;
411	462	u32 cq_depth;
412	463	u32 cons_index;
413	464	u32 *set_ci_db;
414	465	void __iomem *cq_db_l;
415	466	u16 *tptr_addr;
416	467	int arm_sn;
	468	+ int cqe_size;
417	469	unsigned long cqn;
418	470	u32 vector;
419	471	atomic_t refcount;
420	472	struct completion free;
	473	+ struct list_head sq_list; /* all qps on this send cq */
	474	+ struct list_head rq_list; /* all qps on this recv cq */
	475	+ int is_armed; /* cq is armed */
	476	+ struct list_head node; /* all armed cqs are on a list */
	477	+};
	478	+
	479	+struct hns_roce_idx_que {
	480	+ struct hns_roce_mtr mtr;
	481	+ int entry_shift;
	482	+ unsigned long *bitmap;
421	483	};
422	484
423	485	struct hns_roce_srq {
424	486	struct ib_srq ibsrq;
425		- int srqn;
	487	+ unsigned long srqn;
	488	+ u32 wqe_cnt;
	489	+ int max_gs;
	490	+ int wqe_shift;
	491	+ void __iomem *db_reg_l;
	492	+
	493	+ atomic_t refcount;
	494	+ struct completion free;
	495	+
	496	+ struct hns_roce_mtr buf_mtr;
	497	+
	498	+ u64 *wrid;
	499	+ struct hns_roce_idx_que idx_que;
	500	+ spinlock_t lock;
	501	+ int head;
	502	+ int tail;
	503	+ struct mutex mutex;
	504	+ void (event)(struct hns_roce_srq srq, enum hns_roce_event event);
426	505	};
427	506
428	507	struct hns_roce_uar_table {
..	..	@@ -431,16 +510,22 @@
431	510
432	511	struct hns_roce_qp_table {
433	512	struct hns_roce_bitmap bitmap;
434		- spinlock_t lock;
435	513	struct hns_roce_hem_table qp_table;
436	514	struct hns_roce_hem_table irrl_table;
437	515	struct hns_roce_hem_table trrl_table;
	516	+ struct hns_roce_hem_table sccc_table;
	517	+ struct mutex scc_mutex;
438	518	};
439	519
440	520	struct hns_roce_cq_table {
441	521	struct hns_roce_bitmap bitmap;
442		- spinlock_t lock;
443		- struct radix_tree_root tree;
	522	+ struct xarray array;
	523	+ struct hns_roce_hem_table table;
	524	+};
	525	+
	526	+struct hns_roce_srq_table {
	527	+ struct hns_roce_bitmap bitmap;
	528	+ struct xarray xa;
444	529	struct hns_roce_hem_table table;
445	530	};
446	531
..	..	@@ -449,14 +534,18 @@
449	534	};
450	535
451	536	struct hns_roce_av {
452		- __le32 port_pd;
453		- u8 gid_index;
454		- u8 stat_rate;
455		- u8 hop_limit;
456		- __le32 sl_tclass_flowlabel;
457		- u8 dgid[HNS_ROCE_GID_SIZE];
458		- u8 mac[6];
459		- __le16 vlan;
	537	+ u8 port;
	538	+ u8 gid_index;
	539	+ u8 stat_rate;
	540	+ u8 hop_limit;
	541	+ u32 flowlabel;
	542	+ u16 udp_sport;
	543	+ u8 sl;
	544	+ u8 tclass;
	545	+ u8 dgid[HNS_ROCE_GID_SIZE];
	546	+ u8 mac[ETH_ALEN];
	547	+ u16 vlan_id;
	548	+ u8 vlan_en;
460	549	};
461	550
462	551	struct hns_roce_ah {
..	..	@@ -497,7 +586,6 @@
497	586	* close device, switch into poll mode(non event mode)
498	587	*/
499	588	u8 use_events;
500		- u8 toggle;
501	589	};
502	590
503	591	struct hns_roce_cmd_mailbox {
..	..	@@ -522,23 +610,31 @@
522	610	u32 wqe_cnt;
523	611	};
524	612
	613	+enum {
	614	+ HNS_ROCE_FLUSH_FLAG = 0,
	615	+};
	616	+
	617	+struct hns_roce_work {
	618	+ struct hns_roce_dev *hr_dev;
	619	+ struct work_struct work;
	620	+ u32 qpn;
	621	+ u32 cqn;
	622	+ int event_type;
	623	+ int sub_type;
	624	+};
	625	+
525	626	struct hns_roce_qp {
526	627	struct ib_qp ibqp;
527		- struct hns_roce_buf hr_buf;
528	628	struct hns_roce_wq rq;
529	629	struct hns_roce_db rdb;
530	630	struct hns_roce_db sdb;
531		- u8 rdb_en;
532		- u8 sdb_en;
	631	+ unsigned long en_flags;
533	632	u32 doorbell_qpn;
534		- __le32 sq_signal_bits;
535		- u32 sq_next_wqe;
536		- int sq_max_wqes_per_wr;
537		- int sq_spare_wqes;
	633	+ enum ib_sig_type sq_signal_bits;
538	634	struct hns_roce_wq sq;
539	635
540		- struct ib_umem *umem;
541		- struct hns_roce_mtt mtt;
	636	+ struct hns_roce_mtr mtr;
	637	+
542	638	u32 buff_size;
543	639	struct mutex mutex;
544	640	u8 port;
..	..	@@ -559,12 +655,16 @@
559	655
560	656	struct hns_roce_sge sge;
561	657	u32 next_sge;
	658	+ enum ib_mtu path_mtu;
	659	+ u32 max_inline_data;
562	660
	661	+ /* 0: flush needed, 1: unneeded */
	662	+ unsigned long flush_flag;
	663	+ struct hns_roce_work flush_work;
563	664	struct hns_roce_rinl_buf rq_inl_buf;
564		-};
565		-
566		-struct hns_roce_sqp {
567		- struct hns_roce_qp hr_qp;
	665	+ struct list_head node; /* all qps are on a list */
	666	+ struct list_head rq_node; /* all recv qps are on a list */
	667	+ struct list_head sq_node; /* all send qps are on a list */
568	668	};
569	669
570	670	struct hns_roce_ib_iboe {
..	..	@@ -580,7 +680,8 @@
580	680	};
581	681
582	682	struct hns_roce_ceqe {
583		- u32 comp;
	683	+ __le32 comp;
	684	+ __le32 rsv[15];
584	685	};
585	686
586	687	struct hns_roce_aeqe {
..	..	@@ -591,6 +692,12 @@
591	692	u32 rsv0;
592	693	u32 rsv1;
593	694	} qp_event;
	695	+
	696	+ struct {
	697	+ __le32 srq;
	698	+ u32 rsv0;
	699	+ u32 rsv1;
	700	+ } srq_event;
594	701
595	702	struct {
596	703	__le32 cq;
..	..	@@ -611,13 +718,14 @@
611	718	u8 rsv0;
612	719	} __packed cmd;
613	720	} event;
	721	+ __le32 rsv[12];
614	722	};
615	723
616	724	struct hns_roce_eq {
617	725	struct hns_roce_dev *hr_dev;
618	726	void __iomem *doorbell;
619	727
620		- int type_flag;/* Aeq:1 ceq:0 */
	728	+ int type_flag; /* Aeq:1 ceq:0 */
621	729	int eqn;
622	730	u32 entries;
623	731	int log_entries;
..	..	@@ -629,23 +737,11 @@
629	737	int over_ignore;
630	738	int coalesce;
631	739	int arm_st;
632		- u64 eqe_ba;
633		- int eqe_ba_pg_sz;
634		- int eqe_buf_pg_sz;
635	740	int hop_num;
636		- u64 bt_l0; / Base address table for L0 */
637		- u64 *bt_l1; / Base address table for L1 */
638		- u64 **buf;
639		- dma_addr_t l0_dma;
640		- dma_addr_t *l1_dma;
641		- dma_addr_t *buf_dma;
642		- u32 l0_last_num; /* L0 last chunk num */
643		- u32 l1_last_num; /* L1 last chunk num */
644		- int eq_max_cnt;
	741	+ struct hns_roce_mtr mtr;
	742	+ u16 eq_max_cnt;
645	743	int eq_period;
646	744	int shift;
647		- dma_addr_t cur_eqe_ba;
648		- dma_addr_t nxt_eqe_ba;
649	745	int event_type;
650	746	int sub_type;
651	747	};
..	..	@@ -656,58 +752,77 @@
656	752	};
657	753
658	754	struct hns_roce_caps {
	755	+ u64 fw_ver;
659	756	u8 num_ports;
660	757	int gid_table_len[HNS_ROCE_MAX_PORTS];
661	758	int pkey_table_len[HNS_ROCE_MAX_PORTS];
662	759	int local_ca_ack_delay;
663	760	int num_uars;
664	761	u32 phy_num_uars;
665		- u32 max_sq_sg; /* 2 */
666		- u32 max_sq_inline; /* 32 */
667		- u32 max_rq_sg; /* 2 */
	762	+ u32 max_sq_sg;
	763	+ u32 max_sq_inline;
	764	+ u32 max_rq_sg;
668	765	u32 max_extend_sg;
669		- int num_qps; /* 256k */
	766	+ int num_qps;
670	767	int reserved_qps;
671		- u32 max_wqes; /* 16k */
672		- u32 max_sq_desc_sz; /* 64 */
673		- u32 max_rq_desc_sz; /* 64 */
	768	+ int num_qpc_timer;
	769	+ int num_cqc_timer;
	770	+ int num_srqs;
	771	+ u32 max_wqes;
	772	+ u32 max_srq_wrs;
	773	+ u32 max_srq_sges;
	774	+ u32 max_sq_desc_sz;
	775	+ u32 max_rq_desc_sz;
674	776	u32 max_srq_desc_sz;
675	777	int max_qp_init_rdma;
676	778	int max_qp_dest_rdma;
677	779	int num_cqs;
678		- int max_cqes;
679		- int min_cqes;
	780	+ u32 max_cqes;
	781	+ u32 min_cqes;
680	782	u32 min_wqes;
681	783	int reserved_cqs;
682		- int num_aeq_vectors; /* 1 */
	784	+ int reserved_srqs;
	785	+ int num_aeq_vectors;
683	786	int num_comp_vectors;
684	787	int num_other_vectors;
685	788	int num_mtpts;
686	789	u32 num_mtt_segs;
687	790	u32 num_cqe_segs;
	791	+ u32 num_srqwqe_segs;
	792	+ u32 num_idx_segs;
688	793	int reserved_mrws;
689	794	int reserved_uars;
690	795	int num_pds;
691	796	int reserved_pds;
692	797	u32 mtt_entry_sz;
693		- u32 cq_entry_sz;
	798	+ u32 cqe_sz;
694	799	u32 page_size_cap;
695	800	u32 reserved_lkey;
696	801	int mtpt_entry_sz;
697		- int qpc_entry_sz;
	802	+ int qpc_sz;
698	803	int irrl_entry_sz;
699	804	int trrl_entry_sz;
700	805	int cqc_entry_sz;
	806	+ int sccc_sz;
	807	+ int qpc_timer_entry_sz;
	808	+ int cqc_timer_entry_sz;
	809	+ int srqc_entry_sz;
	810	+ int idx_entry_sz;
701	811	u32 pbl_ba_pg_sz;
702	812	u32 pbl_buf_pg_sz;
703	813	u32 pbl_hop_num;
704	814	int aeqe_depth;
705	815	int ceqe_depth;
	816	+ u32 aeqe_size;
	817	+ u32 ceqe_size;
706	818	enum ib_mtu max_mtu;
707	819	u32 qpc_bt_num;
	820	+ u32 qpc_timer_bt_num;
708	821	u32 srqc_bt_num;
709	822	u32 cqc_bt_num;
	823	+ u32 cqc_timer_bt_num;
710	824	u32 mpt_bt_num;
	825	+ u32 sccc_bt_num;
711	826	u32 qpc_ba_pg_sz;
712	827	u32 qpc_buf_pg_sz;
713	828	u32 qpc_hop_num;
..	..	@@ -723,25 +838,52 @@
723	838	u32 mtt_ba_pg_sz;
724	839	u32 mtt_buf_pg_sz;
725	840	u32 mtt_hop_num;
726		- u32 cqe_ba_pg_sz;
	841	+ u32 wqe_sq_hop_num;
	842	+ u32 wqe_sge_hop_num;
	843	+ u32 wqe_rq_hop_num;
	844	+ u32 sccc_ba_pg_sz;
	845	+ u32 sccc_buf_pg_sz;
	846	+ u32 sccc_hop_num;
	847	+ u32 qpc_timer_ba_pg_sz;
	848	+ u32 qpc_timer_buf_pg_sz;
	849	+ u32 qpc_timer_hop_num;
	850	+ u32 cqc_timer_ba_pg_sz;
	851	+ u32 cqc_timer_buf_pg_sz;
	852	+ u32 cqc_timer_hop_num;
	853	+ u32 cqe_ba_pg_sz; /* page_size = 4K(2^cqe_ba_pg_sz) /
727	854	u32 cqe_buf_pg_sz;
728	855	u32 cqe_hop_num;
	856	+ u32 srqwqe_ba_pg_sz;
	857	+ u32 srqwqe_buf_pg_sz;
	858	+ u32 srqwqe_hop_num;
	859	+ u32 idx_ba_pg_sz;
	860	+ u32 idx_buf_pg_sz;
	861	+ u32 idx_hop_num;
729	862	u32 eqe_ba_pg_sz;
730	863	u32 eqe_buf_pg_sz;
731	864	u32 eqe_hop_num;
732	865	u32 sl_num;
733	866	u32 tsq_buf_pg_sz;
734	867	u32 tpq_buf_pg_sz;
735		- u32 chunk_sz; /* chunk size in non multihop mode*/
	868	+ u32 chunk_sz; /* chunk size in non multihop mode */
736	869	u64 flags;
	870	+ u16 default_ceq_max_cnt;
	871	+ u16 default_ceq_period;
	872	+ u16 default_aeq_max_cnt;
	873	+ u16 default_aeq_period;
	874	+ u16 default_aeq_arm_st;
	875	+ u16 default_ceq_arm_st;
737	876	};
738	877
739		-struct hns_roce_work {
740		- struct hns_roce_dev *hr_dev;
741		- struct work_struct work;
742		- u32 qpn;
743		- int event_type;
744		- int sub_type;
	878	+struct hns_roce_dfx_hw {
	879	+ int (query_cqc_info)(struct hns_roce_dev hr_dev, u32 cqn,
	880	+ int *buffer);
	881	+};
	882	+
	883	+enum hns_roce_device_state {
	884	+ HNS_ROCE_DEVICE_STATE_INITED,
	885	+ HNS_ROCE_DEVICE_STATE_RST_DOWN,
	886	+ HNS_ROCE_DEVICE_STATE_UNINIT,
745	887	};
746	888
747	889	struct hns_roce_hw {
..	..	@@ -755,20 +897,24 @@
755	897	u64 out_param, u32 in_modifier, u8 op_modifier, u16 op,
756	898	u16 token, int event);
757	899	int (chk_mbox)(struct hns_roce_dev hr_dev, unsigned long timeout);
	900	+ int (rst_prc_mbox)(struct hns_roce_dev hr_dev);
758	901	int (set_gid)(struct hns_roce_dev hr_dev, u8 port, int gid_index,
759	902	const union ib_gid gid, const struct ib_gid_attr attr);
760	903	int (set_mac)(struct hns_roce_dev hr_dev, u8 phy_port, u8 *addr);
761	904	void (set_mtu)(struct hns_roce_dev hr_dev, u8 phy_port,
762	905	enum ib_mtu mtu);
763		- int (write_mtpt)(void mb_buf, struct hns_roce_mr *mr,
764		- unsigned long mtpt_idx);
	906	+ int (write_mtpt)(struct hns_roce_dev hr_dev, void *mb_buf,
	907	+ struct hns_roce_mr *mr, unsigned long mtpt_idx);
765	908	int (rereg_write_mtpt)(struct hns_roce_dev hr_dev,
766	909	struct hns_roce_mr *mr, int flags, u32 pdn,
767	910	int mr_access_flags, u64 iova, u64 size,
768	911	void *mb_buf);
	912	+ int (frmr_write_mtpt)(struct hns_roce_dev hr_dev, void *mb_buf,
	913	+ struct hns_roce_mr *mr);
	914	+ int (mw_write_mtpt)(void mb_buf, struct hns_roce_mw *mw);
769	915	void (write_cqc)(struct hns_roce_dev hr_dev,
770	916	struct hns_roce_cq hr_cq, void mb_buf, u64 *mtts,
771		- dma_addr_t dma_handle, int nent, u32 vector);
	917	+ dma_addr_t dma_handle);
772	918	int (set_hem)(struct hns_roce_dev hr_dev,
773	919	struct hns_roce_hem_table *table, int obj, int step_idx);
774	920	int (clear_hem)(struct hns_roce_dev hr_dev,
..	..	@@ -779,18 +925,34 @@
779	925	int (modify_qp)(struct ib_qp ibqp, const struct ib_qp_attr *attr,
780	926	int attr_mask, enum ib_qp_state cur_state,
781	927	enum ib_qp_state new_state);
782		- int (destroy_qp)(struct ib_qp ibqp);
	928	+ int (destroy_qp)(struct ib_qp ibqp, struct ib_udata *udata);
	929	+ int (qp_flow_control_init)(struct hns_roce_dev hr_dev,
	930	+ struct hns_roce_qp *hr_qp);
783	931	int (post_send)(struct ib_qp ibqp, const struct ib_send_wr *wr,
784	932	const struct ib_send_wr **bad_wr);
785	933	int (post_recv)(struct ib_qp qp, const struct ib_recv_wr *recv_wr,
786	934	const struct ib_recv_wr **bad_recv_wr);
787	935	int (req_notify_cq)(struct ib_cq ibcq, enum ib_cq_notify_flags flags);
788	936	int (poll_cq)(struct ib_cq ibcq, int num_entries, struct ib_wc *wc);
789		- int (dereg_mr)(struct hns_roce_dev hr_dev, struct hns_roce_mr *mr);
790		- int (destroy_cq)(struct ib_cq ibcq);
	937	+ int (dereg_mr)(struct hns_roce_dev hr_dev, struct hns_roce_mr *mr,
	938	+ struct ib_udata *udata);
	939	+ int (destroy_cq)(struct ib_cq ibcq, struct ib_udata *udata);
791	940	int (modify_cq)(struct ib_cq cq, u16 cq_count, u16 cq_period);
792	941	int (init_eq)(struct hns_roce_dev hr_dev);
793	942	void (cleanup_eq)(struct hns_roce_dev hr_dev);
	943	+ void (write_srqc)(struct hns_roce_dev hr_dev,
	944	+ struct hns_roce_srq *srq, u32 pdn, u16 xrcd, u32 cqn,
	945	+ void mb_buf, u64 mtts_wqe, u64 *mtts_idx,
	946	+ dma_addr_t dma_handle_wqe,
	947	+ dma_addr_t dma_handle_idx);
	948	+ int (modify_srq)(struct ib_srq ibsrq, struct ib_srq_attr *srq_attr,
	949	+ enum ib_srq_attr_mask srq_attr_mask,
	950	+ struct ib_udata *udata);
	951	+ int (query_srq)(struct ib_srq ibsrq, struct ib_srq_attr *attr);
	952	+ int (post_srq_recv)(struct ib_srq ibsrq, const struct ib_recv_wr *wr,
	953	+ const struct ib_recv_wr **bad_wr);
	954	+ const struct ib_device_ops *hns_roce_dev_ops;
	955	+ const struct ib_device_ops *hns_roce_dev_srq_ops;
794	956	};
795	957
796	958	struct hns_roce_dev {
..	..	@@ -804,16 +966,21 @@
804	966	spinlock_t bt_cmd_lock;
805	967	bool active;
806	968	bool is_reset;
	969	+ bool dis_db;
	970	+ unsigned long reset_cnt;
807	971	struct hns_roce_ib_iboe iboe;
	972	+ enum hns_roce_device_state state;
	973	+ struct list_head qp_list; /* list of all qps on this dev */
	974	+ spinlock_t qp_list_lock; /* protect qp_list */
808	975
809	976	struct list_head pgdir_list;
810	977	struct mutex pgdir_mutex;
811	978	int irq[HNS_ROCE_MAX_IRQ_NUM];
812	979	u8 __iomem *reg_base;
813	980	struct hns_roce_caps caps;
814		- struct radix_tree_root qp_table_tree;
	981	+ struct xarray qp_table_xa;
815	982
816		- unsigned char dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM];
	983	+ unsigned char dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN];
817	984	u64 sys_image_guid;
818	985	u32 vendor_id;
819	986	u32 vendor_part_id;
..	..	@@ -825,18 +992,22 @@
825	992	struct hns_roce_uar_table uar_table;
826	993	struct hns_roce_mr_table mr_table;
827	994	struct hns_roce_cq_table cq_table;
	995	+ struct hns_roce_srq_table srq_table;
828	996	struct hns_roce_qp_table qp_table;
829	997	struct hns_roce_eq_table eq_table;
	998	+ struct hns_roce_hem_table qpc_timer_table;
	999	+ struct hns_roce_hem_table cqc_timer_table;
830	1000
831	1001	int cmd_mod;
832	1002	int loop_idc;
833	1003	u32 sdb_offset;
834	1004	u32 odb_offset;
835		- dma_addr_t tptr_dma_addr; /only for hw v1/
836		- u32 tptr_size; /only for hw v1/
	1005	+ dma_addr_t tptr_dma_addr; /* only for hw v1 */
	1006	+ u32 tptr_size; /* only for hw v1 */
837	1007	const struct hns_roce_hw *hw;
838	1008	void *priv;
839	1009	struct workqueue_struct *irq_workq;
	1010	+ const struct hns_roce_dfx_hw *dfx;
840	1011	};
841	1012
842	1013	static inline struct hns_roce_dev to_hr_dev(struct ib_device ib_dev)
..	..	@@ -865,6 +1036,11 @@
865	1036	return container_of(ibmr, struct hns_roce_mr, ibmr);
866	1037	}
867	1038
	1039	+static inline struct hns_roce_mw to_hr_mw(struct ib_mw ibmw)
	1040	+{
	1041	+ return container_of(ibmw, struct hns_roce_mw, ibmw);
	1042	+}
	1043	+
868	1044	static inline struct hns_roce_qp to_hr_qp(struct ib_qp ibqp)
869	1045	{
870	1046	return container_of(ibqp, struct hns_roce_qp, ibqp);
..	..	@@ -880,11 +1056,6 @@
880	1056	return container_of(ibsrq, struct hns_roce_srq, ibsrq);
881	1057	}
882	1058
883		-static inline struct hns_roce_sqp hr_to_hr_sqp(struct hns_roce_qp hr_qp)
884		-{
885		- return container_of(hr_qp, struct hns_roce_sqp, hr_qp);
886		-}
887		-
888	1059	static inline void hns_roce_write64_k(__le32 val[2], void __iomem *dest)
889	1060	{
890	1061	__raw_writeq((u64 ) val, dest);
..	..	@@ -893,19 +1064,78 @@
893	1064	static inline struct hns_roce_qp
894	1065	__hns_roce_qp_lookup(struct hns_roce_dev hr_dev, u32 qpn)
895	1066	{
896		- return radix_tree_lookup(&hr_dev->qp_table_tree,
897		- qpn & (hr_dev->caps.num_qps - 1));
	1067	+ return xa_load(&hr_dev->qp_table_xa, qpn & (hr_dev->caps.num_qps - 1));
	1068	+}
	1069	+
	1070	+static inline bool hns_roce_buf_is_direct(struct hns_roce_buf *buf)
	1071	+{
	1072	+ if (buf->page_list)
	1073	+ return false;
	1074	+
	1075	+ return true;
898	1076	}
899	1077
900	1078	static inline void hns_roce_buf_offset(struct hns_roce_buf buf, int offset)
901	1079	{
902		- u32 page_size = 1 << buf->page_shift;
	1080	+ if (hns_roce_buf_is_direct(buf))
	1081	+ return (char *)(buf->direct.buf) + (offset & (buf->size - 1));
903	1082
904		- if (buf->nbufs == 1)
905		- return (char *)(buf->direct.buf) + offset;
	1083	+ return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
	1084	+ (offset & ((1 << buf->page_shift) - 1));
	1085	+}
	1086	+
	1087	+static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, int idx)
	1088	+{
	1089	+ if (hns_roce_buf_is_direct(buf))
	1090	+ return buf->direct.map + ((dma_addr_t)idx << buf->page_shift);
906	1091	else
907		- return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
908		- (offset & (page_size - 1));
	1092	+ return buf->page_list[idx].map;
	1093	+}
	1094	+
	1095	+#define hr_hw_page_align(x) ALIGN(x, 1 << HNS_HW_PAGE_SHIFT)
	1096	+
	1097	+static inline u64 to_hr_hw_page_addr(u64 addr)
	1098	+{
	1099	+ return addr >> HNS_HW_PAGE_SHIFT;
	1100	+}
	1101	+
	1102	+static inline u32 to_hr_hw_page_shift(u32 page_shift)
	1103	+{
	1104	+ return page_shift - HNS_HW_PAGE_SHIFT;
	1105	+}
	1106	+
	1107	+static inline u32 to_hr_hem_hopnum(u32 hopnum, u32 count)
	1108	+{
	1109	+ if (count > 0)
	1110	+ return hopnum == HNS_ROCE_HOP_NUM_0 ? 0 : hopnum;
	1111	+
	1112	+ return 0;
	1113	+}
	1114	+
	1115	+static inline u32 to_hr_hem_entries_size(u32 count, u32 buf_shift)
	1116	+{
	1117	+ return hr_hw_page_align(count << buf_shift);
	1118	+}
	1119	+
	1120	+static inline u32 to_hr_hem_entries_count(u32 count, u32 buf_shift)
	1121	+{
	1122	+ return hr_hw_page_align(count << buf_shift) >> buf_shift;
	1123	+}
	1124	+
	1125	+static inline u32 to_hr_hem_entries_shift(u32 count, u32 buf_shift)
	1126	+{
	1127	+ if (!count)
	1128	+ return 0;
	1129	+
	1130	+ return ilog2(to_hr_hem_entries_count(count, buf_shift));
	1131	+}
	1132	+
	1133	+#define DSCP_SHIFT 2
	1134	+
	1135	+static inline u8 get_tclass(const struct ib_global_route *grh)
	1136	+{
	1137	+ return grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP ?
	1138	+ grh->traffic_class >> DSCP_SHIFT : grh->traffic_class;
909	1139	}
910	1140
911	1141	int hns_roce_init_uar_table(struct hns_roce_dev *dev);
..	..	@@ -920,24 +1150,31 @@
920	1150	int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev);
921	1151	void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);
922	1152
923		-int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
924		- struct hns_roce_mtt *mtt);
925		-void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev,
926		- struct hns_roce_mtt *mtt);
927		-int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
928		- struct hns_roce_mtt mtt, struct hns_roce_buf buf);
	1153	+/* hns roce hw need current block and next block addr from mtt */
	1154	+#define MTT_MIN_COUNT 2
	1155	+int hns_roce_mtr_find(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,
	1156	+ int offset, u64 mtt_buf, int mtt_max, u64 base_addr);
	1157	+int hns_roce_mtr_create(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,
	1158	+ struct hns_roce_buf_attr *buf_attr,
	1159	+ unsigned int page_shift, struct ib_udata *udata,
	1160	+ unsigned long user_addr);
	1161	+void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
	1162	+ struct hns_roce_mtr *mtr);
	1163	+int hns_roce_mtr_map(struct hns_roce_dev hr_dev, struct hns_roce_mtr mtr,
	1164	+ dma_addr_t *pages, int page_cnt);
929	1165
930	1166	int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);
931	1167	int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
932		-int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev);
933	1168	int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);
934	1169	int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev);
	1170	+int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev);
935	1171
936	1172	void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev);
937	1173	void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev);
938	1174	void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev);
939	1175	void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);
940	1176	void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);
	1177	+void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev);
941	1178
942	1179	int hns_roce_bitmap_alloc(struct hns_roce_bitmap bitmap, unsigned long obj);
943	1180	void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,
..	..	@@ -952,16 +1189,16 @@
952	1189	unsigned long obj, int cnt,
953	1190	int rr);
954	1191
955		-struct ib_ah hns_roce_create_ah(struct ib_pd pd,
956		- struct rdma_ah_attr *ah_attr,
957		- struct ib_udata *udata);
	1192	+int hns_roce_create_ah(struct ib_ah ah, struct rdma_ah_init_attr init_attr,
	1193	+ struct ib_udata *udata);
958	1194	int hns_roce_query_ah(struct ib_ah ibah, struct rdma_ah_attr ah_attr);
959		-int hns_roce_destroy_ah(struct ib_ah *ah);
	1195	+static inline int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags)
	1196	+{
	1197	+ return 0;
	1198	+}
960	1199
961		-struct ib_pd hns_roce_alloc_pd(struct ib_device ib_dev,
962		- struct ib_ucontext *context,
963		- struct ib_udata *udata);
964		-int hns_roce_dealloc_pd(struct ib_pd *pd);
	1200	+int hns_roce_alloc_pd(struct ib_pd pd, struct ib_udata udata);
	1201	+int hns_roce_dealloc_pd(struct ib_pd pd, struct ib_udata udata);
965	1202
966	1203	struct ib_mr hns_roce_get_dma_mr(struct ib_pd pd, int acc);
967	1204	struct ib_mr hns_roce_reg_user_mr(struct ib_pd pd, u64 start, u64 length,
..	..	@@ -970,28 +1207,46 @@
970	1207	int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length,
971	1208	u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
972	1209	struct ib_udata *udata);
973		-int hns_roce_dereg_mr(struct ib_mr *ibmr);
974		-int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
975		- struct hns_roce_cmd_mailbox *mailbox,
976		- unsigned long mpt_index);
	1210	+struct ib_mr hns_roce_alloc_mr(struct ib_pd pd, enum ib_mr_type mr_type,
	1211	+ u32 max_num_sg);
	1212	+int hns_roce_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg, int sg_nents,
	1213	+ unsigned int *sg_offset);
	1214	+int hns_roce_dereg_mr(struct ib_mr ibmr, struct ib_udata udata);
	1215	+int hns_roce_hw_destroy_mpt(struct hns_roce_dev *hr_dev,
	1216	+ struct hns_roce_cmd_mailbox *mailbox,
	1217	+ unsigned long mpt_index);
977	1218	unsigned long key_to_hw_index(u32 key);
978	1219
979		-void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
980		- struct hns_roce_buf *buf);
	1220	+int hns_roce_alloc_mw(struct ib_mw mw, struct ib_udata udata);
	1221	+int hns_roce_dealloc_mw(struct ib_mw *ibmw);
	1222	+
	1223	+void hns_roce_buf_free(struct hns_roce_dev hr_dev, struct hns_roce_buf buf);
981	1224	int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
982	1225	struct hns_roce_buf *buf, u32 page_shift);
983	1226
984		-int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
985		- struct hns_roce_mtt mtt, struct ib_umem umem);
	1227	+int hns_roce_get_kmem_bufs(struct hns_roce_dev hr_dev, dma_addr_t bufs,
	1228	+ int buf_cnt, int start, struct hns_roce_buf *buf);
	1229	+int hns_roce_get_umem_bufs(struct hns_roce_dev hr_dev, dma_addr_t bufs,
	1230	+ int buf_cnt, int start, struct ib_umem *umem,
	1231	+ unsigned int page_shift);
	1232	+
	1233	+int hns_roce_create_srq(struct ib_srq *srq,
	1234	+ struct ib_srq_init_attr *srq_init_attr,
	1235	+ struct ib_udata *udata);
	1236	+int hns_roce_modify_srq(struct ib_srq ibsrq, struct ib_srq_attr srq_attr,
	1237	+ enum ib_srq_attr_mask srq_attr_mask,
	1238	+ struct ib_udata *udata);
	1239	+int hns_roce_destroy_srq(struct ib_srq ibsrq, struct ib_udata udata);
986	1240
987	1241	struct ib_qp hns_roce_create_qp(struct ib_pd ib_pd,
988	1242	struct ib_qp_init_attr *init_attr,
989	1243	struct ib_udata *udata);
990	1244	int hns_roce_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr,
991	1245	int attr_mask, struct ib_udata *udata);
992		-void get_recv_wqe(struct hns_roce_qp hr_qp, int n);
993		-void get_send_wqe(struct hns_roce_qp hr_qp, int n);
994		-void get_send_extend_sge(struct hns_roce_qp hr_qp, int n);
	1246	+void init_flush_work(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp);
	1247	+void hns_roce_get_recv_wqe(struct hns_roce_qp hr_qp, int n);
	1248	+void hns_roce_get_send_wqe(struct hns_roce_qp hr_qp, int n);
	1249	+void hns_roce_get_extend_sge(struct hns_roce_qp hr_qp, int n);
995	1250	bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
996	1251	struct ib_cq *ib_cq);
997	1252	enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state);
..	..	@@ -1000,21 +1255,17 @@
1000	1255	void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1001	1256	struct hns_roce_cq *recv_cq);
1002	1257	void hns_roce_qp_remove(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp);
1003		-void hns_roce_qp_free(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp);
1004		-void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
1005		- int cnt);
	1258	+void hns_roce_qp_destroy(struct hns_roce_dev hr_dev, struct hns_roce_qp hr_qp,
	1259	+ struct ib_udata *udata);
1006	1260	__be32 send_ieth(const struct ib_send_wr *wr);
1007	1261	int to_hr_qp_type(int qp_type);
1008	1262
1009		-struct ib_cq hns_roce_ib_create_cq(struct ib_device ib_dev,
1010		- const struct ib_cq_init_attr *attr,
1011		- struct ib_ucontext *context,
1012		- struct ib_udata *udata);
	1263	+int hns_roce_create_cq(struct ib_cq ib_cq, const struct ib_cq_init_attr attr,
	1264	+ struct ib_udata *udata);
1013	1265
1014		-int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq);
1015		-void hns_roce_free_cq(struct hns_roce_dev hr_dev, struct hns_roce_cq hr_cq);
1016		-
1017		-int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
	1266	+int hns_roce_destroy_cq(struct ib_cq ib_cq, struct ib_udata udata);
	1267	+int hns_roce_db_map_user(struct hns_roce_ucontext *context,
	1268	+ struct ib_udata *udata, unsigned long virt,
1018	1269	struct hns_roce_db *db);
1019	1270	void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
1020	1271	struct hns_roce_db *db);
..	..	@@ -1025,8 +1276,12 @@
1025	1276	void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);
1026	1277	void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
1027	1278	void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);
	1279	+void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type);
1028	1280	int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
	1281	+void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
1029	1282	int hns_roce_init(struct hns_roce_dev *hr_dev);
1030	1283	void hns_roce_exit(struct hns_roce_dev *hr_dev);
1031	1284
	1285	+int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
	1286	+ struct ib_cq *ib_cq);
1032	1287	#endif /* _HNS_ROCE_DEVICE_H */