forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/drivers/infiniband/hw/hns/hns_roce_hem.c
....@@ -39,39 +39,70 @@
3939 #define DMA_ADDR_T_SHIFT		12
4040 #define BT_BA_SHIFT			32
4141 
42
+#define HEM_INDEX_BUF			BIT(0)
43
+#define HEM_INDEX_L0			BIT(1)
44
+#define HEM_INDEX_L1			BIT(2)
45
+struct hns_roce_hem_index {
46
+	u64 buf;
47
+	u64 l0;
48
+	u64 l1;
49
+	u32 inited; /* indicate which index is available */
50
+};
51
+
4252 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
4353 {
44
-	if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) ||
45
-	    (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
46
-	    (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
47
-	    (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
48
-	    (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
49
-	    (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT))
50
-		return true;
54
+	int hop_num = 0;
5155 
52
-	return false;
56
+	switch (type) {
57
+	case HEM_TYPE_QPC:
58
+		hop_num = hr_dev->caps.qpc_hop_num;
59
+		break;
60
+	case HEM_TYPE_MTPT:
61
+		hop_num = hr_dev->caps.mpt_hop_num;
62
+		break;
63
+	case HEM_TYPE_CQC:
64
+		hop_num = hr_dev->caps.cqc_hop_num;
65
+		break;
66
+	case HEM_TYPE_SRQC:
67
+		hop_num = hr_dev->caps.srqc_hop_num;
68
+		break;
69
+	case HEM_TYPE_SCCC:
70
+		hop_num = hr_dev->caps.sccc_hop_num;
71
+		break;
72
+	case HEM_TYPE_QPC_TIMER:
73
+		hop_num = hr_dev->caps.qpc_timer_hop_num;
74
+		break;
75
+	case HEM_TYPE_CQC_TIMER:
76
+		hop_num = hr_dev->caps.cqc_timer_hop_num;
77
+		break;
78
+	default:
79
+		return false;
80
+	}
81
+
82
+	return hop_num ? true : false;
5383 }
54
-EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);
5584 
56
-static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
57
-			    u32 bt_chunk_num, u64 hem_max_num)
85
+static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
86
+				    u32 bt_chunk_num, u64 hem_max_num)
5887 {
88
+	u64 start_idx = round_down(hem_idx, bt_chunk_num);
5989 	u64 check_max_num = start_idx + bt_chunk_num;
6090 	u64 i;
6191 
6292 	for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
63
-		if (hem[i])
93
+		if (i != hem_idx && hem[i])
6494 			return false;
6595 
6696 	return true;
6797 }
6898 
69
-static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
99
+static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
70100 {
101
+	u64 start_idx = round_down(ba_idx, bt_chunk_num);
71102 	int i;
72103 
73104 	for (i = 0; i < bt_chunk_num; i++)
74
-		if (bt[start_idx + i])
105
+		if (i != ba_idx && bt[start_idx + i])
75106 			return false;
76107 
77108 	return true;
....@@ -89,17 +120,13 @@
89120 		return 0;
90121 }
91122 
92
-int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
93
-			   struct hns_roce_hem_table *table, unsigned long *obj,
94
-			   struct hns_roce_hem_mhop *mhop)
123
+static int get_hem_table_config(struct hns_roce_dev *hr_dev,
124
+				struct hns_roce_hem_mhop *mhop,
125
+				u32 type)
95126 {
96127 	struct device *dev = hr_dev->dev;
97
-	u32 chunk_ba_num;
98
-	u32 table_idx;
99
-	u32 bt_num;
100
-	u32 chunk_size;
101128 
102
-	switch (table->type) {
129
+	switch (type) {
103130 	case HEM_TYPE_QPC:
104131 		mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
105132 					     + PAGE_SHIFT);
....@@ -124,6 +151,30 @@
124151 		mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
125152 		mhop->hop_num = hr_dev->caps.cqc_hop_num;
126153 		break;
154
+	case HEM_TYPE_SCCC:
155
+		mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
156
+					     + PAGE_SHIFT);
157
+		mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
158
+					    + PAGE_SHIFT);
159
+		mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
160
+		mhop->hop_num = hr_dev->caps.sccc_hop_num;
161
+		break;
162
+	case HEM_TYPE_QPC_TIMER:
163
+		mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
164
+					     + PAGE_SHIFT);
165
+		mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
166
+					    + PAGE_SHIFT);
167
+		mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
168
+		mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
169
+		break;
170
+	case HEM_TYPE_CQC_TIMER:
171
+		mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
172
+					     + PAGE_SHIFT);
173
+		mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
174
+					    + PAGE_SHIFT);
175
+		mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
176
+		mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
177
+		break;
127178 	case HEM_TYPE_SRQC:
128179 		mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
129180 					     + PAGE_SHIFT);
....@@ -132,37 +183,37 @@
132183 		mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
133184 		mhop->hop_num = hr_dev->caps.srqc_hop_num;
134185 		break;
135
-	case HEM_TYPE_MTT:
136
-		mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
137
-					     + PAGE_SHIFT);
138
-		mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
139
-					     + PAGE_SHIFT);
140
-		mhop->ba_l0_num = mhop->bt_chunk_size / 8;
141
-		mhop->hop_num = hr_dev->caps.mtt_hop_num;
142
-		break;
143
-	case HEM_TYPE_CQE:
144
-		mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
145
-					     + PAGE_SHIFT);
146
-		mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
147
-					     + PAGE_SHIFT);
148
-		mhop->ba_l0_num = mhop->bt_chunk_size / 8;
149
-		mhop->hop_num = hr_dev->caps.cqe_hop_num;
150
-		break;
151186 	default:
152
-		dev_err(dev, "Table %d not support multi-hop addressing!\n",
153
-			 table->type);
187
+		dev_err(dev, "table %u not support multi-hop addressing!\n",
188
+			type);
154189 		return -EINVAL;
155190 	}
191
+
192
+	return 0;
193
+}
194
+
195
+int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
196
+			   struct hns_roce_hem_table *table, unsigned long *obj,
197
+			   struct hns_roce_hem_mhop *mhop)
198
+{
199
+	struct device *dev = hr_dev->dev;
200
+	u32 chunk_ba_num;
201
+	u32 table_idx;
202
+	u32 bt_num;
203
+	u32 chunk_size;
204
+
205
+	if (get_hem_table_config(hr_dev, mhop, table->type))
206
+		return -EINVAL;
156207 
157208 	if (!obj)
158209 		return 0;
159210 
160211 	/*
161
-	 * QPC/MTPT/CQC/SRQC alloc hem for buffer pages.
212
+	 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
162213 	 * MTT/CQE alloc hem for bt pages.
163214 	 */
164215 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
165
-	chunk_ba_num = mhop->bt_chunk_size / 8;
216
+	chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
166217 	chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
167218 			      mhop->bt_chunk_size;
168219 	table_idx = (*obj & (table->num_obj - 1)) /
....@@ -181,8 +232,8 @@
181232 		mhop->l0_idx = table_idx;
182233 		break;
183234 	default:
184
-		dev_err(dev, "Table %d not support hop_num = %d!\n",
185
-			     table->type, mhop->hop_num);
235
+		dev_err(dev, "table %u not support hop_num = %u!\n",
236
+			table->type, mhop->hop_num);
186237 		return -EINVAL;
187238 	}
188239 	if (mhop->l0_idx >= mhop->ba_l0_num)
....@@ -190,7 +241,6 @@
190241 
191242 	return 0;
192243 }
193
-EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);
194244 
195245 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
196246 					       int npages,
....@@ -282,14 +332,14 @@
282332 {
283333 	spinlock_t *lock = &hr_dev->bt_cmd_lock;
284334 	struct device *dev = hr_dev->dev;
285
-	unsigned long end = 0;
335
+	long end;
286336 	unsigned long flags;
287337 	struct hns_roce_hem_iter iter;
288338 	void __iomem *bt_cmd;
289
-	u32 bt_cmd_h_val = 0;
290
-	u32 bt_cmd_val[2];
291
-	u32 bt_cmd_l = 0;
292
-	u64 bt_ba = 0;
339
+	__le32 bt_cmd_val[2];
340
+	__le32 bt_cmd_h = 0;
341
+	__le32 bt_cmd_l;
342
+	u64 bt_ba;
293343 	int ret = 0;
294344 
295345 	/* Find the HEM(Hardware Entry Memory) entry */
....@@ -298,30 +348,20 @@
298348 
299349 	switch (table->type) {
300350 	case HEM_TYPE_QPC:
301
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
302
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
303
-		break;
304351 	case HEM_TYPE_MTPT:
305
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
306
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
307
-			       HEM_TYPE_MTPT);
308
-		break;
309352 	case HEM_TYPE_CQC:
310
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
311
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
312
-		break;
313353 	case HEM_TYPE_SRQC:
314
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
315
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
316
-			       HEM_TYPE_SRQC);
354
+		roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
355
+			ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type);
317356 		break;
318357 	default:
319358 		return ret;
320359 	}
321
-	roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
360
+
361
+	roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
322362 		       ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
323
-	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
324
-	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
363
+	roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
364
+	roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
325365 
326366 	/* Currently iter only a chunk */
327367 	for (hns_roce_hem_first(table->hem[i], &iter);
....@@ -332,27 +372,28 @@
332372 
333373 		bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
334374 
335
-		end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
336
-		while (1) {
337
-			if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
338
-				if (!(time_before(jiffies, end))) {
339
-					dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
340
-					spin_unlock_irqrestore(lock, flags);
341
-					return -EBUSY;
342
-				}
343
-			} else {
375
+		end = HW_SYNC_TIMEOUT_MSECS;
376
+		while (end > 0) {
377
+			if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT))
344378 				break;
345
-			}
379
+
346380 			mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
381
+			end -= HW_SYNC_SLEEP_TIME_INTERVAL;
347382 		}
348383 
349
-		bt_cmd_l = (u32)bt_ba;
350
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
384
+		if (end <= 0) {
385
+			dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
386
+			spin_unlock_irqrestore(lock, flags);
387
+			return -EBUSY;
388
+		}
389
+
390
+		bt_cmd_l = cpu_to_le32(bt_ba);
391
+		roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
351392 			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
352393 			       bt_ba >> BT_BA_SHIFT);
353394 
354395 		bt_cmd_val[0] = bt_cmd_l;
355
-		bt_cmd_val[1] = bt_cmd_h_val;
396
+		bt_cmd_val[1] = bt_cmd_h;
356397 		hns_roce_write64_k(bt_cmd_val,
357398 				   hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
358399 		spin_unlock_irqrestore(lock, flags);
....@@ -361,178 +402,235 @@
361402 	return ret;
362403 }
363404 
405
+static int calc_hem_config(struct hns_roce_dev *hr_dev,
406
+			   struct hns_roce_hem_table *table, unsigned long obj,
407
+			   struct hns_roce_hem_mhop *mhop,
408
+			   struct hns_roce_hem_index *index)
409
+{
410
+	struct ib_device *ibdev = &hr_dev->ib_dev;
411
+	unsigned long mhop_obj = obj;
412
+	u32 l0_idx, l1_idx, l2_idx;
413
+	u32 chunk_ba_num;
414
+	u32 bt_num;
415
+	int ret;
416
+
417
+	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
418
+	if (ret)
419
+		return ret;
420
+
421
+	l0_idx = mhop->l0_idx;
422
+	l1_idx = mhop->l1_idx;
423
+	l2_idx = mhop->l2_idx;
424
+	chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
425
+	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
426
+	switch (bt_num) {
427
+	case 3:
428
+		index->l1 = l0_idx * chunk_ba_num + l1_idx;
429
+		index->l0 = l0_idx;
430
+		index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
431
+			     l1_idx * chunk_ba_num + l2_idx;
432
+		break;
433
+	case 2:
434
+		index->l0 = l0_idx;
435
+		index->buf = l0_idx * chunk_ba_num + l1_idx;
436
+		break;
437
+	case 1:
438
+		index->buf = l0_idx;
439
+		break;
440
+	default:
441
+		ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
442
+			  table->type, mhop->hop_num);
443
+		return -EINVAL;
444
+	}
445
+
446
+	if (unlikely(index->buf >= table->num_hem)) {
447
+		ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
448
+			  table->type, index->buf, table->num_hem);
449
+		return -EINVAL;
450
+	}
451
+
452
+	return 0;
453
+}
454
+
455
+static void free_mhop_hem(struct hns_roce_dev *hr_dev,
456
+			  struct hns_roce_hem_table *table,
457
+			  struct hns_roce_hem_mhop *mhop,
458
+			  struct hns_roce_hem_index *index)
459
+{
460
+	u32 bt_size = mhop->bt_chunk_size;
461
+	struct device *dev = hr_dev->dev;
462
+
463
+	if (index->inited & HEM_INDEX_BUF) {
464
+		hns_roce_free_hem(hr_dev, table->hem[index->buf]);
465
+		table->hem[index->buf] = NULL;
466
+	}
467
+
468
+	if (index->inited & HEM_INDEX_L1) {
469
+		dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
470
+				  table->bt_l1_dma_addr[index->l1]);
471
+		table->bt_l1[index->l1] = NULL;
472
+	}
473
+
474
+	if (index->inited & HEM_INDEX_L0) {
475
+		dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
476
+				  table->bt_l0_dma_addr[index->l0]);
477
+		table->bt_l0[index->l0] = NULL;
478
+	}
479
+}
480
+
481
+static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
482
+			  struct hns_roce_hem_table *table,
483
+			  struct hns_roce_hem_mhop *mhop,
484
+			  struct hns_roce_hem_index *index)
485
+{
486
+	u32 bt_size = mhop->bt_chunk_size;
487
+	struct device *dev = hr_dev->dev;
488
+	struct hns_roce_hem_iter iter;
489
+	gfp_t flag;
490
+	u64 bt_ba;
491
+	u32 size;
492
+	int ret;
493
+
494
+	/* alloc L1 BA's chunk */
495
+	if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
496
+	     check_whether_bt_num_2(table->type, mhop->hop_num)) &&
497
+	     !table->bt_l0[index->l0]) {
498
+		table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
499
+					    &table->bt_l0_dma_addr[index->l0],
500
+					    GFP_KERNEL);
501
+		if (!table->bt_l0[index->l0]) {
502
+			ret = -ENOMEM;
503
+			goto out;
504
+		}
505
+		index->inited |= HEM_INDEX_L0;
506
+	}
507
+
508
+	/* alloc L2 BA's chunk */
509
+	if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
510
+	    !table->bt_l1[index->l1])  {
511
+		table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
512
+					    &table->bt_l1_dma_addr[index->l1],
513
+					    GFP_KERNEL);
514
+		if (!table->bt_l1[index->l1]) {
515
+			ret = -ENOMEM;
516
+			goto err_alloc_hem;
517
+		}
518
+		index->inited |= HEM_INDEX_L1;
519
+		*(table->bt_l0[index->l0] + mhop->l1_idx) =
520
+					       table->bt_l1_dma_addr[index->l1];
521
+	}
522
+
523
+	/*
524
+	 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
525
+	 * alloc bt space chunk for MTT/CQE.
526
+	 */
527
+	size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
528
+	flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;
529
+	table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
530
+						    size, flag);
531
+	if (!table->hem[index->buf]) {
532
+		ret = -ENOMEM;
533
+		goto err_alloc_hem;
534
+	}
535
+
536
+	index->inited |= HEM_INDEX_BUF;
537
+	hns_roce_hem_first(table->hem[index->buf], &iter);
538
+	bt_ba = hns_roce_hem_addr(&iter);
539
+	if (table->type < HEM_TYPE_MTT) {
540
+		if (mhop->hop_num == 2)
541
+			*(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
542
+		else if (mhop->hop_num == 1)
543
+			*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
544
+	} else if (mhop->hop_num == 2) {
545
+		*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
546
+	}
547
+
548
+	return 0;
549
+err_alloc_hem:
550
+	free_mhop_hem(hr_dev, table, mhop, index);
551
+out:
552
+	return ret;
553
+}
554
+
555
+static int set_mhop_hem(struct hns_roce_dev *hr_dev,
556
+			struct hns_roce_hem_table *table, unsigned long obj,
557
+			struct hns_roce_hem_mhop *mhop,
558
+			struct hns_roce_hem_index *index)
559
+{
560
+	struct ib_device *ibdev = &hr_dev->ib_dev;
561
+	int step_idx;
562
+	int ret = 0;
563
+
564
+	if (index->inited & HEM_INDEX_L0) {
565
+		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
566
+		if (ret) {
567
+			ibdev_err(ibdev, "set HEM step 0 failed!\n");
568
+			goto out;
569
+		}
570
+	}
571
+
572
+	if (index->inited & HEM_INDEX_L1) {
573
+		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
574
+		if (ret) {
575
+			ibdev_err(ibdev, "set HEM step 1 failed!\n");
576
+			goto out;
577
+		}
578
+	}
579
+
580
+	if (index->inited & HEM_INDEX_BUF) {
581
+		if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
582
+			step_idx = 0;
583
+		else
584
+			step_idx = mhop->hop_num;
585
+		ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
586
+		if (ret)
587
+			ibdev_err(ibdev, "set HEM step last failed!\n");
588
+	}
589
+out:
590
+	return ret;
591
+}
592
+
364593 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
365594 				   struct hns_roce_hem_table *table,
366595 				   unsigned long obj)
367596 {
368
-	struct device *dev = hr_dev->dev;
369
-	struct hns_roce_hem_mhop mhop;
370
-	struct hns_roce_hem_iter iter;
371
-	u32 buf_chunk_size;
372
-	u32 bt_chunk_size;
373
-	u32 chunk_ba_num;
374
-	u32 hop_num;
375
-	u32 size;
376
-	u32 bt_num;
377
-	u64 hem_idx;
378
-	u64 bt_l1_idx = 0;
379
-	u64 bt_l0_idx = 0;
380
-	u64 bt_ba;
381
-	unsigned long mhop_obj = obj;
382
-	int bt_l1_allocated = 0;
383
-	int bt_l0_allocated = 0;
384
-	int step_idx;
597
+	struct ib_device *ibdev = &hr_dev->ib_dev;
598
+	struct hns_roce_hem_index index = {};
599
+	struct hns_roce_hem_mhop mhop = {};
385600 	int ret;
386601 
387
-	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
388
-	if (ret)
602
+	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
603
+	if (ret) {
604
+		ibdev_err(ibdev, "calc hem config failed!\n");
389605 		return ret;
390
-
391
-	buf_chunk_size = mhop.buf_chunk_size;
392
-	bt_chunk_size = mhop.bt_chunk_size;
393
-	hop_num = mhop.hop_num;
394
-	chunk_ba_num = bt_chunk_size / 8;
395
-
396
-	bt_num = hns_roce_get_bt_num(table->type, hop_num);
397
-	switch (bt_num) {
398
-	case 3:
399
-		hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
400
-			  mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
401
-		bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
402
-		bt_l0_idx = mhop.l0_idx;
403
-		break;
404
-	case 2:
405
-		hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
406
-		bt_l0_idx = mhop.l0_idx;
407
-		break;
408
-	case 1:
409
-		hem_idx = mhop.l0_idx;
410
-		break;
411
-	default:
412
-		dev_err(dev, "Table %d not support hop_num = %d!\n",
413
-			     table->type, hop_num);
414
-		return -EINVAL;
415
-	}
416
-
417
-	if (unlikely(hem_idx >= table->num_hem)) {
418
-		dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n",
419
-			     table->type, hem_idx, table->num_hem);
420
-		return -EINVAL;
421606 	}
422607 
423608 	mutex_lock(&table->mutex);
424
-
425
-	if (table->hem[hem_idx]) {
426
-		++table->hem[hem_idx]->refcount;
609
+	if (table->hem[index.buf]) {
610
+		++table->hem[index.buf]->refcount;
427611 		goto out;
428612 	}
429613 
430
-	/* alloc L1 BA's chunk */
431
-	if ((check_whether_bt_num_3(table->type, hop_num) ||
432
-		check_whether_bt_num_2(table->type, hop_num)) &&
433
-		!table->bt_l0[bt_l0_idx]) {
434
-		table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
435
-					    &(table->bt_l0_dma_addr[bt_l0_idx]),
436
-					    GFP_KERNEL);
437
-		if (!table->bt_l0[bt_l0_idx]) {
438
-			ret = -ENOMEM;
439
-			goto out;
440
-		}
441
-		bt_l0_allocated = 1;
442
-
443
-		/* set base address to hardware */
444
-		if (table->type < HEM_TYPE_MTT) {
445
-			step_idx = 0;
446
-			if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
447
-				ret = -ENODEV;
448
-				dev_err(dev, "set HEM base address to HW failed!\n");
449
-				goto err_dma_alloc_l1;
450
-			}
451
-		}
614
+	ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
615
+	if (ret) {
616
+		ibdev_err(ibdev, "alloc mhop hem failed!\n");
617
+		goto out;
452618 	}
453619 
454
-	/* alloc L2 BA's chunk */
455
-	if (check_whether_bt_num_3(table->type, hop_num) &&
456
-	    !table->bt_l1[bt_l1_idx])  {
457
-		table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
458
-					    &(table->bt_l1_dma_addr[bt_l1_idx]),
459
-					    GFP_KERNEL);
460
-		if (!table->bt_l1[bt_l1_idx]) {
461
-			ret = -ENOMEM;
462
-			goto err_dma_alloc_l1;
463
-		}
464
-		bt_l1_allocated = 1;
465
-		*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
466
-					       table->bt_l1_dma_addr[bt_l1_idx];
467
-
468
-		/* set base address to hardware */
469
-		step_idx = 1;
470
-		if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
471
-			ret = -ENODEV;
472
-			dev_err(dev, "set HEM base address to HW failed!\n");
473
-			goto err_alloc_hem_buf;
474
-		}
475
-	}
476
-
477
-	/*
478
-	 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC.
479
-	 * alloc bt space chunk for MTT/CQE.
480
-	 */
481
-	size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
482
-	table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
483
-						size >> PAGE_SHIFT,
484
-						size,
485
-						(table->lowmem ? GFP_KERNEL :
486
-						GFP_HIGHUSER) | __GFP_NOWARN);
487
-	if (!table->hem[hem_idx]) {
488
-		ret = -ENOMEM;
489
-		goto err_alloc_hem_buf;
490
-	}
491
-
492
-	hns_roce_hem_first(table->hem[hem_idx], &iter);
493
-	bt_ba = hns_roce_hem_addr(&iter);
494
-
620
+	/* set HEM base address to hardware */
495621 	if (table->type < HEM_TYPE_MTT) {
496
-		if (hop_num == 2) {
497
-			*(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
498
-			step_idx = 2;
499
-		} else if (hop_num == 1) {
500
-			*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
501
-			step_idx = 1;
502
-		} else if (hop_num == HNS_ROCE_HOP_NUM_0) {
503
-			step_idx = 0;
504
-		} else {
505
-			ret = -EINVAL;
506
-			goto err_dma_alloc_l1;
622
+		ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
623
+		if (ret) {
624
+			ibdev_err(ibdev, "set HEM address to HW failed!\n");
625
+			goto err_alloc;
507626 		}
508
-
509
-		/* set HEM base address to hardware */
510
-		if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
511
-			ret = -ENODEV;
512
-			dev_err(dev, "set HEM base address to HW failed!\n");
513
-			goto err_alloc_hem_buf;
514
-		}
515
-	} else if (hop_num == 2) {
516
-		*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
517627 	}
518628 
519
-	++table->hem[hem_idx]->refcount;
629
+	++table->hem[index.buf]->refcount;
520630 	goto out;
521631 
522
-err_alloc_hem_buf:
523
-	if (bt_l1_allocated) {
524
-		dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
525
-				  table->bt_l1_dma_addr[bt_l1_idx]);
526
-		table->bt_l1[bt_l1_idx] = NULL;
527
-	}
528
-
529
-err_dma_alloc_l1:
530
-	if (bt_l0_allocated) {
531
-		dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
532
-				  table->bt_l0_dma_addr[bt_l0_idx]);
533
-		table->bt_l0[bt_l0_idx] = NULL;
534
-	}
535
-
632
+err_alloc:
633
+	free_mhop_hem(hr_dev, table, &mhop, &index);
536634 out:
537635 	mutex_unlock(&table->mutex);
538636 	return ret;
....@@ -583,116 +681,75 @@
583681 	return ret;
584682 }
585683 
684
+static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
685
+			   struct hns_roce_hem_table *table, unsigned long obj,
686
+			   struct hns_roce_hem_mhop *mhop,
687
+			   struct hns_roce_hem_index *index)
688
+{
689
+	struct ib_device *ibdev = &hr_dev->ib_dev;
690
+	u32 hop_num = mhop->hop_num;
691
+	u32 chunk_ba_num;
692
+	int step_idx;
693
+
694
+	index->inited = HEM_INDEX_BUF;
695
+	chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
696
+	if (check_whether_bt_num_2(table->type, hop_num)) {
697
+		if (hns_roce_check_hem_null(table->hem, index->buf,
698
+					    chunk_ba_num, table->num_hem))
699
+			index->inited |= HEM_INDEX_L0;
700
+	} else if (check_whether_bt_num_3(table->type, hop_num)) {
701
+		if (hns_roce_check_hem_null(table->hem, index->buf,
702
+					    chunk_ba_num, table->num_hem)) {
703
+			index->inited |= HEM_INDEX_L1;
704
+			if (hns_roce_check_bt_null(table->bt_l1, index->l1,
705
+						   chunk_ba_num))
706
+				index->inited |= HEM_INDEX_L0;
707
+		}
708
+	}
709
+
710
+	if (table->type < HEM_TYPE_MTT) {
711
+		if (hop_num == HNS_ROCE_HOP_NUM_0)
712
+			step_idx = 0;
713
+		else
714
+			step_idx = hop_num;
715
+
716
+		if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
717
+			ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num);
718
+
719
+		if (index->inited & HEM_INDEX_L1)
720
+			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
721
+				ibdev_warn(ibdev, "failed to clear HEM step 1.\n");
722
+
723
+		if (index->inited & HEM_INDEX_L0)
724
+			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
725
+				ibdev_warn(ibdev, "failed to clear HEM step 0.\n");
726
+	}
727
+}
728
+
586729 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
587730 				    struct hns_roce_hem_table *table,
588731 				    unsigned long obj,
589732 				    int check_refcount)
590733 {
591
-	struct device *dev = hr_dev->dev;
592
-	struct hns_roce_hem_mhop mhop;
593
-	unsigned long mhop_obj = obj;
594
-	u32 bt_chunk_size;
595
-	u32 chunk_ba_num;
596
-	u32 hop_num;
597
-	u32 start_idx;
598
-	u32 bt_num;
599
-	u64 hem_idx;
600
-	u64 bt_l1_idx = 0;
734
+	struct ib_device *ibdev = &hr_dev->ib_dev;
735
+	struct hns_roce_hem_index index = {};
736
+	struct hns_roce_hem_mhop mhop = {};
601737 	int ret;
602738 
603
-	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
604
-	if (ret)
605
-		return;
606
-
607
-	bt_chunk_size = mhop.bt_chunk_size;
608
-	hop_num = mhop.hop_num;
609
-	chunk_ba_num = bt_chunk_size / 8;
610
-
611
-	bt_num = hns_roce_get_bt_num(table->type, hop_num);
612
-	switch (bt_num) {
613
-	case 3:
614
-		hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
615
-			  mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
616
-		bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
617
-		break;
618
-	case 2:
619
-		hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
620
-		break;
621
-	case 1:
622
-		hem_idx = mhop.l0_idx;
623
-		break;
624
-	default:
625
-		dev_err(dev, "Table %d not support hop_num = %d!\n",
626
-			     table->type, hop_num);
739
+	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
740
+	if (ret) {
741
+		ibdev_err(ibdev, "calc hem config failed!\n");
627742 		return;
628743 	}
629744 
630745 	mutex_lock(&table->mutex);
631
-
632
-	if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
746
+	if (check_refcount && (--table->hem[index.buf]->refcount > 0)) {
633747 		mutex_unlock(&table->mutex);
634748 		return;
635749 	}
636750 
637
-	if (table->type < HEM_TYPE_MTT && hop_num == 1) {
638
-		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
639
-			dev_warn(dev, "Clear HEM base address failed.\n");
640
-	} else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
641
-		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
642
-			dev_warn(dev, "Clear HEM base address failed.\n");
643
-	} else if (table->type < HEM_TYPE_MTT &&
644
-		   hop_num == HNS_ROCE_HOP_NUM_0) {
645
-		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
646
-			dev_warn(dev, "Clear HEM base address failed.\n");
647
-	}
648
-
649
-	/*
650
-	 * free buffer space chunk for QPC/MTPT/CQC/SRQC.
651
-	 * free bt space chunk for MTT/CQE.
652
-	 */
653
-	hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
654
-	table->hem[hem_idx] = NULL;
655
-
656
-	if (check_whether_bt_num_2(table->type, hop_num)) {
657
-		start_idx = mhop.l0_idx * chunk_ba_num;
658
-		if (hns_roce_check_hem_null(table->hem, start_idx,
659
-					    chunk_ba_num, table->num_hem)) {
660
-			if (table->type < HEM_TYPE_MTT &&
661
-			    hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
662
-				dev_warn(dev, "Clear HEM base address failed.\n");
663
-
664
-			dma_free_coherent(dev, bt_chunk_size,
665
-					  table->bt_l0[mhop.l0_idx],
666
-					  table->bt_l0_dma_addr[mhop.l0_idx]);
667
-			table->bt_l0[mhop.l0_idx] = NULL;
668
-		}
669
-	} else if (check_whether_bt_num_3(table->type, hop_num)) {
670
-		start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
671
-			    mhop.l1_idx * chunk_ba_num;
672
-		if (hns_roce_check_hem_null(table->hem, start_idx,
673
-					    chunk_ba_num, table->num_hem)) {
674
-			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
675
-				dev_warn(dev, "Clear HEM base address failed.\n");
676
-
677
-			dma_free_coherent(dev, bt_chunk_size,
678
-					  table->bt_l1[bt_l1_idx],
679
-					  table->bt_l1_dma_addr[bt_l1_idx]);
680
-			table->bt_l1[bt_l1_idx] = NULL;
681
-
682
-			start_idx = mhop.l0_idx * chunk_ba_num;
683
-			if (hns_roce_check_bt_null(table->bt_l1, start_idx,
684
-						   chunk_ba_num)) {
685
-				if (hr_dev->hw->clear_hem(hr_dev, table, obj,
686
-							  0))
687
-					dev_warn(dev, "Clear HEM base address failed.\n");
688
-
689
-				dma_free_coherent(dev, bt_chunk_size,
690
-					    table->bt_l0[mhop.l0_idx],
691
-					    table->bt_l0_dma_addr[mhop.l0_idx]);
692
-				table->bt_l0[mhop.l0_idx] = NULL;
693
-			}
694
-		}
695
-	}
751
+	clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
752
+	free_mhop_hem(hr_dev, table, &mhop, &index);
696753 
697754 	mutex_unlock(&table->mutex);
698755 }
....@@ -754,12 +811,13 @@
754811 	} else {
755812 		u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
756813 
757
-		hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
814
+		if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
815
+			goto out;
758816 		/* mtt mhop */
759817 		i = mhop.l0_idx;
760818 		j = mhop.l1_idx;
761819 		if (mhop.hop_num == 2)
762
-			hem_idx = i * (mhop.bt_chunk_size / 8) + j;
820
+			hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
763821 		else if (mhop.hop_num == 1 ||
764822 			 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
765823 			hem_idx = i;
....@@ -796,62 +854,12 @@
796854 	mutex_unlock(&table->mutex);
797855 	return addr;
798856 }
799
-EXPORT_SYMBOL_GPL(hns_roce_table_find);
800
-
801
-int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
802
-			     struct hns_roce_hem_table *table,
803
-			     unsigned long start, unsigned long end)
804
-{
805
-	struct hns_roce_hem_mhop mhop;
806
-	unsigned long inc = table->table_chunk_size / table->obj_size;
807
-	unsigned long i;
808
-	int ret;
809
-
810
-	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
811
-		hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
812
-		inc = mhop.bt_chunk_size / table->obj_size;
813
-	}
814
-
815
-	/* Allocate MTT entry memory according to chunk(128K) */
816
-	for (i = start; i <= end; i += inc) {
817
-		ret = hns_roce_table_get(hr_dev, table, i);
818
-		if (ret)
819
-			goto fail;
820
-	}
821
-
822
-	return 0;
823
-
824
-fail:
825
-	while (i > start) {
826
-		i -= inc;
827
-		hns_roce_table_put(hr_dev, table, i);
828
-	}
829
-	return ret;
830
-}
831
-
832
-void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
833
-			      struct hns_roce_hem_table *table,
834
-			      unsigned long start, unsigned long end)
835
-{
836
-	struct hns_roce_hem_mhop mhop;
837
-	unsigned long inc = table->table_chunk_size / table->obj_size;
838
-	unsigned long i;
839
-
840
-	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
841
-		hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
842
-		inc = mhop.bt_chunk_size / table->obj_size;
843
-	}
844
-
845
-	for (i = start; i <= end; i += inc)
846
-		hns_roce_table_put(hr_dev, table, i);
847
-}
848857 
849858 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
850859 			    struct hns_roce_hem_table *table, u32 type,
851860 			    unsigned long obj_size, unsigned long nobj,
852861 			    int use_lowmem)
853862 {
854
-	struct device *dev = hr_dev->dev;
855863 	unsigned long obj_per_chunk;
856864 	unsigned long num_hem;
857865 
....@@ -864,66 +872,24 @@
864872 		if (!table->hem)
865873 			return -ENOMEM;
866874 	} else {
875
+		struct hns_roce_hem_mhop mhop = {};
867876 		unsigned long buf_chunk_size;
868877 		unsigned long bt_chunk_size;
869878 		unsigned long bt_chunk_num;
870879 		unsigned long num_bt_l0 = 0;
871880 		u32 hop_num;
872881 
873
-		switch (type) {
874
-		case HEM_TYPE_QPC:
875
-			buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
876
-					+ PAGE_SHIFT);
877
-			bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
878
-					+ PAGE_SHIFT);
879
-			num_bt_l0 = hr_dev->caps.qpc_bt_num;
880
-			hop_num = hr_dev->caps.qpc_hop_num;
881
-			break;
882
-		case HEM_TYPE_MTPT:
883
-			buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
884
-					+ PAGE_SHIFT);
885
-			bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
886
-					+ PAGE_SHIFT);
887
-			num_bt_l0 = hr_dev->caps.mpt_bt_num;
888
-			hop_num = hr_dev->caps.mpt_hop_num;
889
-			break;
890
-		case HEM_TYPE_CQC:
891
-			buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
892
-					+ PAGE_SHIFT);
893
-			bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
894
-					+ PAGE_SHIFT);
895
-			num_bt_l0 = hr_dev->caps.cqc_bt_num;
896
-			hop_num = hr_dev->caps.cqc_hop_num;
897
-			break;
898
-		case HEM_TYPE_SRQC:
899
-			buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
900
-					+ PAGE_SHIFT);
901
-			bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
902
-					+ PAGE_SHIFT);
903
-			num_bt_l0 = hr_dev->caps.srqc_bt_num;
904
-			hop_num = hr_dev->caps.srqc_hop_num;
905
-			break;
906
-		case HEM_TYPE_MTT:
907
-			buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
908
-					+ PAGE_SHIFT);
909
-			bt_chunk_size = buf_chunk_size;
910
-			hop_num = hr_dev->caps.mtt_hop_num;
911
-			break;
912
-		case HEM_TYPE_CQE:
913
-			buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
914
-					+ PAGE_SHIFT);
915
-			bt_chunk_size = buf_chunk_size;
916
-			hop_num = hr_dev->caps.cqe_hop_num;
917
-			break;
918
-		default:
919
-			dev_err(dev,
920
-			  "Table %d not support to init hem table here!\n",
921
-			  type);
882
+		if (get_hem_table_config(hr_dev, &mhop, type))
922883 			return -EINVAL;
923
-		}
884
+
885
+		buf_chunk_size = mhop.buf_chunk_size;
886
+		bt_chunk_size = mhop.bt_chunk_size;
887
+		num_bt_l0 = mhop.ba_l0_num;
888
+		hop_num = mhop.hop_num;
889
+
924890 		obj_per_chunk = buf_chunk_size / obj_size;
925891 		num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
926
-		bt_chunk_num = bt_chunk_size / 8;
892
+		bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
927893 		if (type >= HEM_TYPE_MTT)
928894 			num_bt_l0 = bt_chunk_num;
929895 
....@@ -1003,7 +969,8 @@
1003969 	int i;
1004970 	u64 obj;
1005971 
1006
-	hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
972
+	if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
973
+		return;
1007974 	buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
1008975 					mhop.bt_chunk_size;
1009976 
....@@ -1050,15 +1017,486 @@
10501017 
10511018 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
10521019 {
1020
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
1021
+		hns_roce_cleanup_hem_table(hr_dev,
1022
+					   &hr_dev->srq_table.table);
10531023 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1024
+	if (hr_dev->caps.qpc_timer_entry_sz)
1025
+		hns_roce_cleanup_hem_table(hr_dev,
1026
+					   &hr_dev->qpc_timer_table);
1027
+	if (hr_dev->caps.cqc_timer_entry_sz)
1028
+		hns_roce_cleanup_hem_table(hr_dev,
1029
+					   &hr_dev->cqc_timer_table);
1030
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
1031
+		hns_roce_cleanup_hem_table(hr_dev,
1032
+					   &hr_dev->qp_table.sccc_table);
10541033 	if (hr_dev->caps.trrl_entry_sz)
10551034 		hns_roce_cleanup_hem_table(hr_dev,
10561035 					   &hr_dev->qp_table.trrl_table);
10571036 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
10581037 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
10591038 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1060
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1061
-		hns_roce_cleanup_hem_table(hr_dev,
1062
-					   &hr_dev->mr_table.mtt_cqe_table);
1063
-	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1039
+}
1040
+
1041
+struct roce_hem_item {
1042
+	struct list_head list; /* link all hems in the same bt level */
1043
+	struct list_head sibling; /* link all hems in last hop for mtt */
1044
+	void *addr;
1045
+	dma_addr_t dma_addr;
1046
+	size_t count; /* max ba numbers */
1047
+	int start; /* start buf offset in this hem */
1048
+	int end; /* end buf offset in this hem */
1049
+};
1050
+
1051
+static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev,
1052
+						   int start, int end,
1053
+						   int count, bool exist_bt,
1054
+						   int bt_level)
1055
+{
1056
+	struct roce_hem_item *hem;
1057
+
1058
+	hem = kzalloc(sizeof(*hem), GFP_KERNEL);
1059
+	if (!hem)
1060
+		return NULL;
1061
+
1062
+	if (exist_bt) {
1063
+		hem->addr = dma_alloc_coherent(hr_dev->dev,
1064
+						   count * BA_BYTE_LEN,
1065
+						   &hem->dma_addr, GFP_KERNEL);
1066
+		if (!hem->addr) {
1067
+			kfree(hem);
1068
+			return NULL;
1069
+		}
1070
+	}
1071
+
1072
+	hem->count = count;
1073
+	hem->start = start;
1074
+	hem->end = end;
1075
+	INIT_LIST_HEAD(&hem->list);
1076
+	INIT_LIST_HEAD(&hem->sibling);
1077
+
1078
+	return hem;
1079
+}
1080
+
1081
+static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1082
+			       struct roce_hem_item *hem, bool exist_bt)
1083
+{
1084
+	if (exist_bt)
1085
+		dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1086
+				  hem->addr, hem->dma_addr);
1087
+	kfree(hem);
1088
+}
1089
+
1090
+static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1091
+			      struct list_head *head, bool exist_bt)
1092
+{
1093
+	struct roce_hem_item *hem, *temp_hem;
1094
+
1095
+	list_for_each_entry_safe(hem, temp_hem, head, list) {
1096
+		list_del(&hem->list);
1097
+		hem_list_free_item(hr_dev, hem, exist_bt);
1098
+	}
1099
+}
1100
+
1101
+static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1102
+			     u64 table_addr)
1103
+{
1104
+	*(u64 *)(base_addr) = table_addr;
1105
+}
1106
+
1107
+/* assign L0 table address to hem from root bt */
1108
+static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1109
+			       struct roce_hem_item *hem, void *cpu_addr,
1110
+			       u64 phy_addr)
1111
+{
1112
+	hem->addr = cpu_addr;
1113
+	hem->dma_addr = (dma_addr_t)phy_addr;
1114
+}
1115
+
1116
+static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem,
1117
+					     int offset)
1118
+{
1119
+	return (hem->start <= offset && offset <= hem->end);
1120
+}
1121
+
1122
+static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1123
+						    int page_offset)
1124
+{
1125
+	struct roce_hem_item *hem, *temp_hem;
1126
+	struct roce_hem_item *found = NULL;
1127
+
1128
+	list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1129
+		if (hem_list_page_is_in_range(hem, page_offset)) {
1130
+			found = hem;
1131
+			break;
1132
+		}
1133
+	}
1134
+
1135
+	return found;
1136
+}
1137
+
1138
+static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1139
+{
1140
+	/*
1141
+	 * hopnum    base address table levels
1142
+	 * 0		L0(buf)
1143
+	 * 1		L0 -> buf
1144
+	 * 2		L0 -> L1 -> buf
1145
+	 * 3		L0 -> L1 -> L2 -> buf
1146
+	 */
1147
+	return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1148
+}
1149
+
1150
+/**
1151
+ * calc base address entries num
1152
+ * @hopnum: num of mutihop addressing
1153
+ * @bt_level: base address table level
1154
+ * @unit: ba entries per bt page
1155
+ */
1156
+static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1157
+{
1158
+	u32 step;
1159
+	int max;
1160
+	int i;
1161
+
1162
+	if (hopnum <= bt_level)
1163
+		return 0;
1164
+	/*
1165
+	 * hopnum  bt_level   range
1166
+	 * 1	      0       unit
1167
+	 * ------------
1168
+	 * 2	      0       unit * unit
1169
+	 * 2	      1       unit
1170
+	 * ------------
1171
+	 * 3	      0       unit * unit * unit
1172
+	 * 3	      1       unit * unit
1173
+	 * 3	      2       unit
1174
+	 */
1175
+	step = 1;
1176
+	max = hopnum - bt_level;
1177
+	for (i = 0; i < max; i++)
1178
+		step = step * unit;
1179
+
1180
+	return step;
1181
+}
1182
+
1183
+/**
1184
+ * calc the root ba entries which could cover all regions
1185
+ * @regions: buf region array
1186
+ * @region_cnt: array size of @regions
1187
+ * @unit: ba entries per bt page
1188
+ */
1189
+int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1190
+				   int region_cnt, int unit)
1191
+{
1192
+	struct hns_roce_buf_region *r;
1193
+	int total = 0;
1194
+	int step;
1195
+	int i;
1196
+
1197
+	for (i = 0; i < region_cnt; i++) {
1198
+		r = (struct hns_roce_buf_region *)&regions[i];
1199
+		if (r->hopnum > 1) {
1200
+			step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1201
+			if (step > 0)
1202
+				total += (r->count + step - 1) / step;
1203
+		} else {
1204
+			total += r->count;
1205
+		}
1206
+	}
1207
+
1208
+	return total;
1209
+}
1210
+
1211
+static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1212
+				 const struct hns_roce_buf_region *r, int unit,
1213
+				 int offset, struct list_head *mid_bt,
1214
+				 struct list_head *btm_bt)
1215
+{
1216
+	struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1217
+	struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1218
+	struct roce_hem_item *cur, *pre;
1219
+	const int hopnum = r->hopnum;
1220
+	int start_aligned;
1221
+	int distance;
1222
+	int ret = 0;
1223
+	int max_ofs;
1224
+	int level;
1225
+	u32 step;
1226
+	int end;
1227
+
1228
+	if (hopnum <= 1)
1229
+		return 0;
1230
+
1231
+	if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1232
+		dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1233
+		return -EINVAL;
1234
+	}
1235
+
1236
+	if (offset < r->offset) {
1237
+		dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
1238
+			offset, r->offset);
1239
+		return -EINVAL;
1240
+	}
1241
+
1242
+	distance = offset - r->offset;
1243
+	max_ofs = r->offset + r->count - 1;
1244
+	for (level = 0; level < hopnum; level++)
1245
+		INIT_LIST_HEAD(&temp_list[level]);
1246
+
1247
+	/* config L1 bt to last bt and link them to corresponding parent */
1248
+	for (level = 1; level < hopnum; level++) {
1249
+		cur = hem_list_search_item(&mid_bt[level], offset);
1250
+		if (cur) {
1251
+			hem_ptrs[level] = cur;
1252
+			continue;
1253
+		}
1254
+
1255
+		step = hem_list_calc_ba_range(hopnum, level, unit);
1256
+		if (step < 1) {
1257
+			ret = -EINVAL;
1258
+			goto err_exit;
1259
+		}
1260
+
1261
+		start_aligned = (distance / step) * step + r->offset;
1262
+		end = min_t(int, start_aligned + step - 1, max_ofs);
1263
+		cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1264
+					  true, level);
1265
+		if (!cur) {
1266
+			ret = -ENOMEM;
1267
+			goto err_exit;
1268
+		}
1269
+		hem_ptrs[level] = cur;
1270
+		list_add(&cur->list, &temp_list[level]);
1271
+		if (hem_list_is_bottom_bt(hopnum, level))
1272
+			list_add(&cur->sibling, &temp_list[0]);
1273
+
1274
+		/* link bt to parent bt */
1275
+		if (level > 1) {
1276
+			pre = hem_ptrs[level - 1];
1277
+			step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1278
+			hem_list_link_bt(hr_dev, pre->addr + step,
1279
+					 cur->dma_addr);
1280
+		}
1281
+	}
1282
+
1283
+	list_splice(&temp_list[0], btm_bt);
1284
+	for (level = 1; level < hopnum; level++)
1285
+		list_splice(&temp_list[level], &mid_bt[level]);
1286
+
1287
+	return 0;
1288
+
1289
+err_exit:
1290
+	for (level = 1; level < hopnum; level++)
1291
+		hem_list_free_all(hr_dev, &temp_list[level], true);
1292
+
1293
+	return ret;
1294
+}
1295
+
1296
+static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1297
+				  struct hns_roce_hem_list *hem_list, int unit,
1298
+				  const struct hns_roce_buf_region *regions,
1299
+				  int region_cnt)
1300
+{
1301
+	struct roce_hem_item *hem, *temp_hem, *root_hem;
1302
+	struct list_head temp_list[HNS_ROCE_MAX_BT_REGION];
1303
+	const struct hns_roce_buf_region *r;
1304
+	struct list_head temp_root;
1305
+	struct list_head temp_btm;
1306
+	void *cpu_base;
1307
+	u64 phy_base;
1308
+	int ret = 0;
1309
+	int ba_num;
1310
+	int offset;
1311
+	int total;
1312
+	int step;
1313
+	int i;
1314
+
1315
+	r = &regions[0];
1316
+	root_hem = hem_list_search_item(&hem_list->root_bt, r->offset);
1317
+	if (root_hem)
1318
+		return 0;
1319
+
1320
+	ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1321
+	if (ba_num < 1)
1322
+		return -ENOMEM;
1323
+
1324
+	INIT_LIST_HEAD(&temp_root);
1325
+	offset = r->offset;
1326
+	/* indicate to last region */
1327
+	r = &regions[region_cnt - 1];
1328
+	root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1329
+				       ba_num, true, 0);
1330
+	if (!root_hem)
1331
+		return -ENOMEM;
1332
+	list_add(&root_hem->list, &temp_root);
1333
+
1334
+	hem_list->root_ba = root_hem->dma_addr;
1335
+
1336
+	INIT_LIST_HEAD(&temp_btm);
1337
+	for (i = 0; i < region_cnt; i++)
1338
+		INIT_LIST_HEAD(&temp_list[i]);
1339
+
1340
+	total = 0;
1341
+	for (i = 0; i < region_cnt && total < ba_num; i++) {
1342
+		r = &regions[i];
1343
+		if (!r->count)
1344
+			continue;
1345
+
1346
+		/* all regions's mid[x][0] shared the root_bt's trunk */
1347
+		cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1348
+		phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1349
+
1350
+		/* if hopnum is 0 or 1, cut a new fake hem from the root bt
1351
+		 * which's address share to all regions.
1352
+		 */
1353
+		if (hem_list_is_bottom_bt(r->hopnum, 0)) {
1354
+			hem = hem_list_alloc_item(hr_dev, r->offset,
1355
+						  r->offset + r->count - 1,
1356
+						  r->count, false, 0);
1357
+			if (!hem) {
1358
+				ret = -ENOMEM;
1359
+				goto err_exit;
1360
+			}
1361
+			hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1362
+			list_add(&hem->list, &temp_list[i]);
1363
+			list_add(&hem->sibling, &temp_btm);
1364
+			total += r->count;
1365
+		} else {
1366
+			step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1367
+			if (step < 1) {
1368
+				ret = -EINVAL;
1369
+				goto err_exit;
1370
+			}
1371
+			/* if exist mid bt, link L1 to L0 */
1372
+			list_for_each_entry_safe(hem, temp_hem,
1373
+					  &hem_list->mid_bt[i][1], list) {
1374
+				offset = (hem->start - r->offset) / step *
1375
+					  BA_BYTE_LEN;
1376
+				hem_list_link_bt(hr_dev, cpu_base + offset,
1377
+						 hem->dma_addr);
1378
+				total++;
1379
+			}
1380
+		}
1381
+	}
1382
+
1383
+	list_splice(&temp_btm, &hem_list->btm_bt);
1384
+	list_splice(&temp_root, &hem_list->root_bt);
1385
+	for (i = 0; i < region_cnt; i++)
1386
+		list_splice(&temp_list[i], &hem_list->mid_bt[i][0]);
1387
+
1388
+	return 0;
1389
+
1390
+err_exit:
1391
+	for (i = 0; i < region_cnt; i++)
1392
+		hem_list_free_all(hr_dev, &temp_list[i], false);
1393
+
1394
+	hem_list_free_all(hr_dev, &temp_root, true);
1395
+
1396
+	return ret;
1397
+}
1398
+
1399
+/* construct the base address table and link them by address hop config */
1400
+int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1401
+			      struct hns_roce_hem_list *hem_list,
1402
+			      const struct hns_roce_buf_region *regions,
1403
+			      int region_cnt, unsigned int bt_pg_shift)
1404
+{
1405
+	const struct hns_roce_buf_region *r;
1406
+	int ofs, end;
1407
+	int ret;
1408
+	int unit;
1409
+	int i;
1410
+
1411
+	if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1412
+		dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1413
+			region_cnt);
1414
+		return -EINVAL;
1415
+	}
1416
+
1417
+	unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
1418
+	for (i = 0; i < region_cnt; i++) {
1419
+		r = &regions[i];
1420
+		if (!r->count)
1421
+			continue;
1422
+
1423
+		end = r->offset + r->count;
1424
+		for (ofs = r->offset; ofs < end; ofs += unit) {
1425
+			ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1426
+						    hem_list->mid_bt[i],
1427
+						    &hem_list->btm_bt);
1428
+			if (ret) {
1429
+				dev_err(hr_dev->dev,
1430
+					"alloc hem trunk fail ret=%d!\n", ret);
1431
+				goto err_alloc;
1432
+			}
1433
+		}
1434
+	}
1435
+
1436
+	ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1437
+				     region_cnt);
1438
+	if (ret)
1439
+		dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1440
+	else
1441
+		return 0;
1442
+
1443
+err_alloc:
1444
+	hns_roce_hem_list_release(hr_dev, hem_list);
1445
+
1446
+	return ret;
1447
+}
1448
+
1449
+void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1450
+			       struct hns_roce_hem_list *hem_list)
1451
+{
1452
+	int i, j;
1453
+
1454
+	for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1455
+		for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1456
+			hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1457
+					  j != 0);
1458
+
1459
+	hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1460
+	INIT_LIST_HEAD(&hem_list->btm_bt);
1461
+	hem_list->root_ba = 0;
1462
+}
1463
+
1464
+void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
1465
+{
1466
+	int i, j;
1467
+
1468
+	INIT_LIST_HEAD(&hem_list->root_bt);
1469
+	INIT_LIST_HEAD(&hem_list->btm_bt);
1470
+	for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1471
+		for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1472
+			INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1473
+}
1474
+
1475
+void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1476
+				 struct hns_roce_hem_list *hem_list,
1477
+				 int offset, int *mtt_cnt, u64 *phy_addr)
1478
+{
1479
+	struct list_head *head = &hem_list->btm_bt;
1480
+	struct roce_hem_item *hem, *temp_hem;
1481
+	void *cpu_base = NULL;
1482
+	u64 phy_base = 0;
1483
+	int nr = 0;
1484
+
1485
+	list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1486
+		if (hem_list_page_is_in_range(hem, offset)) {
1487
+			nr = offset - hem->start;
1488
+			cpu_base = hem->addr + nr * BA_BYTE_LEN;
1489
+			phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1490
+			nr = hem->end + 1 - offset;
1491
+			break;
1492
+		}
1493
+	}
1494
+
1495
+	if (mtt_cnt)
1496
+		*mtt_cnt = nr;
1497
+
1498
+	if (phy_addr)
1499
+		*phy_addr = phy_base;
1500
+
1501
+	return cpu_base;
10641502 }