update kernel to 5.10.198

hc

2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf

 kernel/drivers/infiniband/hw/hns/hns_roce_hem.c
..
..
@@ -36,42 +36,70 @@
36
36
 #include "hns_roce_hem.h"
37
37
 #include "hns_roce_common.h"
38
38
 
39
-#define DMA_ADDR_T_SHIFT		12
40
-#define BT_BA_SHIFT			32
39
+#define HEM_INDEX_BUF			BIT(0)
40
+#define HEM_INDEX_L0			BIT(1)
41
+#define HEM_INDEX_L1			BIT(2)
42
+struct hns_roce_hem_index {
43
+	u64 buf;
44
+	u64 l0;
45
+	u64 l1;
46
+	u32 inited; /* indicate which index is available */
47
+};
41
48
 
42
49
 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
43
50
 {
44
-	if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) ||
45
-	    (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
46
-	    (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
47
-	    (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
48
-	    (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
49
-	    (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT))
50
-		return true;
51
+	int hop_num = 0;
51
52
 
52
-	return false;
53
+	switch (type) {
54
+	case HEM_TYPE_QPC:
55
+		hop_num = hr_dev->caps.qpc_hop_num;
56
+		break;
57
+	case HEM_TYPE_MTPT:
58
+		hop_num = hr_dev->caps.mpt_hop_num;
59
+		break;
60
+	case HEM_TYPE_CQC:
61
+		hop_num = hr_dev->caps.cqc_hop_num;
62
+		break;
63
+	case HEM_TYPE_SRQC:
64
+		hop_num = hr_dev->caps.srqc_hop_num;
65
+		break;
66
+	case HEM_TYPE_SCCC:
67
+		hop_num = hr_dev->caps.sccc_hop_num;
68
+		break;
69
+	case HEM_TYPE_QPC_TIMER:
70
+		hop_num = hr_dev->caps.qpc_timer_hop_num;
71
+		break;
72
+	case HEM_TYPE_CQC_TIMER:
73
+		hop_num = hr_dev->caps.cqc_timer_hop_num;
74
+		break;
75
+	default:
76
+		return false;
77
+	}
78
+
79
+	return hop_num ? true : false;
53
80
 }
54
-EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);
55
81
 
56
-static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
57
-			    u32 bt_chunk_num, u64 hem_max_num)
82
+static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
83
+				    u32 bt_chunk_num, u64 hem_max_num)
58
84
 {
85
+	u64 start_idx = round_down(hem_idx, bt_chunk_num);
59
86
 	u64 check_max_num = start_idx + bt_chunk_num;
60
87
 	u64 i;
61
88
 
62
89
 	for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
63
-		if (hem[i])
90
+		if (i != hem_idx && hem[i])
64
91
 			return false;
65
92
 
66
93
 	return true;
67
94
 }
68
95
 
69
-static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
96
+static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
70
97
 {
98
+	u64 start_idx = round_down(ba_idx, bt_chunk_num);
71
99
 	int i;
72
100
 
73
101
 	for (i = 0; i < bt_chunk_num; i++)
74
-		if (bt[start_idx + i])
102
+		if (i != ba_idx && bt[start_idx + i])
75
103
 			return false;
76
104
 
77
105
 	return true;
..
..
@@ -89,17 +117,13 @@
89
117
 		return 0;
90
118
 }
91
119
 
92
-int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
93
-			   struct hns_roce_hem_table *table, unsigned long *obj,
94
-			   struct hns_roce_hem_mhop *mhop)
120
+static int get_hem_table_config(struct hns_roce_dev *hr_dev,
121
+				struct hns_roce_hem_mhop *mhop,
122
+				u32 type)
95
123
 {
96
124
 	struct device *dev = hr_dev->dev;
97
-	u32 chunk_ba_num;
98
-	u32 table_idx;
99
-	u32 bt_num;
100
-	u32 chunk_size;
101
125
 
102
-	switch (table->type) {
126
+	switch (type) {
103
127
 	case HEM_TYPE_QPC:
104
128
 		mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
105
129
 					     + PAGE_SHIFT);
..
..
@@ -124,6 +148,30 @@
124
148
 		mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
125
149
 		mhop->hop_num = hr_dev->caps.cqc_hop_num;
126
150
 		break;
151
+	case HEM_TYPE_SCCC:
152
+		mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
153
+					     + PAGE_SHIFT);
154
+		mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
155
+					    + PAGE_SHIFT);
156
+		mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
157
+		mhop->hop_num = hr_dev->caps.sccc_hop_num;
158
+		break;
159
+	case HEM_TYPE_QPC_TIMER:
160
+		mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
161
+					     + PAGE_SHIFT);
162
+		mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
163
+					    + PAGE_SHIFT);
164
+		mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
165
+		mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
166
+		break;
167
+	case HEM_TYPE_CQC_TIMER:
168
+		mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
169
+					     + PAGE_SHIFT);
170
+		mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
171
+					    + PAGE_SHIFT);
172
+		mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
173
+		mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
174
+		break;
127
175
 	case HEM_TYPE_SRQC:
128
176
 		mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
129
177
 					     + PAGE_SHIFT);
..
..
@@ -132,37 +180,37 @@
132
180
 		mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
133
181
 		mhop->hop_num = hr_dev->caps.srqc_hop_num;
134
182
 		break;
135
-	case HEM_TYPE_MTT:
136
-		mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
137
-					     + PAGE_SHIFT);
138
-		mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
139
-					     + PAGE_SHIFT);
140
-		mhop->ba_l0_num = mhop->bt_chunk_size / 8;
141
-		mhop->hop_num = hr_dev->caps.mtt_hop_num;
142
-		break;
143
-	case HEM_TYPE_CQE:
144
-		mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
145
-					     + PAGE_SHIFT);
146
-		mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
147
-					     + PAGE_SHIFT);
148
-		mhop->ba_l0_num = mhop->bt_chunk_size / 8;
149
-		mhop->hop_num = hr_dev->caps.cqe_hop_num;
150
-		break;
151
183
 	default:
152
-		dev_err(dev, "Table %d not support multi-hop addressing!\n",
153
-			 table->type);
184
+		dev_err(dev, "table %u not support multi-hop addressing!\n",
185
+			type);
154
186
 		return -EINVAL;
155
187
 	}
188
+
189
+	return 0;
190
+}
191
+
192
+int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
193
+			   struct hns_roce_hem_table *table, unsigned long *obj,
194
+			   struct hns_roce_hem_mhop *mhop)
195
+{
196
+	struct device *dev = hr_dev->dev;
197
+	u32 chunk_ba_num;
198
+	u32 chunk_size;
199
+	u32 table_idx;
200
+	u32 bt_num;
201
+
202
+	if (get_hem_table_config(hr_dev, mhop, table->type))
203
+		return -EINVAL;
156
204
 
157
205
 	if (!obj)
158
206
 		return 0;
159
207
 
160
208
 	/*
161
-	 * QPC/MTPT/CQC/SRQC alloc hem for buffer pages.
209
+	 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
162
210
 	 * MTT/CQE alloc hem for bt pages.
163
211
 	 */
164
212
 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
165
-	chunk_ba_num = mhop->bt_chunk_size / 8;
213
+	chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
166
214
 	chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
167
215
 			      mhop->bt_chunk_size;
168
216
 	table_idx = (*obj & (table->num_obj - 1)) /
..
..
@@ -181,8 +229,8 @@
181
229
 		mhop->l0_idx = table_idx;
182
230
 		break;
183
231
 	default:
184
-		dev_err(dev, "Table %d not support hop_num = %d!\n",
185
-			     table->type, mhop->hop_num);
232
+		dev_err(dev, "table %u not support hop_num = %u!\n",
233
+			table->type, mhop->hop_num);
186
234
 		return -EINVAL;
187
235
 	}
188
236
 	if (mhop->l0_idx >= mhop->ba_l0_num)
..
..
@@ -190,7 +238,6 @@
190
238
 
191
239
 	return 0;
192
240
 }
193
-EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);
194
241
 
195
242
 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
196
243
 					       int npages,
..
..
@@ -210,7 +257,6 @@
210
257
 	if (!hem)
211
258
 		return NULL;
212
259
 
213
-	hem->refcount = 0;
214
260
 	INIT_LIST_HEAD(&hem->chunk_list);
215
261
 
216
262
 	order = get_order(hem_alloc_size);
..
..
@@ -277,87 +323,191 @@
277
323
 	kfree(hem);
278
324
 }
279
325
 
280
-static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
281
-			    struct hns_roce_hem_table *table, unsigned long obj)
326
+static int calc_hem_config(struct hns_roce_dev *hr_dev,
327
+			   struct hns_roce_hem_table *table, unsigned long obj,
328
+			   struct hns_roce_hem_mhop *mhop,
329
+			   struct hns_roce_hem_index *index)
282
330
 {
283
-	spinlock_t *lock = &hr_dev->bt_cmd_lock;
284
-	struct device *dev = hr_dev->dev;
285
-	unsigned long end = 0;
286
-	unsigned long flags;
287
-	struct hns_roce_hem_iter iter;
288
-	void __iomem *bt_cmd;
289
-	u32 bt_cmd_h_val = 0;
290
-	u32 bt_cmd_val[2];
291
-	u32 bt_cmd_l = 0;
292
-	u64 bt_ba = 0;
293
-	int ret = 0;
331
+	struct ib_device *ibdev = &hr_dev->ib_dev;
332
+	unsigned long mhop_obj = obj;
333
+	u32 l0_idx, l1_idx, l2_idx;
334
+	u32 chunk_ba_num;
335
+	u32 bt_num;
336
+	int ret;
294
337
 
295
-	/* Find the HEM(Hardware Entry Memory) entry */
296
-	unsigned long i = (obj & (table->num_obj - 1)) /
297
-			  (table->table_chunk_size / table->obj_size);
338
+	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
339
+	if (ret)
340
+		return ret;
298
341
 
299
-	switch (table->type) {
300
-	case HEM_TYPE_QPC:
301
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
302
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
342
+	l0_idx = mhop->l0_idx;
343
+	l1_idx = mhop->l1_idx;
344
+	l2_idx = mhop->l2_idx;
345
+	chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
346
+	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
347
+	switch (bt_num) {
348
+	case 3:
349
+		index->l1 = l0_idx * chunk_ba_num + l1_idx;
350
+		index->l0 = l0_idx;
351
+		index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
352
+			     l1_idx * chunk_ba_num + l2_idx;
303
353
 		break;
304
-	case HEM_TYPE_MTPT:
305
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
306
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
307
-			       HEM_TYPE_MTPT);
354
+	case 2:
355
+		index->l0 = l0_idx;
356
+		index->buf = l0_idx * chunk_ba_num + l1_idx;
308
357
 		break;
309
-	case HEM_TYPE_CQC:
310
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
311
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
312
-		break;
313
-	case HEM_TYPE_SRQC:
314
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
315
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
316
-			       HEM_TYPE_SRQC);
358
+	case 1:
359
+		index->buf = l0_idx;
317
360
 		break;
318
361
 	default:
319
-		return ret;
362
+		ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
363
+			  table->type, mhop->hop_num);
364
+		return -EINVAL;
320
365
 	}
321
-	roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
322
-		       ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
323
-	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
324
-	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
325
366
 
326
-	/* Currently iter only a chunk */
327
-	for (hns_roce_hem_first(table->hem[i], &iter);
328
-	     !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
329
-		bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
367
+	if (unlikely(index->buf >= table->num_hem)) {
368
+		ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
369
+			  table->type, index->buf, table->num_hem);
370
+		return -EINVAL;
371
+	}
330
372
 
331
-		spin_lock_irqsave(lock, flags);
373
+	return 0;
374
+}
332
375
 
333
-		bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
376
+static void free_mhop_hem(struct hns_roce_dev *hr_dev,
377
+			  struct hns_roce_hem_table *table,
378
+			  struct hns_roce_hem_mhop *mhop,
379
+			  struct hns_roce_hem_index *index)
380
+{
381
+	u32 bt_size = mhop->bt_chunk_size;
382
+	struct device *dev = hr_dev->dev;
334
383
 
335
-		end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
336
-		while (1) {
337
-			if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
338
-				if (!(time_before(jiffies, end))) {
339
-					dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
340
-					spin_unlock_irqrestore(lock, flags);
341
-					return -EBUSY;
342
-				}
343
-			} else {
344
-				break;
345
-			}
346
-			mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
384
+	if (index->inited & HEM_INDEX_BUF) {
385
+		hns_roce_free_hem(hr_dev, table->hem[index->buf]);
386
+		table->hem[index->buf] = NULL;
387
+	}
388
+
389
+	if (index->inited & HEM_INDEX_L1) {
390
+		dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
391
+				  table->bt_l1_dma_addr[index->l1]);
392
+		table->bt_l1[index->l1] = NULL;
393
+	}
394
+
395
+	if (index->inited & HEM_INDEX_L0) {
396
+		dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
397
+				  table->bt_l0_dma_addr[index->l0]);
398
+		table->bt_l0[index->l0] = NULL;
399
+	}
400
+}
401
+
402
+static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
403
+			  struct hns_roce_hem_table *table,
404
+			  struct hns_roce_hem_mhop *mhop,
405
+			  struct hns_roce_hem_index *index)
406
+{
407
+	u32 bt_size = mhop->bt_chunk_size;
408
+	struct device *dev = hr_dev->dev;
409
+	struct hns_roce_hem_iter iter;
410
+	gfp_t flag;
411
+	u64 bt_ba;
412
+	u32 size;
413
+	int ret;
414
+
415
+	/* alloc L1 BA's chunk */
416
+	if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
417
+	     check_whether_bt_num_2(table->type, mhop->hop_num)) &&
418
+	     !table->bt_l0[index->l0]) {
419
+		table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
420
+					    &table->bt_l0_dma_addr[index->l0],
421
+					    GFP_KERNEL);
422
+		if (!table->bt_l0[index->l0]) {
423
+			ret = -ENOMEM;
424
+			goto out;
347
425
 		}
348
-
349
-		bt_cmd_l = (u32)bt_ba;
350
-		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
351
-			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
352
-			       bt_ba >> BT_BA_SHIFT);
353
-
354
-		bt_cmd_val[0] = bt_cmd_l;
355
-		bt_cmd_val[1] = bt_cmd_h_val;
356
-		hns_roce_write64_k(bt_cmd_val,
357
-				   hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
358
-		spin_unlock_irqrestore(lock, flags);
426
+		index->inited |= HEM_INDEX_L0;
359
427
 	}
360
428
 
429
+	/* alloc L2 BA's chunk */
430
+	if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
431
+	    !table->bt_l1[index->l1])  {
432
+		table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
433
+					    &table->bt_l1_dma_addr[index->l1],
434
+					    GFP_KERNEL);
435
+		if (!table->bt_l1[index->l1]) {
436
+			ret = -ENOMEM;
437
+			goto err_alloc_hem;
438
+		}
439
+		index->inited |= HEM_INDEX_L1;
440
+		*(table->bt_l0[index->l0] + mhop->l1_idx) =
441
+					       table->bt_l1_dma_addr[index->l1];
442
+	}
443
+
444
+	/*
445
+	 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
446
+	 * alloc bt space chunk for MTT/CQE.
447
+	 */
448
+	size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
449
+	flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;
450
+	table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
451
+						    size, flag);
452
+	if (!table->hem[index->buf]) {
453
+		ret = -ENOMEM;
454
+		goto err_alloc_hem;
455
+	}
456
+
457
+	index->inited |= HEM_INDEX_BUF;
458
+	hns_roce_hem_first(table->hem[index->buf], &iter);
459
+	bt_ba = hns_roce_hem_addr(&iter);
460
+	if (table->type < HEM_TYPE_MTT) {
461
+		if (mhop->hop_num == 2)
462
+			*(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
463
+		else if (mhop->hop_num == 1)
464
+			*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
465
+	} else if (mhop->hop_num == 2) {
466
+		*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
467
+	}
468
+
469
+	return 0;
470
+err_alloc_hem:
471
+	free_mhop_hem(hr_dev, table, mhop, index);
472
+out:
473
+	return ret;
474
+}
475
+
476
+static int set_mhop_hem(struct hns_roce_dev *hr_dev,
477
+			struct hns_roce_hem_table *table, unsigned long obj,
478
+			struct hns_roce_hem_mhop *mhop,
479
+			struct hns_roce_hem_index *index)
480
+{
481
+	struct ib_device *ibdev = &hr_dev->ib_dev;
482
+	int step_idx;
483
+	int ret = 0;
484
+
485
+	if (index->inited & HEM_INDEX_L0) {
486
+		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
487
+		if (ret) {
488
+			ibdev_err(ibdev, "set HEM step 0 failed!\n");
489
+			goto out;
490
+		}
491
+	}
492
+
493
+	if (index->inited & HEM_INDEX_L1) {
494
+		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
495
+		if (ret) {
496
+			ibdev_err(ibdev, "set HEM step 1 failed!\n");
497
+			goto out;
498
+		}
499
+	}
500
+
501
+	if (index->inited & HEM_INDEX_BUF) {
502
+		if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
503
+			step_idx = 0;
504
+		else
505
+			step_idx = mhop->hop_num;
506
+		ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
507
+		if (ret)
508
+			ibdev_err(ibdev, "set HEM step last failed!\n");
509
+	}
510
+out:
361
511
 	return ret;
362
512
 }
363
513
 
..
..
@@ -365,174 +515,43 @@
365
515
 				   struct hns_roce_hem_table *table,
366
516
 				   unsigned long obj)
367
517
 {
368
-	struct device *dev = hr_dev->dev;
369
-	struct hns_roce_hem_mhop mhop;
370
-	struct hns_roce_hem_iter iter;
371
-	u32 buf_chunk_size;
372
-	u32 bt_chunk_size;
373
-	u32 chunk_ba_num;
374
-	u32 hop_num;
375
-	u32 size;
376
-	u32 bt_num;
377
-	u64 hem_idx;
378
-	u64 bt_l1_idx = 0;
379
-	u64 bt_l0_idx = 0;
380
-	u64 bt_ba;
381
-	unsigned long mhop_obj = obj;
382
-	int bt_l1_allocated = 0;
383
-	int bt_l0_allocated = 0;
384
-	int step_idx;
518
+	struct ib_device *ibdev = &hr_dev->ib_dev;
519
+	struct hns_roce_hem_index index = {};
520
+	struct hns_roce_hem_mhop mhop = {};
385
521
 	int ret;
386
522
 
387
-	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
388
-	if (ret)
523
+	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
524
+	if (ret) {
525
+		ibdev_err(ibdev, "calc hem config failed!\n");
389
526
 		return ret;
390
-
391
-	buf_chunk_size = mhop.buf_chunk_size;
392
-	bt_chunk_size = mhop.bt_chunk_size;
393
-	hop_num = mhop.hop_num;
394
-	chunk_ba_num = bt_chunk_size / 8;
395
-
396
-	bt_num = hns_roce_get_bt_num(table->type, hop_num);
397
-	switch (bt_num) {
398
-	case 3:
399
-		hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
400
-			  mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
401
-		bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
402
-		bt_l0_idx = mhop.l0_idx;
403
-		break;
404
-	case 2:
405
-		hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
406
-		bt_l0_idx = mhop.l0_idx;
407
-		break;
408
-	case 1:
409
-		hem_idx = mhop.l0_idx;
410
-		break;
411
-	default:
412
-		dev_err(dev, "Table %d not support hop_num = %d!\n",
413
-			     table->type, hop_num);
414
-		return -EINVAL;
415
-	}
416
-
417
-	if (unlikely(hem_idx >= table->num_hem)) {
418
-		dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n",
419
-			     table->type, hem_idx, table->num_hem);
420
-		return -EINVAL;
421
527
 	}
422
528
 
423
529
 	mutex_lock(&table->mutex);
424
-
425
-	if (table->hem[hem_idx]) {
426
-		++table->hem[hem_idx]->refcount;
530
+	if (table->hem[index.buf]) {
531
+		refcount_inc(&table->hem[index.buf]->refcount);
427
532
 		goto out;
428
533
 	}
429
534
 
430
-	/* alloc L1 BA's chunk */
431
-	if ((check_whether_bt_num_3(table->type, hop_num) ||
432
-		check_whether_bt_num_2(table->type, hop_num)) &&
433
-		!table->bt_l0[bt_l0_idx]) {
434
-		table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
435
-					    &(table->bt_l0_dma_addr[bt_l0_idx]),
436
-					    GFP_KERNEL);
437
-		if (!table->bt_l0[bt_l0_idx]) {
438
-			ret = -ENOMEM;
439
-			goto out;
440
-		}
441
-		bt_l0_allocated = 1;
442
-
443
-		/* set base address to hardware */
444
-		if (table->type < HEM_TYPE_MTT) {
445
-			step_idx = 0;
446
-			if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
447
-				ret = -ENODEV;
448
-				dev_err(dev, "set HEM base address to HW failed!\n");
449
-				goto err_dma_alloc_l1;
450
-			}
451
-		}
535
+	ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
536
+	if (ret) {
537
+		ibdev_err(ibdev, "alloc mhop hem failed!\n");
538
+		goto out;
452
539
 	}
453
540
 
454
-	/* alloc L2 BA's chunk */
455
-	if (check_whether_bt_num_3(table->type, hop_num) &&
456
-	    !table->bt_l1[bt_l1_idx])  {
457
-		table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
458
-					    &(table->bt_l1_dma_addr[bt_l1_idx]),
459
-					    GFP_KERNEL);
460
-		if (!table->bt_l1[bt_l1_idx]) {
461
-			ret = -ENOMEM;
462
-			goto err_dma_alloc_l1;
463
-		}
464
-		bt_l1_allocated = 1;
465
-		*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
466
-					       table->bt_l1_dma_addr[bt_l1_idx];
467
-
468
-		/* set base address to hardware */
469
-		step_idx = 1;
470
-		if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
471
-			ret = -ENODEV;
472
-			dev_err(dev, "set HEM base address to HW failed!\n");
473
-			goto err_alloc_hem_buf;
474
-		}
475
-	}
476
-
477
-	/*
478
-	 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC.
479
-	 * alloc bt space chunk for MTT/CQE.
480
-	 */
481
-	size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
482
-	table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
483
-						size >> PAGE_SHIFT,
484
-						size,
485
-						(table->lowmem ? GFP_KERNEL :
486
-						GFP_HIGHUSER) | __GFP_NOWARN);
487
-	if (!table->hem[hem_idx]) {
488
-		ret = -ENOMEM;
489
-		goto err_alloc_hem_buf;
490
-	}
491
-
492
-	hns_roce_hem_first(table->hem[hem_idx], &iter);
493
-	bt_ba = hns_roce_hem_addr(&iter);
494
-
541
+	/* set HEM base address to hardware */
495
542
 	if (table->type < HEM_TYPE_MTT) {
496
-		if (hop_num == 2) {
497
-			*(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
498
-			step_idx = 2;
499
-		} else if (hop_num == 1) {
500
-			*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
501
-			step_idx = 1;
502
-		} else if (hop_num == HNS_ROCE_HOP_NUM_0) {
503
-			step_idx = 0;
504
-		} else {
505
-			ret = -EINVAL;
506
-			goto err_dma_alloc_l1;
543
+		ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
544
+		if (ret) {
545
+			ibdev_err(ibdev, "set HEM address to HW failed!\n");
546
+			goto err_alloc;
507
547
 		}
508
-
509
-		/* set HEM base address to hardware */
510
-		if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
511
-			ret = -ENODEV;
512
-			dev_err(dev, "set HEM base address to HW failed!\n");
513
-			goto err_alloc_hem_buf;
514
-		}
515
-	} else if (hop_num == 2) {
516
-		*(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
517
548
 	}
518
549
 
519
-	++table->hem[hem_idx]->refcount;
550
+	refcount_set(&table->hem[index.buf]->refcount, 1);
520
551
 	goto out;
521
552
 
522
-err_alloc_hem_buf:
523
-	if (bt_l1_allocated) {
524
-		dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
525
-				  table->bt_l1_dma_addr[bt_l1_idx]);
526
-		table->bt_l1[bt_l1_idx] = NULL;
527
-	}
528
-
529
-err_dma_alloc_l1:
530
-	if (bt_l0_allocated) {
531
-		dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
532
-				  table->bt_l0_dma_addr[bt_l0_idx]);
533
-		table->bt_l0[bt_l0_idx] = NULL;
534
-	}
535
-
553
+err_alloc:
554
+	free_mhop_hem(hr_dev, table, &mhop, &index);
536
555
 out:
537
556
 	mutex_unlock(&table->mutex);
538
557
 	return ret;
..
..
@@ -542,8 +561,8 @@
542
561
 		       struct hns_roce_hem_table *table, unsigned long obj)
543
562
 {
544
563
 	struct device *dev = hr_dev->dev;
545
-	int ret = 0;
546
564
 	unsigned long i;
565
+	int ret = 0;
547
566
 
548
567
 	if (hns_roce_check_whether_mhop(hr_dev, table->type))
549
568
 		return hns_roce_table_mhop_get(hr_dev, table, obj);
..
..
@@ -554,7 +573,7 @@
554
573
 	mutex_lock(&table->mutex);
555
574
 
556
575
 	if (table->hem[i]) {
557
-		++table->hem[i]->refcount;
576
+		refcount_inc(&table->hem[i]->refcount);
558
577
 		goto out;
559
578
 	}
560
579
 
..
..
@@ -569,18 +588,64 @@
569
588
 	}
570
589
 
571
590
 	/* Set HEM base address(128K/page, pa) to Hardware */
572
-	if (hns_roce_set_hem(hr_dev, table, obj)) {
591
+	ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);
592
+	if (ret) {
573
593
 		hns_roce_free_hem(hr_dev, table->hem[i]);
574
594
 		table->hem[i] = NULL;
575
-		ret = -ENODEV;
576
-		dev_err(dev, "set HEM base address to HW failed.\n");
595
+		dev_err(dev, "set HEM base address to HW failed, ret = %d.\n",
596
+			ret);
577
597
 		goto out;
578
598
 	}
579
599
 
580
-	++table->hem[i]->refcount;
600
+	refcount_set(&table->hem[i]->refcount, 1);
581
601
 out:
582
602
 	mutex_unlock(&table->mutex);
583
603
 	return ret;
604
+}
605
+
606
+static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
607
+			   struct hns_roce_hem_table *table, unsigned long obj,
608
+			   struct hns_roce_hem_mhop *mhop,
609
+			   struct hns_roce_hem_index *index)
610
+{
611
+	struct ib_device *ibdev = &hr_dev->ib_dev;
612
+	u32 hop_num = mhop->hop_num;
613
+	u32 chunk_ba_num;
614
+	int step_idx;
615
+
616
+	index->inited = HEM_INDEX_BUF;
617
+	chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
618
+	if (check_whether_bt_num_2(table->type, hop_num)) {
619
+		if (hns_roce_check_hem_null(table->hem, index->buf,
620
+					    chunk_ba_num, table->num_hem))
621
+			index->inited |= HEM_INDEX_L0;
622
+	} else if (check_whether_bt_num_3(table->type, hop_num)) {
623
+		if (hns_roce_check_hem_null(table->hem, index->buf,
624
+					    chunk_ba_num, table->num_hem)) {
625
+			index->inited |= HEM_INDEX_L1;
626
+			if (hns_roce_check_bt_null(table->bt_l1, index->l1,
627
+						   chunk_ba_num))
628
+				index->inited |= HEM_INDEX_L0;
629
+		}
630
+	}
631
+
632
+	if (table->type < HEM_TYPE_MTT) {
633
+		if (hop_num == HNS_ROCE_HOP_NUM_0)
634
+			step_idx = 0;
635
+		else
636
+			step_idx = hop_num;
637
+
638
+		if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
639
+			ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num);
640
+
641
+		if (index->inited & HEM_INDEX_L1)
642
+			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
643
+				ibdev_warn(ibdev, "failed to clear HEM step 1.\n");
644
+
645
+		if (index->inited & HEM_INDEX_L0)
646
+			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
647
+				ibdev_warn(ibdev, "failed to clear HEM step 0.\n");
648
+	}
584
649
 }
585
650
 
586
651
 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
..
..
@@ -588,111 +653,25 @@
588
653
 				    unsigned long obj,
589
654
 				    int check_refcount)
590
655
 {
591
-	struct device *dev = hr_dev->dev;
592
-	struct hns_roce_hem_mhop mhop;
593
-	unsigned long mhop_obj = obj;
594
-	u32 bt_chunk_size;
595
-	u32 chunk_ba_num;
596
-	u32 hop_num;
597
-	u32 start_idx;
598
-	u32 bt_num;
599
-	u64 hem_idx;
600
-	u64 bt_l1_idx = 0;
656
+	struct ib_device *ibdev = &hr_dev->ib_dev;
657
+	struct hns_roce_hem_index index = {};
658
+	struct hns_roce_hem_mhop mhop = {};
601
659
 	int ret;
602
660
 
603
-	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
604
-	if (ret)
605
-		return;
606
-
607
-	bt_chunk_size = mhop.bt_chunk_size;
608
-	hop_num = mhop.hop_num;
609
-	chunk_ba_num = bt_chunk_size / 8;
610
-
611
-	bt_num = hns_roce_get_bt_num(table->type, hop_num);
612
-	switch (bt_num) {
613
-	case 3:
614
-		hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
615
-			  mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
616
-		bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
617
-		break;
618
-	case 2:
619
-		hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
620
-		break;
621
-	case 1:
622
-		hem_idx = mhop.l0_idx;
623
-		break;
624
-	default:
625
-		dev_err(dev, "Table %d not support hop_num = %d!\n",
626
-			     table->type, hop_num);
661
+	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
662
+	if (ret) {
663
+		ibdev_err(ibdev, "calc hem config failed!\n");
627
664
 		return;
628
665
 	}
629
666
 
630
-	mutex_lock(&table->mutex);
631
-
632
-	if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
633
-		mutex_unlock(&table->mutex);
667
+	if (!check_refcount)
668
+		mutex_lock(&table->mutex);
669
+	else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount,
670
+					      &table->mutex))
634
671
 		return;
635
-	}
636
672
 
637
-	if (table->type < HEM_TYPE_MTT && hop_num == 1) {
638
-		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
639
-			dev_warn(dev, "Clear HEM base address failed.\n");
640
-	} else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
641
-		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
642
-			dev_warn(dev, "Clear HEM base address failed.\n");
643
-	} else if (table->type < HEM_TYPE_MTT &&
644
-		   hop_num == HNS_ROCE_HOP_NUM_0) {
645
-		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
646
-			dev_warn(dev, "Clear HEM base address failed.\n");
647
-	}
648
-
649
-	/*
650
-	 * free buffer space chunk for QPC/MTPT/CQC/SRQC.
651
-	 * free bt space chunk for MTT/CQE.
652
-	 */
653
-	hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
654
-	table->hem[hem_idx] = NULL;
655
-
656
-	if (check_whether_bt_num_2(table->type, hop_num)) {
657
-		start_idx = mhop.l0_idx * chunk_ba_num;
658
-		if (hns_roce_check_hem_null(table->hem, start_idx,
659
-					    chunk_ba_num, table->num_hem)) {
660
-			if (table->type < HEM_TYPE_MTT &&
661
-			    hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
662
-				dev_warn(dev, "Clear HEM base address failed.\n");
663
-
664
-			dma_free_coherent(dev, bt_chunk_size,
665
-					  table->bt_l0[mhop.l0_idx],
666
-					  table->bt_l0_dma_addr[mhop.l0_idx]);
667
-			table->bt_l0[mhop.l0_idx] = NULL;
668
-		}
669
-	} else if (check_whether_bt_num_3(table->type, hop_num)) {
670
-		start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
671
-			    mhop.l1_idx * chunk_ba_num;
672
-		if (hns_roce_check_hem_null(table->hem, start_idx,
673
-					    chunk_ba_num, table->num_hem)) {
674
-			if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
675
-				dev_warn(dev, "Clear HEM base address failed.\n");
676
-
677
-			dma_free_coherent(dev, bt_chunk_size,
678
-					  table->bt_l1[bt_l1_idx],
679
-					  table->bt_l1_dma_addr[bt_l1_idx]);
680
-			table->bt_l1[bt_l1_idx] = NULL;
681
-
682
-			start_idx = mhop.l0_idx * chunk_ba_num;
683
-			if (hns_roce_check_bt_null(table->bt_l1, start_idx,
684
-						   chunk_ba_num)) {
685
-				if (hr_dev->hw->clear_hem(hr_dev, table, obj,
686
-							  0))
687
-					dev_warn(dev, "Clear HEM base address failed.\n");
688
-
689
-				dma_free_coherent(dev, bt_chunk_size,
690
-					    table->bt_l0[mhop.l0_idx],
691
-					    table->bt_l0_dma_addr[mhop.l0_idx]);
692
-				table->bt_l0[mhop.l0_idx] = NULL;
693
-			}
694
-		}
695
-	}
673
+	clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
674
+	free_mhop_hem(hr_dev, table, &mhop, &index);
696
675
 
697
676
 	mutex_unlock(&table->mutex);
698
677
 }
..
..
@@ -711,16 +690,15 @@
711
690
 	i = (obj & (table->num_obj - 1)) /
712
691
 	    (table->table_chunk_size / table->obj_size);
713
692
 
714
-	mutex_lock(&table->mutex);
693
+	if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount,
694
+					 &table->mutex))
695
+		return;
715
696
 
716
-	if (--table->hem[i]->refcount == 0) {
717
-		/* Clear HEM base address */
718
-		if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
719
-			dev_warn(dev, "Clear HEM base address failed.\n");
697
+	if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT))
698
+		dev_warn(dev, "failed to clear HEM base address.\n");
720
699
 
721
-		hns_roce_free_hem(hr_dev, table->hem[i]);
722
-		table->hem[i] = NULL;
723
-	}
700
+	hns_roce_free_hem(hr_dev, table->hem[i]);
701
+	table->hem[i] = NULL;
724
702
 
725
703
 	mutex_unlock(&table->mutex);
726
704
 }
..
..
@@ -732,14 +710,14 @@
732
710
 	struct hns_roce_hem_chunk *chunk;
733
711
 	struct hns_roce_hem_mhop mhop;
734
712
 	struct hns_roce_hem *hem;
735
-	void *addr = NULL;
736
713
 	unsigned long mhop_obj = obj;
737
714
 	unsigned long obj_per_chunk;
738
715
 	unsigned long idx_offset;
739
716
 	int offset, dma_offset;
717
+	void *addr = NULL;
718
+	u32 hem_idx = 0;
740
719
 	int length;
741
720
 	int i, j;
742
-	u32 hem_idx = 0;
743
721
 
744
722
 	if (!table->lowmem)
745
723
 		return NULL;
..
..
@@ -754,12 +732,13 @@
754
732
 	} else {
755
733
 		u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
756
734
 
757
-		hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
735
+		if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
736
+			goto out;
758
737
 		/* mtt mhop */
759
738
 		i = mhop.l0_idx;
760
739
 		j = mhop.l1_idx;
761
740
 		if (mhop.hop_num == 2)
762
-			hem_idx = i * (mhop.bt_chunk_size / 8) + j;
741
+			hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
763
742
 		else if (mhop.hop_num == 1 ||
764
743
 			 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
765
744
 			hem_idx = i;
..
..
@@ -796,62 +775,12 @@
796
775
 	mutex_unlock(&table->mutex);
797
776
 	return addr;
798
777
 }
799
-EXPORT_SYMBOL_GPL(hns_roce_table_find);
800
-
801
-int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
802
-			     struct hns_roce_hem_table *table,
803
-			     unsigned long start, unsigned long end)
804
-{
805
-	struct hns_roce_hem_mhop mhop;
806
-	unsigned long inc = table->table_chunk_size / table->obj_size;
807
-	unsigned long i;
808
-	int ret;
809
-
810
-	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
811
-		hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
812
-		inc = mhop.bt_chunk_size / table->obj_size;
813
-	}
814
-
815
-	/* Allocate MTT entry memory according to chunk(128K) */
816
-	for (i = start; i <= end; i += inc) {
817
-		ret = hns_roce_table_get(hr_dev, table, i);
818
-		if (ret)
819
-			goto fail;
820
-	}
821
-
822
-	return 0;
823
-
824
-fail:
825
-	while (i > start) {
826
-		i -= inc;
827
-		hns_roce_table_put(hr_dev, table, i);
828
-	}
829
-	return ret;
830
-}
831
-
832
-void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
833
-			      struct hns_roce_hem_table *table,
834
-			      unsigned long start, unsigned long end)
835
-{
836
-	struct hns_roce_hem_mhop mhop;
837
-	unsigned long inc = table->table_chunk_size / table->obj_size;
838
-	unsigned long i;
839
-
840
-	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
841
-		hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
842
-		inc = mhop.bt_chunk_size / table->obj_size;
843
-	}
844
-
845
-	for (i = start; i <= end; i += inc)
846
-		hns_roce_table_put(hr_dev, table, i);
847
-}
848
778
 
849
779
 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
850
780
 			    struct hns_roce_hem_table *table, u32 type,
851
781
 			    unsigned long obj_size, unsigned long nobj,
852
782
 			    int use_lowmem)
853
783
 {
854
-	struct device *dev = hr_dev->dev;
855
784
 	unsigned long obj_per_chunk;
856
785
 	unsigned long num_hem;
857
786
 
..
..
@@ -864,66 +793,24 @@
864
793
 		if (!table->hem)
865
794
 			return -ENOMEM;
866
795
 	} else {
796
+		struct hns_roce_hem_mhop mhop = {};
867
797
 		unsigned long buf_chunk_size;
868
798
 		unsigned long bt_chunk_size;
869
799
 		unsigned long bt_chunk_num;
870
800
 		unsigned long num_bt_l0 = 0;
871
801
 		u32 hop_num;
872
802
 
873
-		switch (type) {
874
-		case HEM_TYPE_QPC:
875
-			buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
876
-					+ PAGE_SHIFT);
877
-			bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
878
-					+ PAGE_SHIFT);
879
-			num_bt_l0 = hr_dev->caps.qpc_bt_num;
880
-			hop_num = hr_dev->caps.qpc_hop_num;
881
-			break;
882
-		case HEM_TYPE_MTPT:
883
-			buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
884
-					+ PAGE_SHIFT);
885
-			bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
886
-					+ PAGE_SHIFT);
887
-			num_bt_l0 = hr_dev->caps.mpt_bt_num;
888
-			hop_num = hr_dev->caps.mpt_hop_num;
889
-			break;
890
-		case HEM_TYPE_CQC:
891
-			buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
892
-					+ PAGE_SHIFT);
893
-			bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
894
-					+ PAGE_SHIFT);
895
-			num_bt_l0 = hr_dev->caps.cqc_bt_num;
896
-			hop_num = hr_dev->caps.cqc_hop_num;
897
-			break;
898
-		case HEM_TYPE_SRQC:
899
-			buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
900
-					+ PAGE_SHIFT);
901
-			bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
902
-					+ PAGE_SHIFT);
903
-			num_bt_l0 = hr_dev->caps.srqc_bt_num;
904
-			hop_num = hr_dev->caps.srqc_hop_num;
905
-			break;
906
-		case HEM_TYPE_MTT:
907
-			buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
908
-					+ PAGE_SHIFT);
909
-			bt_chunk_size = buf_chunk_size;
910
-			hop_num = hr_dev->caps.mtt_hop_num;
911
-			break;
912
-		case HEM_TYPE_CQE:
913
-			buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
914
-					+ PAGE_SHIFT);
915
-			bt_chunk_size = buf_chunk_size;
916
-			hop_num = hr_dev->caps.cqe_hop_num;
917
-			break;
918
-		default:
919
-			dev_err(dev,
920
-			  "Table %d not support to init hem table here!\n",
921
-			  type);
803
+		if (get_hem_table_config(hr_dev, &mhop, type))
922
804
 			return -EINVAL;
923
-		}
805
+
806
+		buf_chunk_size = mhop.buf_chunk_size;
807
+		bt_chunk_size = mhop.bt_chunk_size;
808
+		num_bt_l0 = mhop.ba_l0_num;
809
+		hop_num = mhop.hop_num;
810
+
924
811
 		obj_per_chunk = buf_chunk_size / obj_size;
925
812
 		num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
926
-		bt_chunk_num = bt_chunk_size / 8;
813
+		bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
927
814
 		if (type >= HEM_TYPE_MTT)
928
815
 			num_bt_l0 = bt_chunk_num;
929
816
 
..
..
@@ -1000,10 +887,11 @@
1000
887
 {
1001
888
 	struct hns_roce_hem_mhop mhop;
1002
889
 	u32 buf_chunk_size;
1003
-	int i;
1004
890
 	u64 obj;
891
+	int i;
1005
892
 
1006
-	hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
893
+	if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
894
+		return;
1007
895
 	buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
1008
896
 					mhop.bt_chunk_size;
1009
897
 
..
..
@@ -1050,15 +938,486 @@
1050
938
 
1051
939
 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1052
940
 {
941
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
942
+		hns_roce_cleanup_hem_table(hr_dev,
943
+					   &hr_dev->srq_table.table);
1053
944
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
945
+	if (hr_dev->caps.qpc_timer_entry_sz)
946
+		hns_roce_cleanup_hem_table(hr_dev,
947
+					   &hr_dev->qpc_timer_table);
948
+	if (hr_dev->caps.cqc_timer_entry_sz)
949
+		hns_roce_cleanup_hem_table(hr_dev,
950
+					   &hr_dev->cqc_timer_table);
951
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
952
+		hns_roce_cleanup_hem_table(hr_dev,
953
+					   &hr_dev->qp_table.sccc_table);
1054
954
 	if (hr_dev->caps.trrl_entry_sz)
1055
955
 		hns_roce_cleanup_hem_table(hr_dev,
1056
956
 					   &hr_dev->qp_table.trrl_table);
1057
957
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1058
958
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1059
959
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1060
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1061
-		hns_roce_cleanup_hem_table(hr_dev,
1062
-					   &hr_dev->mr_table.mtt_cqe_table);
1063
-	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
960
+}
961
+
962
+struct roce_hem_item {
963
+	struct list_head list; /* link all hems in the same bt level */
964
+	struct list_head sibling; /* link all hems in last hop for mtt */
965
+	void *addr;
966
+	dma_addr_t dma_addr;
967
+	size_t count; /* max ba numbers */
968
+	int start; /* start buf offset in this hem */
969
+	int end; /* end buf offset in this hem */
970
+};
971
+
972
+static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev,
973
+						   int start, int end,
974
+						   int count, bool exist_bt,
975
+						   int bt_level)
976
+{
977
+	struct roce_hem_item *hem;
978
+
979
+	hem = kzalloc(sizeof(*hem), GFP_KERNEL);
980
+	if (!hem)
981
+		return NULL;
982
+
983
+	if (exist_bt) {
984
+		hem->addr = dma_alloc_coherent(hr_dev->dev,
985
+						   count * BA_BYTE_LEN,
986
+						   &hem->dma_addr, GFP_KERNEL);
987
+		if (!hem->addr) {
988
+			kfree(hem);
989
+			return NULL;
990
+		}
991
+	}
992
+
993
+	hem->count = count;
994
+	hem->start = start;
995
+	hem->end = end;
996
+	INIT_LIST_HEAD(&hem->list);
997
+	INIT_LIST_HEAD(&hem->sibling);
998
+
999
+	return hem;
1000
+}
1001
+
1002
+static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1003
+			       struct roce_hem_item *hem, bool exist_bt)
1004
+{
1005
+	if (exist_bt)
1006
+		dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1007
+				  hem->addr, hem->dma_addr);
1008
+	kfree(hem);
1009
+}
1010
+
1011
+static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1012
+			      struct list_head *head, bool exist_bt)
1013
+{
1014
+	struct roce_hem_item *hem, *temp_hem;
1015
+
1016
+	list_for_each_entry_safe(hem, temp_hem, head, list) {
1017
+		list_del(&hem->list);
1018
+		hem_list_free_item(hr_dev, hem, exist_bt);
1019
+	}
1020
+}
1021
+
1022
+static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1023
+			     u64 table_addr)
1024
+{
1025
+	*(u64 *)(base_addr) = table_addr;
1026
+}
1027
+
1028
+/* assign L0 table address to hem from root bt */
1029
+static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1030
+			       struct roce_hem_item *hem, void *cpu_addr,
1031
+			       u64 phy_addr)
1032
+{
1033
+	hem->addr = cpu_addr;
1034
+	hem->dma_addr = (dma_addr_t)phy_addr;
1035
+}
1036
+
1037
+static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem,
1038
+					     int offset)
1039
+{
1040
+	return (hem->start <= offset && offset <= hem->end);
1041
+}
1042
+
1043
+static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1044
+						    int page_offset)
1045
+{
1046
+	struct roce_hem_item *hem, *temp_hem;
1047
+	struct roce_hem_item *found = NULL;
1048
+
1049
+	list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1050
+		if (hem_list_page_is_in_range(hem, page_offset)) {
1051
+			found = hem;
1052
+			break;
1053
+		}
1054
+	}
1055
+
1056
+	return found;
1057
+}
1058
+
1059
+static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1060
+{
1061
+	/*
1062
+	 * hopnum    base address table levels
1063
+	 * 0		L0(buf)
1064
+	 * 1		L0 -> buf
1065
+	 * 2		L0 -> L1 -> buf
1066
+	 * 3		L0 -> L1 -> L2 -> buf
1067
+	 */
1068
+	return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1069
+}
1070
+
1071
+/**
1072
+ * calc base address entries num
1073
+ * @hopnum: num of mutihop addressing
1074
+ * @bt_level: base address table level
1075
+ * @unit: ba entries per bt page
1076
+ */
1077
+static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1078
+{
1079
+	u32 step;
1080
+	int max;
1081
+	int i;
1082
+
1083
+	if (hopnum <= bt_level)
1084
+		return 0;
1085
+	/*
1086
+	 * hopnum  bt_level   range
1087
+	 * 1	      0       unit
1088
+	 * ------------
1089
+	 * 2	      0       unit * unit
1090
+	 * 2	      1       unit
1091
+	 * ------------
1092
+	 * 3	      0       unit * unit * unit
1093
+	 * 3	      1       unit * unit
1094
+	 * 3	      2       unit
1095
+	 */
1096
+	step = 1;
1097
+	max = hopnum - bt_level;
1098
+	for (i = 0; i < max; i++)
1099
+		step = step * unit;
1100
+
1101
+	return step;
1102
+}
1103
+
1104
+/**
1105
+ * calc the root ba entries which could cover all regions
1106
+ * @regions: buf region array
1107
+ * @region_cnt: array size of @regions
1108
+ * @unit: ba entries per bt page
1109
+ */
1110
+int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1111
+				   int region_cnt, int unit)
1112
+{
1113
+	struct hns_roce_buf_region *r;
1114
+	int total = 0;
1115
+	int step;
1116
+	int i;
1117
+
1118
+	for (i = 0; i < region_cnt; i++) {
1119
+		r = (struct hns_roce_buf_region *)&regions[i];
1120
+		if (r->hopnum > 1) {
1121
+			step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1122
+			if (step > 0)
1123
+				total += (r->count + step - 1) / step;
1124
+		} else {
1125
+			total += r->count;
1126
+		}
1127
+	}
1128
+
1129
+	return total;
1130
+}
1131
+
1132
+static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1133
+				 const struct hns_roce_buf_region *r, int unit,
1134
+				 int offset, struct list_head *mid_bt,
1135
+				 struct list_head *btm_bt)
1136
+{
1137
+	struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1138
+	struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1139
+	struct roce_hem_item *cur, *pre;
1140
+	const int hopnum = r->hopnum;
1141
+	int start_aligned;
1142
+	int distance;
1143
+	int ret = 0;
1144
+	int max_ofs;
1145
+	int level;
1146
+	u32 step;
1147
+	int end;
1148
+
1149
+	if (hopnum <= 1)
1150
+		return 0;
1151
+
1152
+	if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1153
+		dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1154
+		return -EINVAL;
1155
+	}
1156
+
1157
+	if (offset < r->offset) {
1158
+		dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
1159
+			offset, r->offset);
1160
+		return -EINVAL;
1161
+	}
1162
+
1163
+	distance = offset - r->offset;
1164
+	max_ofs = r->offset + r->count - 1;
1165
+	for (level = 0; level < hopnum; level++)
1166
+		INIT_LIST_HEAD(&temp_list[level]);
1167
+
1168
+	/* config L1 bt to last bt and link them to corresponding parent */
1169
+	for (level = 1; level < hopnum; level++) {
1170
+		cur = hem_list_search_item(&mid_bt[level], offset);
1171
+		if (cur) {
1172
+			hem_ptrs[level] = cur;
1173
+			continue;
1174
+		}
1175
+
1176
+		step = hem_list_calc_ba_range(hopnum, level, unit);
1177
+		if (step < 1) {
1178
+			ret = -EINVAL;
1179
+			goto err_exit;
1180
+		}
1181
+
1182
+		start_aligned = (distance / step) * step + r->offset;
1183
+		end = min_t(int, start_aligned + step - 1, max_ofs);
1184
+		cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1185
+					  true, level);
1186
+		if (!cur) {
1187
+			ret = -ENOMEM;
1188
+			goto err_exit;
1189
+		}
1190
+		hem_ptrs[level] = cur;
1191
+		list_add(&cur->list, &temp_list[level]);
1192
+		if (hem_list_is_bottom_bt(hopnum, level))
1193
+			list_add(&cur->sibling, &temp_list[0]);
1194
+
1195
+		/* link bt to parent bt */
1196
+		if (level > 1) {
1197
+			pre = hem_ptrs[level - 1];
1198
+			step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1199
+			hem_list_link_bt(hr_dev, pre->addr + step,
1200
+					 cur->dma_addr);
1201
+		}
1202
+	}
1203
+
1204
+	list_splice(&temp_list[0], btm_bt);
1205
+	for (level = 1; level < hopnum; level++)
1206
+		list_splice(&temp_list[level], &mid_bt[level]);
1207
+
1208
+	return 0;
1209
+
1210
+err_exit:
1211
+	for (level = 1; level < hopnum; level++)
1212
+		hem_list_free_all(hr_dev, &temp_list[level], true);
1213
+
1214
+	return ret;
1215
+}
1216
+
1217
+static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1218
+				  struct hns_roce_hem_list *hem_list, int unit,
1219
+				  const struct hns_roce_buf_region *regions,
1220
+				  int region_cnt)
1221
+{
1222
+	struct list_head temp_list[HNS_ROCE_MAX_BT_REGION];
1223
+	struct roce_hem_item *hem, *temp_hem, *root_hem;
1224
+	const struct hns_roce_buf_region *r;
1225
+	struct list_head temp_root;
1226
+	struct list_head temp_btm;
1227
+	void *cpu_base;
1228
+	u64 phy_base;
1229
+	int ret = 0;
1230
+	int ba_num;
1231
+	int offset;
1232
+	int total;
1233
+	int step;
1234
+	int i;
1235
+
1236
+	r = &regions[0];
1237
+	root_hem = hem_list_search_item(&hem_list->root_bt, r->offset);
1238
+	if (root_hem)
1239
+		return 0;
1240
+
1241
+	ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1242
+	if (ba_num < 1)
1243
+		return -ENOMEM;
1244
+
1245
+	INIT_LIST_HEAD(&temp_root);
1246
+	offset = r->offset;
1247
+	/* indicate to last region */
1248
+	r = &regions[region_cnt - 1];
1249
+	root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1250
+				       ba_num, true, 0);
1251
+	if (!root_hem)
1252
+		return -ENOMEM;
1253
+	list_add(&root_hem->list, &temp_root);
1254
+
1255
+	hem_list->root_ba = root_hem->dma_addr;
1256
+
1257
+	INIT_LIST_HEAD(&temp_btm);
1258
+	for (i = 0; i < region_cnt; i++)
1259
+		INIT_LIST_HEAD(&temp_list[i]);
1260
+
1261
+	total = 0;
1262
+	for (i = 0; i < region_cnt && total < ba_num; i++) {
1263
+		r = &regions[i];
1264
+		if (!r->count)
1265
+			continue;
1266
+
1267
+		/* all regions's mid[x][0] shared the root_bt's trunk */
1268
+		cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1269
+		phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1270
+
1271
+		/* if hopnum is 0 or 1, cut a new fake hem from the root bt
1272
+		 * which's address share to all regions.
1273
+		 */
1274
+		if (hem_list_is_bottom_bt(r->hopnum, 0)) {
1275
+			hem = hem_list_alloc_item(hr_dev, r->offset,
1276
+						  r->offset + r->count - 1,
1277
+						  r->count, false, 0);
1278
+			if (!hem) {
1279
+				ret = -ENOMEM;
1280
+				goto err_exit;
1281
+			}
1282
+			hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1283
+			list_add(&hem->list, &temp_list[i]);
1284
+			list_add(&hem->sibling, &temp_btm);
1285
+			total += r->count;
1286
+		} else {
1287
+			step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1288
+			if (step < 1) {
1289
+				ret = -EINVAL;
1290
+				goto err_exit;
1291
+			}
1292
+			/* if exist mid bt, link L1 to L0 */
1293
+			list_for_each_entry_safe(hem, temp_hem,
1294
+					  &hem_list->mid_bt[i][1], list) {
1295
+				offset = (hem->start - r->offset) / step *
1296
+					  BA_BYTE_LEN;
1297
+				hem_list_link_bt(hr_dev, cpu_base + offset,
1298
+						 hem->dma_addr);
1299
+				total++;
1300
+			}
1301
+		}
1302
+	}
1303
+
1304
+	list_splice(&temp_btm, &hem_list->btm_bt);
1305
+	list_splice(&temp_root, &hem_list->root_bt);
1306
+	for (i = 0; i < region_cnt; i++)
1307
+		list_splice(&temp_list[i], &hem_list->mid_bt[i][0]);
1308
+
1309
+	return 0;
1310
+
1311
+err_exit:
1312
+	for (i = 0; i < region_cnt; i++)
1313
+		hem_list_free_all(hr_dev, &temp_list[i], false);
1314
+
1315
+	hem_list_free_all(hr_dev, &temp_root, true);
1316
+
1317
+	return ret;
1318
+}
1319
+
1320
+/* construct the base address table and link them by address hop config */
1321
+int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1322
+			      struct hns_roce_hem_list *hem_list,
1323
+			      const struct hns_roce_buf_region *regions,
1324
+			      int region_cnt, unsigned int bt_pg_shift)
1325
+{
1326
+	const struct hns_roce_buf_region *r;
1327
+	int ofs, end;
1328
+	int unit;
1329
+	int ret;
1330
+	int i;
1331
+
1332
+	if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1333
+		dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1334
+			region_cnt);
1335
+		return -EINVAL;
1336
+	}
1337
+
1338
+	unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
1339
+	for (i = 0; i < region_cnt; i++) {
1340
+		r = &regions[i];
1341
+		if (!r->count)
1342
+			continue;
1343
+
1344
+		end = r->offset + r->count;
1345
+		for (ofs = r->offset; ofs < end; ofs += unit) {
1346
+			ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1347
+						    hem_list->mid_bt[i],
1348
+						    &hem_list->btm_bt);
1349
+			if (ret) {
1350
+				dev_err(hr_dev->dev,
1351
+					"alloc hem trunk fail ret=%d!\n", ret);
1352
+				goto err_alloc;
1353
+			}
1354
+		}
1355
+	}
1356
+
1357
+	ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1358
+				     region_cnt);
1359
+	if (ret)
1360
+		dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1361
+	else
1362
+		return 0;
1363
+
1364
+err_alloc:
1365
+	hns_roce_hem_list_release(hr_dev, hem_list);
1366
+
1367
+	return ret;
1368
+}
1369
+
1370
+void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1371
+			       struct hns_roce_hem_list *hem_list)
1372
+{
1373
+	int i, j;
1374
+
1375
+	for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1376
+		for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1377
+			hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1378
+					  j != 0);
1379
+
1380
+	hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1381
+	INIT_LIST_HEAD(&hem_list->btm_bt);
1382
+	hem_list->root_ba = 0;
1383
+}
1384
+
1385
+void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
1386
+{
1387
+	int i, j;
1388
+
1389
+	INIT_LIST_HEAD(&hem_list->root_bt);
1390
+	INIT_LIST_HEAD(&hem_list->btm_bt);
1391
+	for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1392
+		for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1393
+			INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1394
+}
1395
+
1396
+void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1397
+				 struct hns_roce_hem_list *hem_list,
1398
+				 int offset, int *mtt_cnt, u64 *phy_addr)
1399
+{
1400
+	struct list_head *head = &hem_list->btm_bt;
1401
+	struct roce_hem_item *hem, *temp_hem;
1402
+	void *cpu_base = NULL;
1403
+	u64 phy_base = 0;
1404
+	int nr = 0;
1405
+
1406
+	list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1407
+		if (hem_list_page_is_in_range(hem, offset)) {
1408
+			nr = offset - hem->start;
1409
+			cpu_base = hem->addr + nr * BA_BYTE_LEN;
1410
+			phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1411
+			nr = hem->end + 1 - offset;
1412
+			break;
1413
+		}
1414
+	}
1415
+
1416
+	if (mtt_cnt)
1417
+		*mtt_cnt = nr;
1418
+
1419
+	if (phy_addr)
1420
+		*phy_addr = phy_base;
1421
+
1422
+	return cpu_base;
1064
1423
 }