| .. | .. |
|---|
| 36 | 36 | #include "hns_roce_hem.h" |
|---|
| 37 | 37 | #include "hns_roce_common.h" |
|---|
| 38 | 38 | |
|---|
| 39 | | -#define DMA_ADDR_T_SHIFT 12 |
|---|
| 40 | | -#define BT_BA_SHIFT 32 |
|---|
| 41 | | - |
|---|
| 42 | 39 | #define HEM_INDEX_BUF BIT(0) |
|---|
| 43 | 40 | #define HEM_INDEX_L0 BIT(1) |
|---|
| 44 | 41 | #define HEM_INDEX_L1 BIT(2) |
|---|
| .. | .. |
|---|
| 198 | 195 | { |
|---|
| 199 | 196 | struct device *dev = hr_dev->dev; |
|---|
| 200 | 197 | u32 chunk_ba_num; |
|---|
| 198 | + u32 chunk_size; |
|---|
| 201 | 199 | u32 table_idx; |
|---|
| 202 | 200 | u32 bt_num; |
|---|
| 203 | | - u32 chunk_size; |
|---|
| 204 | 201 | |
|---|
| 205 | 202 | if (get_hem_table_config(hr_dev, mhop, table->type)) |
|---|
| 206 | 203 | return -EINVAL; |
|---|
| .. | .. |
|---|
| 260 | 257 | if (!hem) |
|---|
| 261 | 258 | return NULL; |
|---|
| 262 | 259 | |
|---|
| 263 | | - hem->refcount = 0; |
|---|
| 264 | 260 | INIT_LIST_HEAD(&hem->chunk_list); |
|---|
| 265 | 261 | |
|---|
| 266 | 262 | order = get_order(hem_alloc_size); |
|---|
| .. | .. |
|---|
| 325 | 321 | } |
|---|
| 326 | 322 | |
|---|
| 327 | 323 | kfree(hem); |
|---|
| 328 | | -} |
|---|
| 329 | | - |
|---|
| 330 | | -static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, |
|---|
| 331 | | - struct hns_roce_hem_table *table, unsigned long obj) |
|---|
| 332 | | -{ |
|---|
| 333 | | - spinlock_t *lock = &hr_dev->bt_cmd_lock; |
|---|
| 334 | | - struct device *dev = hr_dev->dev; |
|---|
| 335 | | - long end; |
|---|
| 336 | | - unsigned long flags; |
|---|
| 337 | | - struct hns_roce_hem_iter iter; |
|---|
| 338 | | - void __iomem *bt_cmd; |
|---|
| 339 | | - __le32 bt_cmd_val[2]; |
|---|
| 340 | | - __le32 bt_cmd_h = 0; |
|---|
| 341 | | - __le32 bt_cmd_l; |
|---|
| 342 | | - u64 bt_ba; |
|---|
| 343 | | - int ret = 0; |
|---|
| 344 | | - |
|---|
| 345 | | - /* Find the HEM(Hardware Entry Memory) entry */ |
|---|
| 346 | | - unsigned long i = (obj & (table->num_obj - 1)) / |
|---|
| 347 | | - (table->table_chunk_size / table->obj_size); |
|---|
| 348 | | - |
|---|
| 349 | | - switch (table->type) { |
|---|
| 350 | | - case HEM_TYPE_QPC: |
|---|
| 351 | | - case HEM_TYPE_MTPT: |
|---|
| 352 | | - case HEM_TYPE_CQC: |
|---|
| 353 | | - case HEM_TYPE_SRQC: |
|---|
| 354 | | - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, |
|---|
| 355 | | - ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type); |
|---|
| 356 | | - break; |
|---|
| 357 | | - default: |
|---|
| 358 | | - return ret; |
|---|
| 359 | | - } |
|---|
| 360 | | - |
|---|
| 361 | | - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, |
|---|
| 362 | | - ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); |
|---|
| 363 | | - roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); |
|---|
| 364 | | - roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); |
|---|
| 365 | | - |
|---|
| 366 | | - /* Currently iter only a chunk */ |
|---|
| 367 | | - for (hns_roce_hem_first(table->hem[i], &iter); |
|---|
| 368 | | - !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { |
|---|
| 369 | | - bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT; |
|---|
| 370 | | - |
|---|
| 371 | | - spin_lock_irqsave(lock, flags); |
|---|
| 372 | | - |
|---|
| 373 | | - bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; |
|---|
| 374 | | - |
|---|
| 375 | | - end = HW_SYNC_TIMEOUT_MSECS; |
|---|
| 376 | | - while (end > 0) { |
|---|
| 377 | | - if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT)) |
|---|
| 378 | | - break; |
|---|
| 379 | | - |
|---|
| 380 | | - mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); |
|---|
| 381 | | - end -= HW_SYNC_SLEEP_TIME_INTERVAL; |
|---|
| 382 | | - } |
|---|
| 383 | | - |
|---|
| 384 | | - if (end <= 0) { |
|---|
| 385 | | - dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); |
|---|
| 386 | | - spin_unlock_irqrestore(lock, flags); |
|---|
| 387 | | - return -EBUSY; |
|---|
| 388 | | - } |
|---|
| 389 | | - |
|---|
| 390 | | - bt_cmd_l = cpu_to_le32(bt_ba); |
|---|
| 391 | | - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, |
|---|
| 392 | | - ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, |
|---|
| 393 | | - bt_ba >> BT_BA_SHIFT); |
|---|
| 394 | | - |
|---|
| 395 | | - bt_cmd_val[0] = bt_cmd_l; |
|---|
| 396 | | - bt_cmd_val[1] = bt_cmd_h; |
|---|
| 397 | | - hns_roce_write64_k(bt_cmd_val, |
|---|
| 398 | | - hr_dev->reg_base + ROCEE_BT_CMD_L_REG); |
|---|
| 399 | | - spin_unlock_irqrestore(lock, flags); |
|---|
| 400 | | - } |
|---|
| 401 | | - |
|---|
| 402 | | - return ret; |
|---|
| 403 | 324 | } |
|---|
| 404 | 325 | |
|---|
| 405 | 326 | static int calc_hem_config(struct hns_roce_dev *hr_dev, |
|---|
| .. | .. |
|---|
| 607 | 528 | |
|---|
| 608 | 529 | mutex_lock(&table->mutex); |
|---|
| 609 | 530 | if (table->hem[index.buf]) { |
|---|
| 610 | | - ++table->hem[index.buf]->refcount; |
|---|
| 531 | + refcount_inc(&table->hem[index.buf]->refcount); |
|---|
| 611 | 532 | goto out; |
|---|
| 612 | 533 | } |
|---|
| 613 | 534 | |
|---|
| .. | .. |
|---|
| 626 | 547 | } |
|---|
| 627 | 548 | } |
|---|
| 628 | 549 | |
|---|
| 629 | | - ++table->hem[index.buf]->refcount; |
|---|
| 550 | + refcount_set(&table->hem[index.buf]->refcount, 1); |
|---|
| 630 | 551 | goto out; |
|---|
| 631 | 552 | |
|---|
| 632 | 553 | err_alloc: |
|---|
| .. | .. |
|---|
| 640 | 561 | struct hns_roce_hem_table *table, unsigned long obj) |
|---|
| 641 | 562 | { |
|---|
| 642 | 563 | struct device *dev = hr_dev->dev; |
|---|
| 643 | | - int ret = 0; |
|---|
| 644 | 564 | unsigned long i; |
|---|
| 565 | + int ret = 0; |
|---|
| 645 | 566 | |
|---|
| 646 | 567 | if (hns_roce_check_whether_mhop(hr_dev, table->type)) |
|---|
| 647 | 568 | return hns_roce_table_mhop_get(hr_dev, table, obj); |
|---|
| .. | .. |
|---|
| 652 | 573 | mutex_lock(&table->mutex); |
|---|
| 653 | 574 | |
|---|
| 654 | 575 | if (table->hem[i]) { |
|---|
| 655 | | - ++table->hem[i]->refcount; |
|---|
| 576 | + refcount_inc(&table->hem[i]->refcount); |
|---|
| 656 | 577 | goto out; |
|---|
| 657 | 578 | } |
|---|
| 658 | 579 | |
|---|
| .. | .. |
|---|
| 667 | 588 | } |
|---|
| 668 | 589 | |
|---|
| 669 | 590 | /* Set HEM base address(128K/page, pa) to Hardware */ |
|---|
| 670 | | - if (hns_roce_set_hem(hr_dev, table, obj)) { |
|---|
| 591 | + ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); |
|---|
| 592 | + if (ret) { |
|---|
| 671 | 593 | hns_roce_free_hem(hr_dev, table->hem[i]); |
|---|
| 672 | 594 | table->hem[i] = NULL; |
|---|
| 673 | | - ret = -ENODEV; |
|---|
| 674 | | - dev_err(dev, "set HEM base address to HW failed.\n"); |
|---|
| 595 | + dev_err(dev, "set HEM base address to HW failed, ret = %d.\n", |
|---|
| 596 | + ret); |
|---|
| 675 | 597 | goto out; |
|---|
| 676 | 598 | } |
|---|
| 677 | 599 | |
|---|
| 678 | | - ++table->hem[i]->refcount; |
|---|
| 600 | + refcount_set(&table->hem[i]->refcount, 1); |
|---|
| 679 | 601 | out: |
|---|
| 680 | 602 | mutex_unlock(&table->mutex); |
|---|
| 681 | 603 | return ret; |
|---|
| .. | .. |
|---|
| 742 | 664 | return; |
|---|
| 743 | 665 | } |
|---|
| 744 | 666 | |
|---|
| 745 | | - mutex_lock(&table->mutex); |
|---|
| 746 | | - if (check_refcount && (--table->hem[index.buf]->refcount > 0)) { |
|---|
| 747 | | - mutex_unlock(&table->mutex); |
|---|
| 667 | + if (!check_refcount) |
|---|
| 668 | + mutex_lock(&table->mutex); |
|---|
| 669 | + else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, |
|---|
| 670 | + &table->mutex)) |
|---|
| 748 | 671 | return; |
|---|
| 749 | | - } |
|---|
| 750 | 672 | |
|---|
| 751 | 673 | clear_mhop_hem(hr_dev, table, obj, &mhop, &index); |
|---|
| 752 | 674 | free_mhop_hem(hr_dev, table, &mhop, &index); |
|---|
| .. | .. |
|---|
| 768 | 690 | i = (obj & (table->num_obj - 1)) / |
|---|
| 769 | 691 | (table->table_chunk_size / table->obj_size); |
|---|
| 770 | 692 | |
|---|
| 771 | | - mutex_lock(&table->mutex); |
|---|
| 693 | + if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, |
|---|
| 694 | + &table->mutex)) |
|---|
| 695 | + return; |
|---|
| 772 | 696 | |
|---|
| 773 | | - if (--table->hem[i]->refcount == 0) { |
|---|
| 774 | | - /* Clear HEM base address */ |
|---|
| 775 | | - if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) |
|---|
| 776 | | - dev_warn(dev, "Clear HEM base address failed.\n"); |
|---|
| 697 | + if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) |
|---|
| 698 | + dev_warn(dev, "failed to clear HEM base address.\n"); |
|---|
| 777 | 699 | |
|---|
| 778 | | - hns_roce_free_hem(hr_dev, table->hem[i]); |
|---|
| 779 | | - table->hem[i] = NULL; |
|---|
| 780 | | - } |
|---|
| 700 | + hns_roce_free_hem(hr_dev, table->hem[i]); |
|---|
| 701 | + table->hem[i] = NULL; |
|---|
| 781 | 702 | |
|---|
| 782 | 703 | mutex_unlock(&table->mutex); |
|---|
| 783 | 704 | } |
|---|
| .. | .. |
|---|
| 789 | 710 | struct hns_roce_hem_chunk *chunk; |
|---|
| 790 | 711 | struct hns_roce_hem_mhop mhop; |
|---|
| 791 | 712 | struct hns_roce_hem *hem; |
|---|
| 792 | | - void *addr = NULL; |
|---|
| 793 | 713 | unsigned long mhop_obj = obj; |
|---|
| 794 | 714 | unsigned long obj_per_chunk; |
|---|
| 795 | 715 | unsigned long idx_offset; |
|---|
| 796 | 716 | int offset, dma_offset; |
|---|
| 717 | + void *addr = NULL; |
|---|
| 718 | + u32 hem_idx = 0; |
|---|
| 797 | 719 | int length; |
|---|
| 798 | 720 | int i, j; |
|---|
| 799 | | - u32 hem_idx = 0; |
|---|
| 800 | 721 | |
|---|
| 801 | 722 | if (!table->lowmem) |
|---|
| 802 | 723 | return NULL; |
|---|
| .. | .. |
|---|
| 966 | 887 | { |
|---|
| 967 | 888 | struct hns_roce_hem_mhop mhop; |
|---|
| 968 | 889 | u32 buf_chunk_size; |
|---|
| 969 | | - int i; |
|---|
| 970 | 890 | u64 obj; |
|---|
| 891 | + int i; |
|---|
| 971 | 892 | |
|---|
| 972 | 893 | if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) |
|---|
| 973 | 894 | return; |
|---|
| .. | .. |
|---|
| 1298 | 1219 | const struct hns_roce_buf_region *regions, |
|---|
| 1299 | 1220 | int region_cnt) |
|---|
| 1300 | 1221 | { |
|---|
| 1301 | | - struct roce_hem_item *hem, *temp_hem, *root_hem; |
|---|
| 1302 | 1222 | struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; |
|---|
| 1223 | + struct roce_hem_item *hem, *temp_hem, *root_hem; |
|---|
| 1303 | 1224 | const struct hns_roce_buf_region *r; |
|---|
| 1304 | 1225 | struct list_head temp_root; |
|---|
| 1305 | 1226 | struct list_head temp_btm; |
|---|
| .. | .. |
|---|
| 1404 | 1325 | { |
|---|
| 1405 | 1326 | const struct hns_roce_buf_region *r; |
|---|
| 1406 | 1327 | int ofs, end; |
|---|
| 1407 | | - int ret; |
|---|
| 1408 | 1328 | int unit; |
|---|
| 1329 | + int ret; |
|---|
| 1409 | 1330 | int i; |
|---|
| 1410 | 1331 | |
|---|
| 1411 | 1332 | if (region_cnt > HNS_ROCE_MAX_BT_REGION) { |
|---|