.. | .. |
---|
60 | 60 | __be64 mtt_seg; |
---|
61 | 61 | __be32 mtt_sz; /* Arbel only */ |
---|
62 | 62 | u32 reserved[2]; |
---|
63 | | -} __attribute__((packed)); |
---|
| 63 | +} __packed; |
---|
64 | 64 | |
---|
65 | 65 | #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28) |
---|
66 | 66 | #define MTHCA_MPT_FLAG_MIO (1 << 17) |
---|
.. | .. |
---|
541 | 541 | return err; |
---|
542 | 542 | } |
---|
543 | 543 | |
---|
544 | | -/* Free mr or fmr */ |
---|
| 544 | +/* Free mr */ |
---|
545 | 545 | static void mthca_free_region(struct mthca_dev *dev, u32 lkey) |
---|
546 | 546 | { |
---|
547 | 547 | mthca_table_put(dev, dev->mr_table.mpt_table, |
---|
.. | .. |
---|
562 | 562 | |
---|
563 | 563 | mthca_free_region(dev, mr->ibmr.lkey); |
---|
564 | 564 | mthca_free_mtt(dev, mr->mtt); |
---|
565 | | -} |
---|
566 | | - |
---|
567 | | -int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, |
---|
568 | | - u32 access, struct mthca_fmr *mr) |
---|
569 | | -{ |
---|
570 | | - struct mthca_mpt_entry *mpt_entry; |
---|
571 | | - struct mthca_mailbox *mailbox; |
---|
572 | | - u64 mtt_seg; |
---|
573 | | - u32 key, idx; |
---|
574 | | - int list_len = mr->attr.max_pages; |
---|
575 | | - int err = -ENOMEM; |
---|
576 | | - int i; |
---|
577 | | - |
---|
578 | | - if (mr->attr.page_shift < 12 || mr->attr.page_shift >= 32) |
---|
579 | | - return -EINVAL; |
---|
580 | | - |
---|
581 | | - /* For Arbel, all MTTs must fit in the same page. */ |
---|
582 | | - if (mthca_is_memfree(dev) && |
---|
583 | | - mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE) |
---|
584 | | - return -EINVAL; |
---|
585 | | - |
---|
586 | | - mr->maps = 0; |
---|
587 | | - |
---|
588 | | - key = mthca_alloc(&dev->mr_table.mpt_alloc); |
---|
589 | | - if (key == -1) |
---|
590 | | - return -ENOMEM; |
---|
591 | | - key = adjust_key(dev, key); |
---|
592 | | - |
---|
593 | | - idx = key & (dev->limits.num_mpts - 1); |
---|
594 | | - mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); |
---|
595 | | - |
---|
596 | | - if (mthca_is_memfree(dev)) { |
---|
597 | | - err = mthca_table_get(dev, dev->mr_table.mpt_table, key); |
---|
598 | | - if (err) |
---|
599 | | - goto err_out_mpt_free; |
---|
600 | | - |
---|
601 | | - mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key, NULL); |
---|
602 | | - BUG_ON(!mr->mem.arbel.mpt); |
---|
603 | | - } else |
---|
604 | | - mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base + |
---|
605 | | - sizeof *(mr->mem.tavor.mpt) * idx; |
---|
606 | | - |
---|
607 | | - mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy); |
---|
608 | | - if (IS_ERR(mr->mtt)) { |
---|
609 | | - err = PTR_ERR(mr->mtt); |
---|
610 | | - goto err_out_table; |
---|
611 | | - } |
---|
612 | | - |
---|
613 | | - mtt_seg = mr->mtt->first_seg * dev->limits.mtt_seg_size; |
---|
614 | | - |
---|
615 | | - if (mthca_is_memfree(dev)) { |
---|
616 | | - mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table, |
---|
617 | | - mr->mtt->first_seg, |
---|
618 | | - &mr->mem.arbel.dma_handle); |
---|
619 | | - BUG_ON(!mr->mem.arbel.mtts); |
---|
620 | | - } else |
---|
621 | | - mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg; |
---|
622 | | - |
---|
623 | | - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); |
---|
624 | | - if (IS_ERR(mailbox)) { |
---|
625 | | - err = PTR_ERR(mailbox); |
---|
626 | | - goto err_out_free_mtt; |
---|
627 | | - } |
---|
628 | | - |
---|
629 | | - mpt_entry = mailbox->buf; |
---|
630 | | - |
---|
631 | | - mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | |
---|
632 | | - MTHCA_MPT_FLAG_MIO | |
---|
633 | | - MTHCA_MPT_FLAG_REGION | |
---|
634 | | - access); |
---|
635 | | - |
---|
636 | | - mpt_entry->page_size = cpu_to_be32(mr->attr.page_shift - 12); |
---|
637 | | - mpt_entry->key = cpu_to_be32(key); |
---|
638 | | - mpt_entry->pd = cpu_to_be32(pd); |
---|
639 | | - memset(&mpt_entry->start, 0, |
---|
640 | | - sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start)); |
---|
641 | | - mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg); |
---|
642 | | - |
---|
643 | | - if (0) { |
---|
644 | | - mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); |
---|
645 | | - for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { |
---|
646 | | - if (i % 4 == 0) |
---|
647 | | - printk("[%02x] ", i * 4); |
---|
648 | | - printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); |
---|
649 | | - if ((i + 1) % 4 == 0) |
---|
650 | | - printk("\n"); |
---|
651 | | - } |
---|
652 | | - } |
---|
653 | | - |
---|
654 | | - err = mthca_SW2HW_MPT(dev, mailbox, |
---|
655 | | - key & (dev->limits.num_mpts - 1)); |
---|
656 | | - if (err) { |
---|
657 | | - mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); |
---|
658 | | - goto err_out_mailbox_free; |
---|
659 | | - } |
---|
660 | | - |
---|
661 | | - mthca_free_mailbox(dev, mailbox); |
---|
662 | | - return 0; |
---|
663 | | - |
---|
664 | | -err_out_mailbox_free: |
---|
665 | | - mthca_free_mailbox(dev, mailbox); |
---|
666 | | - |
---|
667 | | -err_out_free_mtt: |
---|
668 | | - mthca_free_mtt(dev, mr->mtt); |
---|
669 | | - |
---|
670 | | -err_out_table: |
---|
671 | | - mthca_table_put(dev, dev->mr_table.mpt_table, key); |
---|
672 | | - |
---|
673 | | -err_out_mpt_free: |
---|
674 | | - mthca_free(&dev->mr_table.mpt_alloc, key); |
---|
675 | | - return err; |
---|
676 | | -} |
---|
677 | | - |
---|
678 | | -int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr) |
---|
679 | | -{ |
---|
680 | | - if (fmr->maps) |
---|
681 | | - return -EBUSY; |
---|
682 | | - |
---|
683 | | - mthca_free_region(dev, fmr->ibmr.lkey); |
---|
684 | | - mthca_free_mtt(dev, fmr->mtt); |
---|
685 | | - |
---|
686 | | - return 0; |
---|
687 | | -} |
---|
688 | | - |
---|
689 | | -static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list, |
---|
690 | | - int list_len, u64 iova) |
---|
691 | | -{ |
---|
692 | | - int i, page_mask; |
---|
693 | | - |
---|
694 | | - if (list_len > fmr->attr.max_pages) |
---|
695 | | - return -EINVAL; |
---|
696 | | - |
---|
697 | | - page_mask = (1 << fmr->attr.page_shift) - 1; |
---|
698 | | - |
---|
699 | | - /* We are getting page lists, so va must be page aligned. */ |
---|
700 | | - if (iova & page_mask) |
---|
701 | | - return -EINVAL; |
---|
702 | | - |
---|
703 | | - /* Trust the user not to pass misaligned data in page_list */ |
---|
704 | | - if (0) |
---|
705 | | - for (i = 0; i < list_len; ++i) { |
---|
706 | | - if (page_list[i] & ~page_mask) |
---|
707 | | - return -EINVAL; |
---|
708 | | - } |
---|
709 | | - |
---|
710 | | - if (fmr->maps >= fmr->attr.max_maps) |
---|
711 | | - return -EINVAL; |
---|
712 | | - |
---|
713 | | - return 0; |
---|
714 | | -} |
---|
715 | | - |
---|
716 | | - |
---|
717 | | -int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, |
---|
718 | | - int list_len, u64 iova) |
---|
719 | | -{ |
---|
720 | | - struct mthca_fmr *fmr = to_mfmr(ibfmr); |
---|
721 | | - struct mthca_dev *dev = to_mdev(ibfmr->device); |
---|
722 | | - struct mthca_mpt_entry mpt_entry; |
---|
723 | | - u32 key; |
---|
724 | | - int i, err; |
---|
725 | | - |
---|
726 | | - err = mthca_check_fmr(fmr, page_list, list_len, iova); |
---|
727 | | - if (err) |
---|
728 | | - return err; |
---|
729 | | - |
---|
730 | | - ++fmr->maps; |
---|
731 | | - |
---|
732 | | - key = tavor_key_to_hw_index(fmr->ibmr.lkey); |
---|
733 | | - key += dev->limits.num_mpts; |
---|
734 | | - fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); |
---|
735 | | - |
---|
736 | | - writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); |
---|
737 | | - |
---|
738 | | - for (i = 0; i < list_len; ++i) { |
---|
739 | | - __be64 mtt_entry = cpu_to_be64(page_list[i] | |
---|
740 | | - MTHCA_MTT_FLAG_PRESENT); |
---|
741 | | - mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i); |
---|
742 | | - } |
---|
743 | | - |
---|
744 | | - mpt_entry.lkey = cpu_to_be32(key); |
---|
745 | | - mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift)); |
---|
746 | | - mpt_entry.start = cpu_to_be64(iova); |
---|
747 | | - |
---|
748 | | - __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key); |
---|
749 | | - memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start, |
---|
750 | | - offsetof(struct mthca_mpt_entry, window_count) - |
---|
751 | | - offsetof(struct mthca_mpt_entry, start)); |
---|
752 | | - |
---|
753 | | - writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt); |
---|
754 | | - |
---|
755 | | - return 0; |
---|
756 | | -} |
---|
757 | | - |
---|
758 | | -int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, |
---|
759 | | - int list_len, u64 iova) |
---|
760 | | -{ |
---|
761 | | - struct mthca_fmr *fmr = to_mfmr(ibfmr); |
---|
762 | | - struct mthca_dev *dev = to_mdev(ibfmr->device); |
---|
763 | | - u32 key; |
---|
764 | | - int i, err; |
---|
765 | | - |
---|
766 | | - err = mthca_check_fmr(fmr, page_list, list_len, iova); |
---|
767 | | - if (err) |
---|
768 | | - return err; |
---|
769 | | - |
---|
770 | | - ++fmr->maps; |
---|
771 | | - |
---|
772 | | - key = arbel_key_to_hw_index(fmr->ibmr.lkey); |
---|
773 | | - if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) |
---|
774 | | - key += SINAI_FMR_KEY_INC; |
---|
775 | | - else |
---|
776 | | - key += dev->limits.num_mpts; |
---|
777 | | - fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); |
---|
778 | | - |
---|
779 | | - *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; |
---|
780 | | - |
---|
781 | | - wmb(); |
---|
782 | | - |
---|
783 | | - dma_sync_single_for_cpu(&dev->pdev->dev, fmr->mem.arbel.dma_handle, |
---|
784 | | - list_len * sizeof(u64), DMA_TO_DEVICE); |
---|
785 | | - |
---|
786 | | - for (i = 0; i < list_len; ++i) |
---|
787 | | - fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] | |
---|
788 | | - MTHCA_MTT_FLAG_PRESENT); |
---|
789 | | - |
---|
790 | | - dma_sync_single_for_device(&dev->pdev->dev, fmr->mem.arbel.dma_handle, |
---|
791 | | - list_len * sizeof(u64), DMA_TO_DEVICE); |
---|
792 | | - |
---|
793 | | - fmr->mem.arbel.mpt->key = cpu_to_be32(key); |
---|
794 | | - fmr->mem.arbel.mpt->lkey = cpu_to_be32(key); |
---|
795 | | - fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift)); |
---|
796 | | - fmr->mem.arbel.mpt->start = cpu_to_be64(iova); |
---|
797 | | - |
---|
798 | | - wmb(); |
---|
799 | | - |
---|
800 | | - *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW; |
---|
801 | | - |
---|
802 | | - wmb(); |
---|
803 | | - |
---|
804 | | - return 0; |
---|
805 | | -} |
---|
806 | | - |
---|
807 | | -void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) |
---|
808 | | -{ |
---|
809 | | - if (!fmr->maps) |
---|
810 | | - return; |
---|
811 | | - |
---|
812 | | - fmr->maps = 0; |
---|
813 | | - |
---|
814 | | - writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); |
---|
815 | | -} |
---|
816 | | - |
---|
817 | | -void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) |
---|
818 | | -{ |
---|
819 | | - if (!fmr->maps) |
---|
820 | | - return; |
---|
821 | | - |
---|
822 | | - fmr->maps = 0; |
---|
823 | | - |
---|
824 | | - *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; |
---|
825 | 565 | } |
---|
826 | 566 | |
---|
827 | 567 | int mthca_init_mr_table(struct mthca_dev *dev) |
---|