hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c
kernel/drivers/infiniband/hw/mthca/mthca_mr.c
....@@ -60,7 +60,7 @@
6060 __be64 mtt_seg;
6161 __be32 mtt_sz; /* Arbel only */
6262 u32 reserved[2];
63
-} __attribute__((packed));
63
+} __packed;
6464
6565 #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
6666 #define MTHCA_MPT_FLAG_MIO (1 << 17)
....@@ -541,7 +541,7 @@
541541 return err;
542542 }
543543
544
-/* Free mr or fmr */
544
+/* Free mr */
545545 static void mthca_free_region(struct mthca_dev *dev, u32 lkey)
546546 {
547547 mthca_table_put(dev, dev->mr_table.mpt_table,
....@@ -562,266 +562,6 @@
562562
563563 mthca_free_region(dev, mr->ibmr.lkey);
564564 mthca_free_mtt(dev, mr->mtt);
565
-}
566
-
567
-int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd,
568
- u32 access, struct mthca_fmr *mr)
569
-{
570
- struct mthca_mpt_entry *mpt_entry;
571
- struct mthca_mailbox *mailbox;
572
- u64 mtt_seg;
573
- u32 key, idx;
574
- int list_len = mr->attr.max_pages;
575
- int err = -ENOMEM;
576
- int i;
577
-
578
- if (mr->attr.page_shift < 12 || mr->attr.page_shift >= 32)
579
- return -EINVAL;
580
-
581
- /* For Arbel, all MTTs must fit in the same page. */
582
- if (mthca_is_memfree(dev) &&
583
- mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE)
584
- return -EINVAL;
585
-
586
- mr->maps = 0;
587
-
588
- key = mthca_alloc(&dev->mr_table.mpt_alloc);
589
- if (key == -1)
590
- return -ENOMEM;
591
- key = adjust_key(dev, key);
592
-
593
- idx = key & (dev->limits.num_mpts - 1);
594
- mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
595
-
596
- if (mthca_is_memfree(dev)) {
597
- err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
598
- if (err)
599
- goto err_out_mpt_free;
600
-
601
- mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key, NULL);
602
- BUG_ON(!mr->mem.arbel.mpt);
603
- } else
604
- mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base +
605
- sizeof *(mr->mem.tavor.mpt) * idx;
606
-
607
- mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy);
608
- if (IS_ERR(mr->mtt)) {
609
- err = PTR_ERR(mr->mtt);
610
- goto err_out_table;
611
- }
612
-
613
- mtt_seg = mr->mtt->first_seg * dev->limits.mtt_seg_size;
614
-
615
- if (mthca_is_memfree(dev)) {
616
- mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table,
617
- mr->mtt->first_seg,
618
- &mr->mem.arbel.dma_handle);
619
- BUG_ON(!mr->mem.arbel.mtts);
620
- } else
621
- mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg;
622
-
623
- mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
624
- if (IS_ERR(mailbox)) {
625
- err = PTR_ERR(mailbox);
626
- goto err_out_free_mtt;
627
- }
628
-
629
- mpt_entry = mailbox->buf;
630
-
631
- mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
632
- MTHCA_MPT_FLAG_MIO |
633
- MTHCA_MPT_FLAG_REGION |
634
- access);
635
-
636
- mpt_entry->page_size = cpu_to_be32(mr->attr.page_shift - 12);
637
- mpt_entry->key = cpu_to_be32(key);
638
- mpt_entry->pd = cpu_to_be32(pd);
639
- memset(&mpt_entry->start, 0,
640
- sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start));
641
- mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg);
642
-
643
- if (0) {
644
- mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
645
- for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
646
- if (i % 4 == 0)
647
- printk("[%02x] ", i * 4);
648
- printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i]));
649
- if ((i + 1) % 4 == 0)
650
- printk("\n");
651
- }
652
- }
653
-
654
- err = mthca_SW2HW_MPT(dev, mailbox,
655
- key & (dev->limits.num_mpts - 1));
656
- if (err) {
657
- mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
658
- goto err_out_mailbox_free;
659
- }
660
-
661
- mthca_free_mailbox(dev, mailbox);
662
- return 0;
663
-
664
-err_out_mailbox_free:
665
- mthca_free_mailbox(dev, mailbox);
666
-
667
-err_out_free_mtt:
668
- mthca_free_mtt(dev, mr->mtt);
669
-
670
-err_out_table:
671
- mthca_table_put(dev, dev->mr_table.mpt_table, key);
672
-
673
-err_out_mpt_free:
674
- mthca_free(&dev->mr_table.mpt_alloc, key);
675
- return err;
676
-}
677
-
678
-int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr)
679
-{
680
- if (fmr->maps)
681
- return -EBUSY;
682
-
683
- mthca_free_region(dev, fmr->ibmr.lkey);
684
- mthca_free_mtt(dev, fmr->mtt);
685
-
686
- return 0;
687
-}
688
-
689
-static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list,
690
- int list_len, u64 iova)
691
-{
692
- int i, page_mask;
693
-
694
- if (list_len > fmr->attr.max_pages)
695
- return -EINVAL;
696
-
697
- page_mask = (1 << fmr->attr.page_shift) - 1;
698
-
699
- /* We are getting page lists, so va must be page aligned. */
700
- if (iova & page_mask)
701
- return -EINVAL;
702
-
703
- /* Trust the user not to pass misaligned data in page_list */
704
- if (0)
705
- for (i = 0; i < list_len; ++i) {
706
- if (page_list[i] & ~page_mask)
707
- return -EINVAL;
708
- }
709
-
710
- if (fmr->maps >= fmr->attr.max_maps)
711
- return -EINVAL;
712
-
713
- return 0;
714
-}
715
-
716
-
717
-int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
718
- int list_len, u64 iova)
719
-{
720
- struct mthca_fmr *fmr = to_mfmr(ibfmr);
721
- struct mthca_dev *dev = to_mdev(ibfmr->device);
722
- struct mthca_mpt_entry mpt_entry;
723
- u32 key;
724
- int i, err;
725
-
726
- err = mthca_check_fmr(fmr, page_list, list_len, iova);
727
- if (err)
728
- return err;
729
-
730
- ++fmr->maps;
731
-
732
- key = tavor_key_to_hw_index(fmr->ibmr.lkey);
733
- key += dev->limits.num_mpts;
734
- fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key);
735
-
736
- writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
737
-
738
- for (i = 0; i < list_len; ++i) {
739
- __be64 mtt_entry = cpu_to_be64(page_list[i] |
740
- MTHCA_MTT_FLAG_PRESENT);
741
- mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i);
742
- }
743
-
744
- mpt_entry.lkey = cpu_to_be32(key);
745
- mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift));
746
- mpt_entry.start = cpu_to_be64(iova);
747
-
748
- __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key);
749
- memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start,
750
- offsetof(struct mthca_mpt_entry, window_count) -
751
- offsetof(struct mthca_mpt_entry, start));
752
-
753
- writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt);
754
-
755
- return 0;
756
-}
757
-
758
-int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
759
- int list_len, u64 iova)
760
-{
761
- struct mthca_fmr *fmr = to_mfmr(ibfmr);
762
- struct mthca_dev *dev = to_mdev(ibfmr->device);
763
- u32 key;
764
- int i, err;
765
-
766
- err = mthca_check_fmr(fmr, page_list, list_len, iova);
767
- if (err)
768
- return err;
769
-
770
- ++fmr->maps;
771
-
772
- key = arbel_key_to_hw_index(fmr->ibmr.lkey);
773
- if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
774
- key += SINAI_FMR_KEY_INC;
775
- else
776
- key += dev->limits.num_mpts;
777
- fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key);
778
-
779
- *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
780
-
781
- wmb();
782
-
783
- dma_sync_single_for_cpu(&dev->pdev->dev, fmr->mem.arbel.dma_handle,
784
- list_len * sizeof(u64), DMA_TO_DEVICE);
785
-
786
- for (i = 0; i < list_len; ++i)
787
- fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] |
788
- MTHCA_MTT_FLAG_PRESENT);
789
-
790
- dma_sync_single_for_device(&dev->pdev->dev, fmr->mem.arbel.dma_handle,
791
- list_len * sizeof(u64), DMA_TO_DEVICE);
792
-
793
- fmr->mem.arbel.mpt->key = cpu_to_be32(key);
794
- fmr->mem.arbel.mpt->lkey = cpu_to_be32(key);
795
- fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift));
796
- fmr->mem.arbel.mpt->start = cpu_to_be64(iova);
797
-
798
- wmb();
799
-
800
- *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW;
801
-
802
- wmb();
803
-
804
- return 0;
805
-}
806
-
807
-void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
808
-{
809
- if (!fmr->maps)
810
- return;
811
-
812
- fmr->maps = 0;
813
-
814
- writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
815
-}
816
-
817
-void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
818
-{
819
- if (!fmr->maps)
820
- return;
821
-
822
- fmr->maps = 0;
823
-
824
- *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
825565 }
826566
827567 int mthca_init_mr_table(struct mthca_dev *dev)