forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/arch/powerpc/kernel/iommu.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
34  * 
....@@ -6,20 +7,6 @@
67  *               and  Ben. Herrenschmidt, IBM Corporation
78  *
89  * Dynamic DMA mapping support, bus-independent parts.
9
- *
10
- * This program is free software; you can redistribute it and/or modify
11
- * it under the terms of the GNU General Public License as published by
12
- * the Free Software Foundation; either version 2 of the License, or
13
- * (at your option) any later version.
14
- * 
15
- * This program is distributed in the hope that it will be useful,
16
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
17
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18
- * GNU General Public License for more details.
19
- * 
20
- * You should have received a copy of the GNU General Public License
21
- * along with this program; if not, write to the Free Software
22
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
2310  */
2411 
2512 
....@@ -47,6 +34,7 @@
4734 #include <asm/fadump.h>
4835 #include <asm/vio.h>
4936 #include <asm/tce.h>
37
+#include <asm/mmu_context.h>
5038 
5139 #define DBG(...)
5240 
....@@ -184,7 +172,6 @@
184172 	int largealloc = npages > 15;
185173 	int pass = 0;
186174 	unsigned long align_mask;
187
-	unsigned long boundary_size;
188175 	unsigned long flags;
189176 	unsigned int pool_nr;
190177 	struct iommu_pool *pool;
....@@ -197,11 +184,11 @@
197184 	if (unlikely(npages == 0)) {
198185 		if (printk_ratelimit())
199186 			WARN_ON(1);
200
-		return IOMMU_MAPPING_ERROR;
187
+		return DMA_MAPPING_ERROR;
201188 	}
202189 
203190 	if (should_fail_iommu(dev))
204
-		return IOMMU_MAPPING_ERROR;
191
+		return DMA_MAPPING_ERROR;
205192 
206193 	/*
207194 	 * We don't need to disable preemption here because any CPU can
....@@ -248,15 +235,9 @@
248235 		}
249236 	}
250237 
251
-	if (dev)
252
-		boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
253
-				      1 << tbl->it_page_shift);
254
-	else
255
-		boundary_size = ALIGN(1UL << 32, 1 << tbl->it_page_shift);
256
-	/* 4GB boundary for iseries_hv_alloc and iseries_hv_map */
257
-
258238 	n = iommu_area_alloc(tbl->it_map, limit, start, npages, tbl->it_offset,
259
-			     boundary_size >> tbl->it_page_shift, align_mask);
239
+			dma_get_seg_boundary_nr_pages(dev, tbl->it_page_shift),
240
+			align_mask);
260241 	if (n == -1) {
261242 		if (likely(pass == 0)) {
262243 			/* First try the pool from the start */
....@@ -277,7 +258,7 @@
277258 		} else {
278259 			/* Give up */
279260 			spin_unlock_irqrestore(&(pool->lock), flags);
280
-			return IOMMU_MAPPING_ERROR;
261
+			return DMA_MAPPING_ERROR;
281262 		}
282263 	}
283264 
....@@ -309,13 +290,13 @@
309290 			      unsigned long attrs)
310291 {
311292 	unsigned long entry;
312
-	dma_addr_t ret = IOMMU_MAPPING_ERROR;
293
+	dma_addr_t ret = DMA_MAPPING_ERROR;
313294 	int build_fail;
314295 
315296 	entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order);
316297 
317
-	if (unlikely(entry == IOMMU_MAPPING_ERROR))
318
-		return IOMMU_MAPPING_ERROR;
298
+	if (unlikely(entry == DMA_MAPPING_ERROR))
299
+		return DMA_MAPPING_ERROR;
319300 
320301 	entry += tbl->it_offset;	/* Offset into real TCE table */
321302 	ret = entry << tbl->it_page_shift;	/* Set the return dma address */
....@@ -327,12 +308,12 @@
327308 
328309 	/* tbl->it_ops->set() only returns non-zero for transient errors.
329310 	 * Clean up the table bitmap in this case and return
330
-	 * IOMMU_MAPPING_ERROR. For all other errors the functionality is
311
+	 * DMA_MAPPING_ERROR. For all other errors the functionality is
331312 	 * not altered.
332313 	 */
333314 	if (unlikely(build_fail)) {
334315 		__iommu_free(tbl, ret, npages);
335
-		return IOMMU_MAPPING_ERROR;
316
+		return DMA_MAPPING_ERROR;
336317 	}
337318 
338319 	/* Flush/invalidate TLB caches if necessary */
....@@ -477,7 +458,7 @@
477458 		DBG("  - vaddr: %lx, size: %lx\n", vaddr, slen);
478459 
479460 		/* Handle failure */
480
-		if (unlikely(entry == IOMMU_MAPPING_ERROR)) {
461
+		if (unlikely(entry == DMA_MAPPING_ERROR)) {
481462 			if (!(attrs & DMA_ATTR_NO_WARN) &&
482463 			    printk_ratelimit())
483464 				dev_info(dev, "iommu_alloc failed, tbl %p "
....@@ -544,7 +525,7 @@
544525 	 */
545526 	if (outcount < incount) {
546527 		outs = sg_next(outs);
547
-		outs->dma_address = IOMMU_MAPPING_ERROR;
528
+		outs->dma_address = DMA_MAPPING_ERROR;
548529 		outs->dma_length = 0;
549530 	}
550531 
....@@ -562,7 +543,7 @@
562543 			npages = iommu_num_pages(s->dma_address, s->dma_length,
563544 						 IOMMU_PAGE_SIZE(tbl));
564545 			__iommu_free(tbl, vaddr, npages);
565
-			s->dma_address = IOMMU_MAPPING_ERROR;
546
+			s->dma_address = DMA_MAPPING_ERROR;
566547 			s->dma_length = 0;
567548 		}
568549 		if (s == outs)
....@@ -645,11 +626,54 @@
645626 #endif
646627 }
647628 
629
+static void iommu_table_reserve_pages(struct iommu_table *tbl,
630
+		unsigned long res_start, unsigned long res_end)
631
+{
632
+	int i;
633
+
634
+	WARN_ON_ONCE(res_end < res_start);
635
+	/*
636
+	 * Reserve page 0 so it will not be used for any mappings.
637
+	 * This avoids buggy drivers that consider page 0 to be invalid
638
+	 * to crash the machine or even lose data.
639
+	 */
640
+	if (tbl->it_offset == 0)
641
+		set_bit(0, tbl->it_map);
642
+
643
+	tbl->it_reserved_start = res_start;
644
+	tbl->it_reserved_end = res_end;
645
+
646
+	/* Check if res_start..res_end isn't empty and overlaps the table */
647
+	if (res_start && res_end &&
648
+			(tbl->it_offset + tbl->it_size < res_start ||
649
+			 res_end < tbl->it_offset))
650
+		return;
651
+
652
+	for (i = tbl->it_reserved_start; i < tbl->it_reserved_end; ++i)
653
+		set_bit(i - tbl->it_offset, tbl->it_map);
654
+}
655
+
656
+static void iommu_table_release_pages(struct iommu_table *tbl)
657
+{
658
+	int i;
659
+
660
+	/*
661
+	 * In case we have reserved the first bit, we should not emit
662
+	 * the warning below.
663
+	 */
664
+	if (tbl->it_offset == 0)
665
+		clear_bit(0, tbl->it_map);
666
+
667
+	for (i = tbl->it_reserved_start; i < tbl->it_reserved_end; ++i)
668
+		clear_bit(i - tbl->it_offset, tbl->it_map);
669
+}
670
+
648671 /*
649672  * Build a iommu_table structure.  This contains a bit map which
650673  * is used to manage allocation of the tce space.
651674  */
652
-struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid)
675
+struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid,
676
+		unsigned long res_start, unsigned long res_end)
653677 {
654678 	unsigned long sz;
655679 	static int welcomed = 0;
....@@ -668,13 +692,7 @@
668692 	tbl->it_map = page_address(page);
669693 	memset(tbl->it_map, 0, sz);
670694 
671
-	/*
672
-	 * Reserve page 0 so it will not be used for any mappings.
673
-	 * This avoids buggy drivers that consider page 0 to be invalid
674
-	 * to crash the machine or even lose data.
675
-	 */
676
-	if (tbl->it_offset == 0)
677
-		set_bit(0, tbl->it_map);
695
+	iommu_table_reserve_pages(tbl, res_start, res_end);
678696 
679697 	/* We only split the IOMMU table if we have 1GB or more of space */
680698 	if ((tbl->it_size << tbl->it_page_shift) >= (1UL * 1024 * 1024 * 1024))
....@@ -726,12 +744,7 @@
726744 		return;
727745 	}
728746 
729
-	/*
730
-	 * In case we have reserved the first bit, we should not emit
731
-	 * the warning below.
732
-	 */
733
-	if (tbl->it_offset == 0)
734
-		clear_bit(0, tbl->it_map);
747
+	iommu_table_release_pages(tbl);
735748 
736749 	/* verify that table contains no entries */
737750 	if (!bitmap_empty(tbl->it_map, tbl->it_size))
....@@ -776,7 +789,7 @@
776789 			  unsigned long mask, enum dma_data_direction direction,
777790 			  unsigned long attrs)
778791 {
779
-	dma_addr_t dma_handle = IOMMU_MAPPING_ERROR;
792
+	dma_addr_t dma_handle = DMA_MAPPING_ERROR;
780793 	void *vaddr;
781794 	unsigned long uaddr;
782795 	unsigned int npages, align;
....@@ -796,7 +809,7 @@
796809 		dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction,
797810 					 mask >> tbl->it_page_shift, align,
798811 					 attrs);
799
-		if (dma_handle == IOMMU_MAPPING_ERROR) {
812
+		if (dma_handle == DMA_MAPPING_ERROR) {
800813 			if (!(attrs & DMA_ATTR_NO_WARN) &&
801814 			    printk_ratelimit())  {
802815 				dev_info(dev, "iommu_alloc failed, tbl %p "
....@@ -868,7 +881,7 @@
868881 	io_order = get_iommu_order(size, tbl);
869882 	mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
870883 			      mask >> tbl->it_page_shift, io_order, 0);
871
-	if (mapping == IOMMU_MAPPING_ERROR) {
884
+	if (mapping == DMA_MAPPING_ERROR) {
872885 		free_pages((unsigned long)ret, order);
873886 		return NULL;
874887 	}
....@@ -993,25 +1006,32 @@
9931006 }
9941007 EXPORT_SYMBOL_GPL(iommu_tce_check_gpa);
9951008 
996
-long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry,
997
-		unsigned long *hpa, enum dma_data_direction *direction)
1009
+extern long iommu_tce_xchg_no_kill(struct mm_struct *mm,
1010
+		struct iommu_table *tbl,
1011
+		unsigned long entry, unsigned long *hpa,
1012
+		enum dma_data_direction *direction)
9981013 {
9991014 	long ret;
1015
+	unsigned long size = 0;
10001016 
1001
-	ret = tbl->it_ops->exchange(tbl, entry, hpa, direction);
1002
-
1017
+	ret = tbl->it_ops->xchg_no_kill(tbl, entry, hpa, direction, false);
10031018 	if (!ret && ((*direction == DMA_FROM_DEVICE) ||
1004
-			(*direction == DMA_BIDIRECTIONAL)))
1019
+			(*direction == DMA_BIDIRECTIONAL)) &&
1020
+			!mm_iommu_is_devmem(mm, *hpa, tbl->it_page_shift,
1021
+					&size))
10051022 		SetPageDirty(pfn_to_page(*hpa >> PAGE_SHIFT));
1006
-
1007
-	/* if (unlikely(ret))
1008
-		pr_err("iommu_tce: %s failed on hwaddr=%lx ioba=%lx kva=%lx ret=%d\n",
1009
-			__func__, hwaddr, entry << tbl->it_page_shift,
1010
-				hwaddr, ret); */
10111023 
10121024 	return ret;
10131025 }
1014
-EXPORT_SYMBOL_GPL(iommu_tce_xchg);
1026
+EXPORT_SYMBOL_GPL(iommu_tce_xchg_no_kill);
1027
+
1028
+void iommu_tce_kill(struct iommu_table *tbl,
1029
+		unsigned long entry, unsigned long pages)
1030
+{
1031
+	if (tbl->it_ops->tce_kill)
1032
+		tbl->it_ops->tce_kill(tbl, entry, pages, false);
1033
+}
1034
+EXPORT_SYMBOL_GPL(iommu_tce_kill);
10151035 
10161036 int iommu_take_ownership(struct iommu_table *tbl)
10171037 {
....@@ -1025,22 +1045,21 @@
10251045 	 * requires exchange() callback defined so if it is not
10261046 	 * implemented, we disallow taking ownership over the table.
10271047 	 */
1028
-	if (!tbl->it_ops->exchange)
1048
+	if (!tbl->it_ops->xchg_no_kill)
10291049 		return -EINVAL;
10301050 
10311051 	spin_lock_irqsave(&tbl->large_pool.lock, flags);
10321052 	for (i = 0; i < tbl->nr_pools; i++)
10331053 		spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock);
10341054 
1035
-	if (tbl->it_offset == 0)
1036
-		clear_bit(0, tbl->it_map);
1055
+	iommu_table_release_pages(tbl);
10371056 
10381057 	if (!bitmap_empty(tbl->it_map, tbl->it_size)) {
10391058 		pr_err("iommu_tce: it_map is not empty");
10401059 		ret = -EBUSY;
1041
-		/* Restore bit#0 set by iommu_init_table() */
1042
-		if (tbl->it_offset == 0)
1043
-			set_bit(0, tbl->it_map);
1060
+		/* Undo iommu_table_release_pages, i.e. restore bit#0, etc */
1061
+		iommu_table_reserve_pages(tbl, tbl->it_reserved_start,
1062
+				tbl->it_reserved_end);
10441063 	} else {
10451064 		memset(tbl->it_map, 0xff, sz);
10461065 	}
....@@ -1063,9 +1082,8 @@
10631082 
10641083 	memset(tbl->it_map, 0, sz);
10651084 
1066
-	/* Restore bit#0 set by iommu_init_table() */
1067
-	if (tbl->it_offset == 0)
1068
-		set_bit(0, tbl->it_map);
1085
+	iommu_table_reserve_pages(tbl, tbl->it_reserved_start,
1086
+			tbl->it_reserved_end);
10691087 
10701088 	for (i = 0; i < tbl->nr_pools; i++)
10711089 		spin_unlock(&tbl->pools[i].lock);
....@@ -1073,11 +1091,8 @@
10731091 }
10741092 EXPORT_SYMBOL_GPL(iommu_release_ownership);
10751093 
1076
-int iommu_add_device(struct device *dev)
1094
+int iommu_add_device(struct iommu_table_group *table_group, struct device *dev)
10771095 {
1078
-	struct iommu_table *tbl;
1079
-	struct iommu_table_group_link *tgl;
1080
-
10811096 	/*
10821097 	 * The sysfs entries should be populated before
10831098 	 * binding IOMMU group. If sysfs entries isn't
....@@ -1086,39 +1101,17 @@
10861101 	if (!device_is_registered(dev))
10871102 		return -ENOENT;
10881103 
1089
-	if (dev->iommu_group) {
1104
+	if (device_iommu_mapped(dev)) {
10901105 		pr_debug("%s: Skipping device %s with iommu group %d\n",
10911106 			 __func__, dev_name(dev),
10921107 			 iommu_group_id(dev->iommu_group));
10931108 		return -EBUSY;
10941109 	}
10951110 
1096
-	tbl = get_iommu_table_base(dev);
1097
-	if (!tbl) {
1098
-		pr_debug("%s: Skipping device %s with no tbl\n",
1099
-			 __func__, dev_name(dev));
1100
-		return 0;
1101
-	}
1102
-
1103
-	tgl = list_first_entry_or_null(&tbl->it_group_list,
1104
-			struct iommu_table_group_link, next);
1105
-	if (!tgl) {
1106
-		pr_debug("%s: Skipping device %s with no group\n",
1107
-			 __func__, dev_name(dev));
1108
-		return 0;
1109
-	}
11101111 	pr_debug("%s: Adding %s to iommu group %d\n",
1111
-		 __func__, dev_name(dev),
1112
-		 iommu_group_id(tgl->table_group->group));
1112
+		 __func__, dev_name(dev),  iommu_group_id(table_group->group));
11131113 
1114
-	if (PAGE_SIZE < IOMMU_PAGE_SIZE(tbl)) {
1115
-		pr_err("%s: Invalid IOMMU page size %lx (%lx) on %s\n",
1116
-		       __func__, IOMMU_PAGE_SIZE(tbl),
1117
-		       PAGE_SIZE, dev_name(dev));
1118
-		return -EINVAL;
1119
-	}
1120
-
1121
-	return iommu_group_add_device(tgl->table_group->group, dev);
1114
+	return iommu_group_add_device(table_group->group, dev);
11221115 }
11231116 EXPORT_SYMBOL_GPL(iommu_add_device);
11241117 
....@@ -1129,7 +1122,7 @@
11291122 	 * and we needn't detach them from the associated
11301123 	 * IOMMU groups
11311124 	 */
1132
-	if (!dev->iommu_group) {
1125
+	if (!device_iommu_mapped(dev)) {
11331126 		pr_debug("iommu_tce: skipping device %s with no tbl\n",
11341127 			 dev_name(dev));
11351128 		return;
....@@ -1138,31 +1131,4 @@
11381131 	iommu_group_remove_device(dev);
11391132 }
11401133 EXPORT_SYMBOL_GPL(iommu_del_device);
1141
-
1142
-static int tce_iommu_bus_notifier(struct notifier_block *nb,
1143
-                unsigned long action, void *data)
1144
-{
1145
-        struct device *dev = data;
1146
-
1147
-        switch (action) {
1148
-        case BUS_NOTIFY_ADD_DEVICE:
1149
-                return iommu_add_device(dev);
1150
-        case BUS_NOTIFY_DEL_DEVICE:
1151
-                if (dev->iommu_group)
1152
-                        iommu_del_device(dev);
1153
-                return 0;
1154
-        default:
1155
-                return 0;
1156
-        }
1157
-}
1158
-
1159
-static struct notifier_block tce_iommu_bus_nb = {
1160
-        .notifier_call = tce_iommu_bus_notifier,
1161
-};
1162
-
1163
-int __init tce_iommu_bus_notifier_init(void)
1164
-{
1165
-        bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
1166
-        return 0;
1167
-}
11681134 #endif /* CONFIG_IOMMU_API */