forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 cde9070d9970eef1f7ec2360586c802a16230ad8 
 kernel/arch/x86/kernel/amd_gart_64.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Dynamic DMA mapping support for AMD Hammer.
34  *
....@@ -5,10 +6,9 @@
56  * This allows to use PCI devices that only support 32bit addresses on systems
67  * with more than 4GB.
78  *
8
- * See Documentation/DMA-API-HOWTO.txt for the interface specification.
9
+ * See Documentation/core-api/dma-api-howto.rst for the interface specification.
910  *
1011  * Copyright 2002 Andi Kleen, SuSE Labs.
11
- * Subject to the GNU General Public License v2 only.
1212  */
1313 
1414 #include <linux/types.h>
....@@ -32,8 +32,8 @@
3232 #include <linux/gfp.h>
3333 #include <linux/atomic.h>
3434 #include <linux/dma-direct.h>
35
+#include <linux/dma-map-ops.h>
3536 #include <asm/mtrr.h>
36
-#include <asm/pgtable.h>
3737 #include <asm/proto.h>
3838 #include <asm/iommu.h>
3939 #include <asm/gart.h>
....@@ -49,8 +49,6 @@
4949 static unsigned long iommu_pages;	/* .. and in pages */
5050 
5151 static u32 *iommu_gatt_base;		/* Remapping table */
52
-
53
-static dma_addr_t bad_dma_addr;
5452 
5553 /*
5654  * If this is disabled the IOMMU will use an optimized flushing strategy
....@@ -73,8 +71,6 @@
7371 #define GPTE_ENCODE(x) \
7472 	(((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT)
7573 #define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28))
76
-
77
-#define EMERGENCY_PAGES 32 /* = 128KB */
7874 
7975 #ifdef CONFIG_AGP
8076 #define AGPEXTERN extern
....@@ -101,8 +97,7 @@
10197 
10298 	base_index = ALIGN(iommu_bus_base & dma_get_seg_boundary(dev),
10399 			   PAGE_SIZE) >> PAGE_SHIFT;
104
-	boundary_size = ALIGN((u64)dma_get_seg_boundary(dev) + 1,
105
-			      PAGE_SIZE) >> PAGE_SHIFT;
100
+	boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT);
106101 
107102 	spin_lock_irqsave(&iommu_bitmap_lock, flags);
108103 	offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, next_bit,
....@@ -155,9 +150,6 @@
155150 
156151 #ifdef CONFIG_IOMMU_LEAK
157152 /* Debugging aid for drivers that don't free their IOMMU tables */
158
-static int leak_trace;
159
-static int iommu_leak_pages = 20;
160
-
161153 static void dump_leak(void)
162154 {
163155 	static int dump;
....@@ -166,7 +158,7 @@
166158 		return;
167159 	dump = 1;
168160 
169
-	show_stack(NULL, NULL);
161
+	show_stack(NULL, NULL, KERN_ERR);
170162 	debug_dma_dump_mappings(NULL);
171163 }
172164 #endif
....@@ -184,14 +176,6 @@
184176 	 */
185177 
186178 	dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n", size);
187
-
188
-	if (size > PAGE_SIZE*EMERGENCY_PAGES) {
189
-		if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL)
190
-			panic("PCI-DMA: Memory would be corrupted\n");
191
-		if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL)
192
-			panic(KERN_ERR
193
-				"PCI-DMA: Random memory would be DMAed\n");
194
-	}
195179 #ifdef CONFIG_IOMMU_LEAK
196180 	dump_leak();
197181 #endif
....@@ -200,13 +184,13 @@
200184 static inline int
201185 need_iommu(struct device *dev, unsigned long addr, size_t size)
202186 {
203
-	return force_iommu || !dma_capable(dev, addr, size);
187
+	return force_iommu || !dma_capable(dev, addr, size, true);
204188 }
205189 
206190 static inline int
207191 nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
208192 {
209
-	return !dma_capable(dev, addr, size);
193
+	return !dma_capable(dev, addr, size, true);
210194 }
211195 
212196 /* Map a single continuous physical area into the IOMMU.
....@@ -220,7 +204,7 @@
220204 	int i;
221205 
222206 	if (unlikely(phys_mem + size > GART_MAX_PHYS_ADDR))
223
-		return bad_dma_addr;
207
+		return DMA_MAPPING_ERROR;
224208 
225209 	iommu_page = alloc_iommu(dev, npages, align_mask);
226210 	if (iommu_page == -1) {
....@@ -229,7 +213,7 @@
229213 		if (panic_on_overflow)
230214 			panic("dma_map_area overflow %lu bytes\n", size);
231215 		iommu_full(dev, size, dir);
232
-		return bad_dma_addr;
216
+		return DMA_MAPPING_ERROR;
233217 	}
234218 
235219 	for (i = 0; i < npages; i++) {
....@@ -247,9 +231,6 @@
247231 {
248232 	unsigned long bus;
249233 	phys_addr_t paddr = page_to_phys(page) + offset;
250
-
251
-	if (!dev)
252
-		dev = &x86_dma_fallback_dev;
253234 
254235 	if (!need_iommu(dev, paddr, size))
255236 		return paddr;
....@@ -271,7 +252,15 @@
271252 	int npages;
272253 	int i;
273254 
274
-	if (dma_addr < iommu_bus_base + EMERGENCY_PAGES*PAGE_SIZE ||
255
+	if (WARN_ON_ONCE(dma_addr == DMA_MAPPING_ERROR))
256
+		return;
257
+
258
+	/*
259
+	 * This driver will not always use a GART mapping, but might have
260
+	 * created a direct mapping instead.  If that is the case there is
261
+	 * nothing to unmap here.
262
+	 */
263
+	if (dma_addr < iommu_bus_base ||
275264 	    dma_addr >= iommu_bus_base + iommu_size)
276265 		return;
277266 
....@@ -315,7 +304,7 @@
315304 
316305 		if (nonforced_iommu(dev, addr, s->length)) {
317306 			addr = dma_map_area(dev, addr, s->length, dir, 0);
318
-			if (addr == bad_dma_addr) {
307
+			if (addr == DMA_MAPPING_ERROR) {
319308 				if (i > 0)
320309 					gart_unmap_sg(dev, sg, i, dir, 0);
321310 				nents = 0;
....@@ -399,9 +388,6 @@
399388 	if (nents == 0)
400389 		return 0;
401390 
402
-	if (!dev)
403
-		dev = &x86_dma_fallback_dev;
404
-
405391 	out		= 0;
406392 	start		= 0;
407393 	start_sg	= sg;
....@@ -471,7 +457,7 @@
471457 
472458 	iommu_full(dev, pages << PAGE_SHIFT, dir);
473459 	for_each_sg(sg, s, nents, i)
474
-		s->dma_address = bad_dma_addr;
460
+		s->dma_address = DMA_MAPPING_ERROR;
475461 	return 0;
476462 }
477463 
....@@ -490,7 +476,7 @@
490476 	*dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
491477 			DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
492478 	flush_gart();
493
-	if (unlikely(*dma_addr == bad_dma_addr))
479
+	if (unlikely(*dma_addr == DMA_MAPPING_ERROR))
494480 		goto out_free;
495481 	return vaddr;
496482 out_free:
....@@ -505,11 +491,6 @@
505491 {
506492 	gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
507493 	dma_direct_free(dev, size, vaddr, dma_addr, attrs);
508
-}
509
-
510
-static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr)
511
-{
512
-	return (dma_addr == bad_dma_addr);
513494 }
514495 
515496 static int no_agp;
....@@ -528,10 +509,9 @@
528509 	iommu_size -= round_up(a, PMD_PAGE_SIZE) - a;
529510 
530511 	if (iommu_size < 64*1024*1024) {
531
-		pr_warning(
532
-			"PCI-DMA: Warning: Small IOMMU %luMB."
512
+		pr_warn("PCI-DMA: Warning: Small IOMMU %luMB."
533513 			" Consider increasing the AGP aperture in BIOS\n",
534
-				iommu_size >> 20);
514
+			iommu_size >> 20);
535515 	}
536516 
537517 	return iommu_size;
....@@ -683,8 +663,7 @@
683663 
684664  nommu:
685665 	/* Should not happen anymore */
686
-	pr_warning("PCI-DMA: More than 4GB of RAM and no IOMMU\n"
687
-	       "falling back to iommu=soft.\n");
666
+	pr_warn("PCI-DMA: More than 4GB of RAM and no IOMMU - falling back to iommu=soft.\n");
688667 	return -1;
689668 }
690669 
....@@ -695,8 +674,12 @@
695674 	.unmap_page			= gart_unmap_page,
696675 	.alloc				= gart_alloc_coherent,
697676 	.free				= gart_free_coherent,
698
-	.mapping_error			= gart_mapping_error,
677
+	.mmap				= dma_common_mmap,
678
+	.get_sgtable			= dma_common_get_sgtable,
699679 	.dma_supported			= dma_direct_supported,
680
+	.get_required_mask		= dma_direct_get_required_mask,
681
+	.alloc_pages			= dma_direct_alloc_pages,
682
+	.free_pages			= dma_direct_free_pages,
700683 };
701684 
702685 static void gart_iommu_shutdown(void)
....@@ -730,7 +713,6 @@
730713 	unsigned long aper_base, aper_size;
731714 	unsigned long start_pfn, end_pfn;
732715 	unsigned long scratch;
733
-	long i;
734716 
735717 	if (!amd_nb_has_feature(AMD_NB_GART))
736718 		return 0;
....@@ -750,8 +732,8 @@
750732 	    !gart_iommu_aperture ||
751733 	    (no_agp && init_amd_gatt(&info) < 0)) {
752734 		if (max_pfn > MAX_DMA32_PFN) {
753
-			pr_warning("More than 4GB of memory but GART IOMMU not available.\n");
754
-			pr_warning("falling back to iommu=soft.\n");
735
+			pr_warn("More than 4GB of memory but GART IOMMU not available.\n");
736
+			pr_warn("falling back to iommu=soft.\n");
755737 		}
756738 		return 0;
757739 	}
....@@ -763,7 +745,8 @@
763745 
764746 	start_pfn = PFN_DOWN(aper_base);
765747 	if (!pfn_range_is_mapped(start_pfn, end_pfn))
766
-		init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
748
+		init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT,
749
+				    PAGE_KERNEL);
767750 
768751 	pr_info("PCI-DMA: using GART IOMMU.\n");
769752 	iommu_size = check_iommu_size(info.aper_base, aper_size);
....@@ -774,29 +757,12 @@
774757 	if (!iommu_gart_bitmap)
775758 		panic("Cannot allocate iommu bitmap\n");
776759 
777
-#ifdef CONFIG_IOMMU_LEAK
778
-	if (leak_trace) {
779
-		int ret;
780
-
781
-		ret = dma_debug_resize_entries(iommu_pages);
782
-		if (ret)
783
-			pr_debug("PCI-DMA: Cannot trace all the entries\n");
784
-	}
785
-#endif
786
-
787
-	/*
788
-	 * Out of IOMMU space handling.
789
-	 * Reserve some invalid pages at the beginning of the GART.
790
-	 */
791
-	bitmap_set(iommu_gart_bitmap, 0, EMERGENCY_PAGES);
792
-
793760 	pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n",
794761 	       iommu_size >> 20);
795762 
796763 	agp_memory_reserved	= iommu_size;
797764 	iommu_start		= aper_size - iommu_size;
798765 	iommu_bus_base		= info.aper_base + iommu_start;
799
-	bad_dma_addr		= iommu_bus_base;
800766 	iommu_gatt_base		= agp_gatt_table + (iommu_start>>PAGE_SHIFT);
801767 
802768 	/*
....@@ -838,8 +804,6 @@
838804 	if (!scratch)
839805 		panic("Cannot allocate iommu scratch page");
840806 	gart_unmapped_entry = GPTE_ENCODE(__pa(scratch));
841
-	for (i = EMERGENCY_PAGES; i < iommu_pages; i++)
842
-		iommu_gatt_base[i] = gart_unmapped_entry;
843807 
844808 	flush_gart();
845809 	dma_ops = &gart_dma_ops;
....@@ -853,16 +817,6 @@
853817 {
854818 	int arg;
855819 
856
-#ifdef CONFIG_IOMMU_LEAK
857
-	if (!strncmp(p, "leak", 4)) {
858
-		leak_trace = 1;
859
-		p += 4;
860
-		if (*p == '=')
861
-			++p;
862
-		if (isdigit(*p) && get_option(&p, &arg))
863
-			iommu_leak_pages = arg;
864
-	}
865
-#endif
866820 	if (isdigit(*p) && get_option(&p, &arg))
867821 		iommu_size = arg;
868822 	if (!strncmp(p, "fullflush", 9))