forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 37f49e37ab4cb5d0bc4c60eb5c6d4dd57db767bb 
 kernel/include/linux/io-pgtable.h
....@@ -1,9 +1,9 @@
11 /* SPDX-License-Identifier: GPL-2.0 */
22 #ifndef __IO_PGTABLE_H
33 #define __IO_PGTABLE_H
4
-#include <linux/bitops.h>
54 
6
-#include <linux/scatterlist.h>
5
+#include <linux/bitops.h>
6
+#include <linux/iommu.h>
77 
88 /*
99  * Public API for use by IOMMU drivers
....@@ -14,33 +14,32 @@
1414 	ARM_64_LPAE_S1,
1515 	ARM_64_LPAE_S2,
1616 	ARM_V7S,
17
-	ARM_V8L_FAST,
17
+	ARM_MALI_LPAE,
1818 	IO_PGTABLE_NUM_FMTS,
1919 };
2020 
2121 /**
22
- * struct iommu_gather_ops - IOMMU callbacks for TLB and page table management.
22
+ * struct iommu_flush_ops - IOMMU callbacks for TLB and page table management.
2323  *
24
- * @tlb_flush_all: Synchronously invalidate the entire TLB context.
25
- * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range.
26
- * @tlb_sync:      Ensure any queued TLB invalidation has taken effect, and
27
- *                 any corresponding page table updates are visible to the
28
- *                 IOMMU.
29
- * @alloc_pages_exact: Allocate page table memory (optional, defaults to
30
- *                     alloc_pages_exact)
31
- * @free_pages_exact:  Free page table memory (optional, defaults to
32
- *                     free_pages_exact)
24
+ * @tlb_flush_all:  Synchronously invalidate the entire TLB context.
25
+ * @tlb_flush_walk: Synchronously invalidate all intermediate TLB state
26
+ *                  (sometimes referred to as the "walk cache") for a virtual
27
+ *                  address range.
28
+ * @tlb_add_page:   Optional callback to queue up leaf TLB invalidation for a
29
+ *                  single page.  IOMMUs that cannot batch TLB invalidation
30
+ *                  operations efficiently will typically issue them here, but
31
+ *                  others may decide to update the iommu_iotlb_gather structure
32
+ *                  and defer the invalidation until iommu_iotlb_sync() instead.
3333  *
3434  * Note that these can all be called in atomic context and must therefore
3535  * not block.
3636  */
37
-struct iommu_gather_ops {
37
+struct iommu_flush_ops {
3838 	void (*tlb_flush_all)(void *cookie);
39
-	void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule,
40
-			      bool leaf, void *cookie);
41
-	void (*tlb_sync)(void *cookie);
42
-	void *(*alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask);
43
-	void (*free_pages_exact)(void *cookie, void *virt, size_t size);
39
+	void (*tlb_flush_walk)(unsigned long iova, size_t size, size_t granule,
40
+			       void *cookie);
41
+	void (*tlb_add_page)(struct iommu_iotlb_gather *gather,
42
+			     unsigned long iova, size_t granule, void *cookie);
4443 };
4544 
4645 /**
....@@ -52,6 +51,8 @@
5251  *                 tables.
5352  * @ias:           Input address (iova) size, in bits.
5453  * @oas:           Output address (paddr) size, in bits.
54
+ * @coherent_walk  A flag to indicate whether or not page table walks made
55
+ *                 by the IOMMU are coherent with the CPU caches.
5556  * @tlb:           TLB management callbacks for this set of tables.
5657  * @iommu_dev:     The device representing the DMA configuration for the
5758  *                 page table walker.
....@@ -67,114 +68,103 @@
6768 	 *	hardware which does not implement the permissions of a given
6869 	 *	format, and/or requires some format-specific default value.
6970 	 *
70
-	 * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid
71
-	 *	(unmapped) entries but the hardware might do so anyway, perform
72
-	 *	TLB maintenance when mapping as well as when unmapping.
71
+	 * IO_PGTABLE_QUIRK_ARM_MTK_EXT: (ARM v7s format) MediaTek IOMMUs extend
72
+	 *	to support up to 35 bits PA where the bit32, bit33 and bit34 are
73
+	 *	encoded in the bit9, bit4 and bit5 of the PTE respectively.
7374 	 *
74
-	 * IO_PGTABLE_QUIRK_ARM_MTK_4GB: (ARM v7s format) Set bit 9 in all
75
-	 *	PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit
76
-	 *	when the SoC is in "4GB mode" and they can only access the high
77
-	 *	remap of DRAM (0x1_00000000 to 0x1_ffffffff).
75
+	 * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
76
+	 *	on unmap, for DMA domains using the flush queue mechanism for
77
+	 *	delayed invalidation.
7878 	 *
79
-
80
-	 * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever
81
-	 *	be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a
82
-	 *	software-emulated IOMMU), such that pagetable updates need not
83
-	 *	be treated as explicit DMA data.
84
-	 *
85
-
86
-	 * IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE:
87
-	 *	Having page tables which are non coherent, but cached in a
88
-	 *	system cache requires SH=Non-Shareable. This applies to the
89
-	 *	qsmmuv500 model. For data buffers SH=Non-Shareable is not
90
-	 *	required.
91
-
92
-	 * IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT: Override the attributes
93
-	 *	set in TCR for the page table walker. Use attributes specified
94
-	 *	by the upstream hw instead.
95
-	 *
96
-	 * IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA: Override the attributes
97
-	 *	set in TCR for the page table walker with Write-Back,
98
-	 *	no Write-Allocate cacheable encoding.
99
-	 *
79
+	 * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table
80
+	 *	for use in the upper half of a split address space.
10081 	 */
10182 	#define IO_PGTABLE_QUIRK_ARM_NS		BIT(0)
10283 	#define IO_PGTABLE_QUIRK_NO_PERMS	BIT(1)
103
-	#define IO_PGTABLE_QUIRK_TLBI_ON_MAP	BIT(2)
104
-	#define IO_PGTABLE_QUIRK_ARM_MTK_4GB	BIT(3)
105
-	#define IO_PGTABLE_QUIRK_NO_DMA		BIT(4)
106
-	#define IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE BIT(5)
107
-	#define IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT	BIT(6)
108
-	#define IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA	BIT(7)
84
+	#define IO_PGTABLE_QUIRK_ARM_MTK_EXT	BIT(3)
85
+	#define IO_PGTABLE_QUIRK_NON_STRICT	BIT(4)
86
+	#define IO_PGTABLE_QUIRK_ARM_TTBR1	BIT(5)
10987 	unsigned long			quirks;
11088 	unsigned long			pgsize_bitmap;
11189 	unsigned int			ias;
11290 	unsigned int			oas;
113
-	const struct iommu_gather_ops	*tlb;
91
+	bool				coherent_walk;
92
+	const struct iommu_flush_ops	*tlb;
11493 	struct device			*iommu_dev;
115
-	dma_addr_t			iova_base;
116
-	dma_addr_t			iova_end;
11794 
11895 	/* Low-level data specific to the table format */
11996 	union {
12097 		struct {
121
-			u64	ttbr[2];
122
-			u64	tcr;
123
-			u64	mair[2];
98
+			u64	ttbr;
99
+			struct {
100
+				u32	ips:3;
101
+				u32	tg:2;
102
+				u32	sh:2;
103
+				u32	orgn:2;
104
+				u32	irgn:2;
105
+				u32	tsz:6;
106
+			}	tcr;
107
+			u64	mair;
124108 		} arm_lpae_s1_cfg;
125109 
126110 		struct {
127111 			u64	vttbr;
128
-			u64	vtcr;
112
+			struct {
113
+				u32	ps:3;
114
+				u32	tg:2;
115
+				u32	sh:2;
116
+				u32	orgn:2;
117
+				u32	irgn:2;
118
+				u32	sl:2;
119
+				u32	tsz:6;
120
+			}	vtcr;
129121 		} arm_lpae_s2_cfg;
130122 
131123 		struct {
132
-			u32	ttbr[2];
124
+			u32	ttbr;
133125 			u32	tcr;
134126 			u32	nmrr;
135127 			u32	prrr;
136128 		} arm_v7s_cfg;
137129 
138130 		struct {
139
-			u64	ttbr[2];
140
-			u64	tcr;
141
-			u64	mair[2];
142
-			void	*pmds;
143
-		} av8l_fast_cfg;
131
+			u64	transtab;
132
+			u64	memattr;
133
+		} arm_mali_lpae_cfg;
144134 	};
145135 };
146136 
147137 /**
148138  * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
149139  *
150
- * @map:		Map a physically contiguous memory region.
151
- * @map_sg:		Map a scatterlist.  Returns the number of bytes mapped,
152
- *			or -ve val on failure.  The size parameter contains the
153
- *			size of the partial mapping in case of failure.
154
- * @unmap:		Unmap a physically contiguous memory region.
155
- * @iova_to_phys:	Translate iova to physical address.
156
- * @is_iova_coherent:	Checks coherency of given IOVA. Returns True if coherent
157
- *			and False if non-coherent.
158
- * @iova_to_pte:	Translate iova to Page Table Entry (PTE).
140
+ * @map:          Map a physically contiguous memory region.
141
+ * @map_pages:    Map a physically contiguous range of pages of the same size.
142
+ * @map_sg:       Map a scatter-gather list of physically contiguous memory
143
+ *                chunks. The mapped pointer argument is used to store how
144
+ *                many bytes are mapped.
145
+ * @unmap:        Unmap a physically contiguous memory region.
146
+ * @unmap_pages:  Unmap a range of virtually contiguous pages of the same size.
147
+ * @iova_to_phys: Translate iova to physical address.
159148  *
160149  * These functions map directly onto the iommu_ops member functions with
161150  * the same names.
162151  */
163152 struct io_pgtable_ops {
164153 	int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
165
-		   phys_addr_t paddr, size_t size, int prot);
166
-	size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
167
-			size_t size);
154
+		   phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
155
+	int (*map_pages)(struct io_pgtable_ops *ops, unsigned long iova,
156
+			 phys_addr_t paddr, size_t pgsize, size_t pgcount,
157
+			 int prot, gfp_t gfp, size_t *mapped);
168158 	int (*map_sg)(struct io_pgtable_ops *ops, unsigned long iova,
169
-		      struct scatterlist *sg, unsigned int nents,
170
-		      int prot, size_t *size);
159
+		      struct scatterlist *sg, unsigned int nents, int prot,
160
+		      gfp_t gfp, size_t *mapped);
161
+	size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
162
+			size_t size, struct iommu_iotlb_gather *gather);
163
+	size_t (*unmap_pages)(struct io_pgtable_ops *ops, unsigned long iova,
164
+			      size_t pgsize, size_t pgcount,
165
+			      struct iommu_iotlb_gather *gather);
171166 	phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops,
172167 				    unsigned long iova);
173
-	bool (*is_iova_coherent)(struct io_pgtable_ops *ops,
174
-				unsigned long iova);
175
-	uint64_t (*iova_to_pte)(struct io_pgtable_ops *ops,
176
-		    unsigned long iova);
177
-
178168 };
179169 
180170 /**
....@@ -225,24 +215,25 @@
225215 
226216 static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop)
227217 {
228
-	if (!iop->cfg.tlb)
229
-		return;
230
-	iop->cfg.tlb->tlb_flush_all(iop->cookie);
218
+	if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_all)
219
+		iop->cfg.tlb->tlb_flush_all(iop->cookie);
231220 }
232221 
233
-static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop,
234
-		unsigned long iova, size_t size, size_t granule, bool leaf)
222
+static inline void
223
+io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova,
224
+			  size_t size, size_t granule)
235225 {
236
-	if (!iop->cfg.tlb)
237
-		return;
238
-	iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie);
226
+	if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_walk)
227
+		iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie);
239228 }
240229 
241
-static inline void io_pgtable_tlb_sync(struct io_pgtable *iop)
230
+static inline void
231
+io_pgtable_tlb_add_page(struct io_pgtable *iop,
232
+			struct iommu_iotlb_gather * gather, unsigned long iova,
233
+			size_t granule)
242234 {
243
-	if (!iop->cfg.tlb)
244
-		return;
245
-	iop->cfg.tlb->tlb_sync(iop->cookie);
235
+	if (iop->cfg.tlb && iop->cfg.tlb->tlb_add_page)
236
+		iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie);
246237 }
247238 
248239 /**
....@@ -262,31 +253,6 @@
262253 extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
263254 extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
264255 extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
265
-extern struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns;
266
-extern struct io_pgtable_init_fns io_pgtable_arm_msm_secure_init_fns;
267
-
268
-/**
269
- * io_pgtable_alloc_pages_exact:
270
- *	allocate an exact number of physically-contiguous pages.
271
- * @size: the number of bytes to allocate
272
- * @gfp_mask: GFP flags for the allocation
273
- *
274
- * Like alloc_pages_exact(), but with some additional accounting for debug
275
- * purposes.
276
- */
277
-void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
278
-				   size_t size, gfp_t gfp_mask);
279
-
280
-/**
281
- * io_pgtable_free_pages_exact:
282
- *	release memory allocated via io_pgtable_alloc_pages_exact()
283
- * @virt: the value returned by alloc_pages_exact.
284
- * @size: size of allocation, same value as passed to alloc_pages_exact().
285
- *
286
- * Like free_pages_exact(), but with some additional accounting for debug
287
- * purposes.
288
- */
289
-void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
290
-				 void *virt, size_t size);
256
+extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns;
291257 
292258 #endif /* __IO_PGTABLE_H */