~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,19 +1,7 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
1	2	/*
2	3	* Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3	4	* Author: Joerg Roedel <joerg.roedel@amd.com>
4		- *
5		- * This program is free software; you can redistribute it and/or modify it
6		- * under the terms of the GNU General Public License version 2 as published
7		- * by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program; if not, write to the Free Software
16		- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17	5	*/
18	6
19	7	#ifndef __LINUX_IOMMU_H
..	..	@@ -25,6 +13,8 @@
25	13	#include <linux/errno.h>
26	14	#include <linux/err.h>
27	15	#include <linux/of.h>
	16	+#include <linux/ioasid.h>
	17	+#include <uapi/linux/iommu.h>
28	18
29	19	#define IOMMU_READ (1 << 0)
30	20	#define IOMMU_WRITE (1 << 1)
..	..	@@ -41,11 +31,18 @@
41	31	* if the IOMMU page table format is equivalent.
42	32	*/
43	33	#define IOMMU_PRIV (1 << 5)
44		-/* Use upstream device's bus attribute */
45		-#define IOMMU_USE_UPSTREAM_HINT (1 << 6)
46		-
47		-/* Use upstream device's bus attribute with no write-allocate cache policy */
48		-#define IOMMU_USE_LLC_NWA (1 << 7)
	34	+/*
	35	+ * Allow caching in a transparent outer level of cache, also known as
	36	+ * the last-level or system cache, with a read/write allocation policy.
	37	+ * Does not depend on IOMMU_CACHE. Incompatible with IOMMU_SYS_CACHE_NWA.
	38	+ */
	39	+#define IOMMU_SYS_CACHE (1 << 6)
	40	+/*
	41	+ * Allow caching in a transparent outer level of cache, also known as
	42	+ * the last-level or system cache, with a read allocation policy.
	43	+ * Does not depend on IOMMU_CACHE. Incompatible with IOMMU_SYS_CACHE.
	44	+ */
	45	+#define IOMMU_SYS_CACHE_NWA (1 << 7)
49	46
50	47	struct iommu_ops;
51	48	struct iommu_group;
..	..	@@ -53,26 +50,21 @@
53	50	struct device;
54	51	struct iommu_domain;
55	52	struct notifier_block;
	53	+struct iommu_sva;
	54	+struct iommu_fault_event;
56	55
57	56	/* iommu fault flags */
58		-#define IOMMU_FAULT_READ (1 << 0)
59		-#define IOMMU_FAULT_WRITE (1 << 1)
60		-#define IOMMU_FAULT_TRANSLATION (1 << 2)
61		-#define IOMMU_FAULT_PERMISSION (1 << 3)
62		-#define IOMMU_FAULT_EXTERNAL (1 << 4)
63		-#define IOMMU_FAULT_TRANSACTION_STALLED (1 << 5)
	57	+#define IOMMU_FAULT_READ 0x0
	58	+#define IOMMU_FAULT_WRITE 0x1
64	59
65	60	typedef int (iommu_fault_handler_t)(struct iommu_domain ,
66	61	struct device , unsigned long, int, void );
	62	+typedef int (iommu_dev_fault_handler_t)(struct iommu_fault , void *);
67	63
68	64	struct iommu_domain_geometry {
69	65	dma_addr_t aperture_start; /* First address that can be mapped */
70	66	dma_addr_t aperture_end; /* Last address that can be mapped */
71	67	bool force_aperture; /* DMA only allowed in mappable range? */
72		-};
73		-
74		-struct iommu_pgtbl_info {
75		- void *ops;
76	68	};
77	69
78	70	/* Domain feature flags */
..	..	@@ -99,8 +91,6 @@
99	91	#define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING \| \
100	92	__IOMMU_DOMAIN_DMA_API)
101	93
102		-
103		-#define IOMMU_DOMAIN_NAME_LEN 32
104	94	struct iommu_domain {
105	95	unsigned type;
106	96	const struct iommu_ops *ops;
..	..	@@ -109,8 +99,6 @@
109	99	void *handler_token;
110	100	struct iommu_domain_geometry geometry;
111	101	void *iova_cookie;
112		- bool is_debug_domain;
113		- char name[IOMMU_DOMAIN_NAME_LEN];
114	102	};
115	103
116	104	enum iommu_cap {
..	..	@@ -131,11 +119,6 @@
131	119	* DOMAIN_ATTR_FSL_PAMUV1 corresponds to the above mentioned contraints.
132	120	* The caller can invoke iommu_domain_get_attr to check if the underlying
133	121	* iommu implementation supports these constraints.
134		- *
135		- * DOMAIN_ATTR_NO_CFRE
136		- * Some bus implementations may enter a bad state if iommu reports an error
137		- * on context fault. As context faults are not always fatal, this must be
138		- * avoided.
139	122	*/
140	123
141	124	enum iommu_attr {
..	..	@@ -146,27 +129,7 @@
146	129	DOMAIN_ATTR_FSL_PAMU_ENABLE,
147	130	DOMAIN_ATTR_FSL_PAMUV1,
148	131	DOMAIN_ATTR_NESTING, /* two stages of translation */
149		- DOMAIN_ATTR_PT_BASE_ADDR,
150		- DOMAIN_ATTR_CONTEXT_BANK,
151		- DOMAIN_ATTR_DYNAMIC,
152		- DOMAIN_ATTR_TTBR0,
153		- DOMAIN_ATTR_CONTEXTIDR,
154		- DOMAIN_ATTR_PROCID,
155		- DOMAIN_ATTR_NON_FATAL_FAULTS,
156		- DOMAIN_ATTR_S1_BYPASS,
157		- DOMAIN_ATTR_ATOMIC,
158		- DOMAIN_ATTR_SECURE_VMID,
159		- DOMAIN_ATTR_FAST,
160		- DOMAIN_ATTR_PGTBL_INFO,
161		- DOMAIN_ATTR_USE_UPSTREAM_HINT,
162		- DOMAIN_ATTR_EARLY_MAP,
163		- DOMAIN_ATTR_PAGE_TABLE_IS_COHERENT,
164		- DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT,
165		- DOMAIN_ATTR_BITMAP_IOVA_ALLOCATOR,
166		- DOMAIN_ATTR_USE_LLC_NWA,
167		- DOMAIN_ATTR_FAULT_MODEL_NO_CFRE,
168		- DOMAIN_ATTR_FAULT_MODEL_NO_STALL,
169		- DOMAIN_ATTR_FAULT_MODEL_HUPCF,
	132	+ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
170	133	DOMAIN_ATTR_MAX,
171	134	};
172	135
..	..	@@ -174,6 +137,12 @@
174	137	enum iommu_resv_type {
175	138	/* Memory regions which must be mapped 1:1 at all times */
176	139	IOMMU_RESV_DIRECT,
	140	+ /*
	141	+ * Memory regions which are advertised to be 1:1 but are
	142	+ * commonly considered relaxable in some conditions,
	143	+ * for instance in device assignment use case (USB, Graphics)
	144	+ */
	145	+ IOMMU_RESV_DIRECT_RELAXABLE,
177	146	/* Arbitrary "never map this or give it to a device" address ranges */
178	147	IOMMU_RESV_RESERVED,
179	148	/* Hardware MSI region (untranslated) */
..	..	@@ -198,9 +167,32 @@
198	167	enum iommu_resv_type type;
199	168	};
200	169
201		-extern struct dentry *iommu_debugfs_top;
	170	+/* Per device IOMMU features */
	171	+enum iommu_dev_features {
	172	+ IOMMU_DEV_FEAT_AUX, /* Aux-domain feature */
	173	+ IOMMU_DEV_FEAT_SVA, /* Shared Virtual Addresses */
	174	+};
	175	+
	176	+#define IOMMU_PASID_INVALID (-1U)
202	177
203	178	#ifdef CONFIG_IOMMU_API
	179	+
	180	+/**
	181	+ * struct iommu_iotlb_gather - Range information for a pending IOTLB flush
	182	+ *
	183	+ * @start: IOVA representing the start of the range to be flushed
	184	+ * @end: IOVA representing the end of the range to be flushed (inclusive)
	185	+ * @pgsize: The interval at which to perform the flush
	186	+ *
	187	+ * This structure is intended to be updated by multiple calls to the
	188	+ * ->unmap() function in struct iommu_ops before eventually being passed
	189	+ * into ->iotlb_sync().
	190	+ */
	191	+struct iommu_iotlb_gather {
	192	+ unsigned long start;
	193	+ unsigned long end;
	194	+ size_t pgsize;
	195	+};
204	196
205	197	/**
206	198	* struct iommu_ops - iommu ops and capabilities
..	..	@@ -210,17 +202,21 @@
210	202	* @attach_dev: attach device to an iommu domain
211	203	* @detach_dev: detach device from an iommu domain
212	204	* @map: map a physically contiguous memory region to an iommu domain
	205	+ * @map_pages: map a physically contiguous set of pages of the same size to
	206	+ * an iommu domain.
	207	+ * @map_sg: map a scatter-gather list of physically contiguous chunks to
	208	+ * an iommu domain.
213	209	* @unmap: unmap a physically contiguous memory region from an iommu domain
214		- * @map_sg: map a scatter-gather list of physically contiguous memory chunks
215		- * to an iommu domain
216		- * @flush_tlb_all: Synchronously flush all hardware TLBs for this domain
217		- * @tlb_range_add: Add a given iova range to the flush queue for this domain
218		- * @tlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
	210	+ * @unmap_pages: unmap a number of pages of the same size from an iommu domain
	211	+ * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
	212	+ * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
	213	+ * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
219	214	* queue
220	215	* @iova_to_phys: translate iova to physical address
221		- * @iova_to_phys_hard: translate iova to physical address using IOMMU hardware
222		- * @add_device: add device to iommu grouping
223		- * @remove_device: remove device from iommu grouping
	216	+ * @probe_device: Add device to iommu driver handling
	217	+ * @release_device: Remove device from iommu driver handling
	218	+ * @probe_finalize: Do final setup work after the device is added to an IOMMU
	219	+ * group and attached to the groups domain
224	220	* @device_group: find iommu group for a particular device
225	221	* @domain_get_attr: Query domain attributes
226	222	* @domain_set_attr: Change domain attributes
..	..	@@ -229,14 +225,27 @@
229	225	* @apply_resv_region: Temporary helper call-back for iova reserved ranges
230	226	* @domain_window_enable: Configure and enable a particular window for a domain
231	227	* @domain_window_disable: Disable a particular window for a domain
232		- * @domain_set_windows: Set the number of windows for a domain
233		- * @domain_get_windows: Return the number of windows for a domain
234	228	* @of_xlate: add OF master IDs to iommu grouping
	229	+ * @is_attach_deferred: Check if domain attach should be deferred from iommu
	230	+ * driver init to device driver init (default no)
	231	+ * @dev_has/enable/disable_feat: per device entries to check/enable/disable
	232	+ * iommu specific features.
	233	+ * @dev_feat_enabled: check enabled feature
	234	+ * @aux_attach/detach_dev: aux-domain specific attach/detach entries.
	235	+ * @aux_get_pasid: get the pasid given an aux-domain
	236	+ * @sva_bind: Bind process address space to device
	237	+ * @sva_unbind: Unbind process address space from device
	238	+ * @sva_get_pasid: Get PASID associated to a SVA handle
	239	+ * @page_response: handle page request response
	240	+ * @cache_invalidate: invalidate translation caches
	241	+ * @sva_bind_gpasid: bind guest pasid and mm
	242	+ * @sva_unbind_gpasid: unbind guest pasid and mm
	243	+ * @def_domain_type: device default domain type, return value:
	244	+ * - IOMMU_DOMAIN_IDENTITY: must use an identity domain
	245	+ * - IOMMU_DOMAIN_DMA: must use a dma domain
	246	+ * - 0: use the default setting
235	247	* @pgsize_bitmap: bitmap of all possible supported page sizes
236		- * @trigger_fault: trigger a fault on the device attached to an iommu domain
237		- * @tlbi_domain: Invalidate all TLBs covering an iommu domain
238		- * @enable_config_clocks: Enable all config clocks for this domain's IOMMU
239		- * @disable_config_clocks: Disable all config clocks for this domain's IOMMU
	248	+ * @owner: Driver module providing these ops
240	249	*/
241	250	struct iommu_ops {
242	251	bool (*capable)(enum iommu_cap);
..	..	@@ -248,20 +257,27 @@
248	257	int (attach_dev)(struct iommu_domain domain, struct device *dev);
249	258	void (detach_dev)(struct iommu_domain domain, struct device *dev);
250	259	int (map)(struct iommu_domain domain, unsigned long iova,
251		- phys_addr_t paddr, size_t size, int prot);
	260	+ phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
	261	+ int (map_pages)(struct iommu_domain domain, unsigned long iova,
	262	+ phys_addr_t paddr, size_t pgsize, size_t pgcount,
	263	+ int prot, gfp_t gfp, size_t *mapped);
	264	+ int (map_sg)(struct iommu_domain domain, unsigned long iova,
	265	+ struct scatterlist *sg, unsigned int nents, int prot,
	266	+ gfp_t gfp, size_t *mapped);
252	267	size_t (unmap)(struct iommu_domain domain, unsigned long iova,
253		- size_t size);
254		- size_t (map_sg)(struct iommu_domain domain, unsigned long iova,
255		- struct scatterlist *sg, unsigned int nents, int prot);
	268	+ size_t size, struct iommu_iotlb_gather *iotlb_gather);
	269	+ size_t (unmap_pages)(struct iommu_domain domain, unsigned long iova,
	270	+ size_t pgsize, size_t pgcount,
	271	+ struct iommu_iotlb_gather *iotlb_gather);
256	272	void (flush_iotlb_all)(struct iommu_domain domain);
257		- void (iotlb_range_add)(struct iommu_domain domain,
258		- unsigned long iova, size_t size);
259		- void (iotlb_sync)(struct iommu_domain domain);
	273	+ void (iotlb_sync_map)(struct iommu_domain domain, unsigned long iova,
	274	+ size_t size);
	275	+ void (iotlb_sync)(struct iommu_domain domain,
	276	+ struct iommu_iotlb_gather *iotlb_gather);
260	277	phys_addr_t (iova_to_phys)(struct iommu_domain domain, dma_addr_t iova);
261		- phys_addr_t (iova_to_phys_hard)(struct iommu_domain domain,
262		- dma_addr_t iova);
263		- int (add_device)(struct device dev);
264		- void (remove_device)(struct device dev);
	278	+ struct iommu_device (probe_device)(struct device *dev);
	279	+ void (release_device)(struct device dev);
	280	+ void (probe_finalize)(struct device dev);
265	281	struct iommu_group (device_group)(struct device *dev);
266	282	int (domain_get_attr)(struct iommu_domain domain,
267	283	enum iommu_attr attr, void *data);
..	..	@@ -279,22 +295,40 @@
279	295	int (domain_window_enable)(struct iommu_domain domain, u32 wnd_nr,
280	296	phys_addr_t paddr, u64 size, int prot);
281	297	void (domain_window_disable)(struct iommu_domain domain, u32 wnd_nr);
282		- /* Set the number of windows per domain */
283		- int (domain_set_windows)(struct iommu_domain domain, u32 w_count);
284		- /* Get the number of windows per domain */
285		- u32 (domain_get_windows)(struct iommu_domain domain);
286		- void (trigger_fault)(struct iommu_domain domain, unsigned long flags);
287		- void (tlbi_domain)(struct iommu_domain domain);
288		- int (enable_config_clocks)(struct iommu_domain domain);
289		- void (disable_config_clocks)(struct iommu_domain domain);
290		- uint64_t (iova_to_pte)(struct iommu_domain domain,
291		- dma_addr_t iova);
292	298
293	299	int (of_xlate)(struct device dev, struct of_phandle_args *args);
294	300	bool (is_attach_deferred)(struct iommu_domain domain, struct device *dev);
295	301
296		- bool (is_iova_coherent)(struct iommu_domain domain, dma_addr_t iova);
	302	+ /* Per device IOMMU features */
	303	+ bool (dev_has_feat)(struct device dev, enum iommu_dev_features f);
	304	+ bool (dev_feat_enabled)(struct device dev, enum iommu_dev_features f);
	305	+ int (dev_enable_feat)(struct device dev, enum iommu_dev_features f);
	306	+ int (dev_disable_feat)(struct device dev, enum iommu_dev_features f);
	307	+
	308	+ /* Aux-domain specific attach/detach entries */
	309	+ int (aux_attach_dev)(struct iommu_domain domain, struct device *dev);
	310	+ void (aux_detach_dev)(struct iommu_domain domain, struct device *dev);
	311	+ int (aux_get_pasid)(struct iommu_domain domain, struct device *dev);
	312	+
	313	+ struct iommu_sva (sva_bind)(struct device dev, struct mm_struct mm,
	314	+ void *drvdata);
	315	+ void (sva_unbind)(struct iommu_sva handle);
	316	+ u32 (sva_get_pasid)(struct iommu_sva handle);
	317	+
	318	+ int (page_response)(struct device dev,
	319	+ struct iommu_fault_event *evt,
	320	+ struct iommu_page_response *msg);
	321	+ int (cache_invalidate)(struct iommu_domain domain, struct device *dev,
	322	+ struct iommu_cache_invalidate_info *inv_info);
	323	+ int (sva_bind_gpasid)(struct iommu_domain domain,
	324	+ struct device dev, struct iommu_gpasid_bind_data data);
	325	+
	326	+ int (sva_unbind_gpasid)(struct device dev, u32 pasid);
	327	+
	328	+ int (def_domain_type)(struct device dev);
	329	+
297	330	unsigned long pgsize_bitmap;
	331	+ struct module *owner;
298	332	};
299	333
300	334	/**
..	..	@@ -311,6 +345,53 @@
311	345	struct device *dev;
312	346	};
313	347
	348	+/**
	349	+ * struct iommu_fault_event - Generic fault event
	350	+ *
	351	+ * Can represent recoverable faults such as a page requests or
	352	+ * unrecoverable faults such as DMA or IRQ remapping faults.
	353	+ *
	354	+ * @fault: fault descriptor
	355	+ * @list: pending fault event list, used for tracking responses
	356	+ */
	357	+struct iommu_fault_event {
	358	+ struct iommu_fault fault;
	359	+ struct list_head list;
	360	+};
	361	+
	362	+/**
	363	+ * struct iommu_fault_param - per-device IOMMU fault data
	364	+ * @handler: Callback function to handle IOMMU faults at device level
	365	+ * @data: handler private data
	366	+ * @faults: holds the pending faults which needs response
	367	+ * @lock: protect pending faults list
	368	+ */
	369	+struct iommu_fault_param {
	370	+ iommu_dev_fault_handler_t handler;
	371	+ void *data;
	372	+ struct list_head faults;
	373	+ struct mutex lock;
	374	+};
	375	+
	376	+/**
	377	+ * struct dev_iommu - Collection of per-device IOMMU data
	378	+ *
	379	+ * @fault_param: IOMMU detected device fault reporting data
	380	+ * @fwspec: IOMMU fwspec data
	381	+ * @iommu_dev: IOMMU device this device is linked to
	382	+ * @priv: IOMMU Driver private data
	383	+ *
	384	+ * TODO: migrate other per device data pointers under iommu_dev_data, e.g.
	385	+ * struct iommu_group *iommu_group;
	386	+ */
	387	+struct dev_iommu {
	388	+ struct mutex lock;
	389	+ struct iommu_fault_param *fault_param;
	390	+ struct iommu_fwspec *fwspec;
	391	+ struct iommu_device *iommu_dev;
	392	+ void *priv;
	393	+};
	394	+
314	395	int iommu_device_register(struct iommu_device *iommu);
315	396	void iommu_device_unregister(struct iommu_device *iommu);
316	397	int iommu_device_sysfs_add(struct iommu_device *iommu,
..	..	@@ -321,11 +402,18 @@
321	402	int iommu_device_link(struct iommu_device iommu, struct device link);
322	403	void iommu_device_unlink(struct iommu_device iommu, struct device link);
323	404
324		-static inline void iommu_device_set_ops(struct iommu_device *iommu,
325		- const struct iommu_ops *ops)
	405	+static inline void __iommu_device_set_ops(struct iommu_device *iommu,
	406	+ const struct iommu_ops *ops)
326	407	{
327	408	iommu->ops = ops;
328	409	}
	410	+
	411	+#define iommu_device_set_ops(iommu, ops) \
	412	+do { \
	413	+ struct iommu_ops __ops = (struct iommu_ops )(ops); \
	414	+ __ops->owner = THIS_MODULE; \
	415	+ __iommu_device_set_ops(iommu, __ops); \
	416	+} while (0)
329	417
330	418	static inline void iommu_device_set_fwnode(struct iommu_device *iommu,
331	419	struct fwnode_handle *fwnode)
..	..	@@ -338,6 +426,13 @@
338	426	return (struct iommu_device *)dev_get_drvdata(dev);
339	427	}
340	428
	429	+static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
	430	+{
	431	+ *gather = (struct iommu_iotlb_gather) {
	432	+ .start = ULONG_MAX,
	433	+ };
	434	+}
	435	+
341	436	#define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */
342	437	#define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */
343	438	#define IOMMU_GROUP_NOTIFY_BIND_DRIVER 3 /* Pre Driver bind */
..	..	@@ -346,6 +441,7 @@
346	441	#define IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER 6 /* Post Driver unbind */
347	442
348	443	extern int bus_set_iommu(struct bus_type bus, const struct iommu_ops ops);
	444	+extern int bus_iommu_probe(struct bus_type *bus);
349	445	extern bool iommu_present(struct bus_type *bus);
350	446	extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap);
351	447	extern struct iommu_domain iommu_domain_alloc(struct bus_type bus);
..	..	@@ -355,33 +451,43 @@
355	451	struct device *dev);
356	452	extern void iommu_detach_device(struct iommu_domain *domain,
357	453	struct device *dev);
	454	+extern int iommu_uapi_cache_invalidate(struct iommu_domain *domain,
	455	+ struct device *dev,
	456	+ void __user *uinfo);
	457	+
	458	+extern int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain,
	459	+ struct device dev, void __user udata);
	460	+extern int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain,
	461	+ struct device dev, void __user udata);
	462	+extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
	463	+ struct device *dev, ioasid_t pasid);
358	464	extern struct iommu_domain iommu_get_domain_for_dev(struct device dev);
359		-extern size_t iommu_pgsize(unsigned long pgsize_bitmap,
360		- unsigned long addr_merge, size_t size);
	465	+extern struct iommu_domain iommu_get_dma_domain(struct device dev);
361	466	extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
362	467	phys_addr_t paddr, size_t size, int prot);
	468	+extern int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
	469	+ phys_addr_t paddr, size_t size, int prot);
363	470	extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
364	471	size_t size);
365	472	extern size_t iommu_unmap_fast(struct iommu_domain *domain,
366		- unsigned long iova, size_t size);
	473	+ unsigned long iova, size_t size,
	474	+ struct iommu_iotlb_gather *iotlb_gather);
367	475	extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
368		- struct scatterlist *sg, unsigned int nents,
369		- int prot);
370		-extern size_t default_iommu_map_sg(struct iommu_domain *domain,
371		- unsigned long iova,
372		- struct scatterlist *sg, unsigned int nents,
373		- int prot);
	476	+ struct scatterlist *sg,unsigned int nents, int prot);
	477	+extern size_t iommu_map_sg_atomic(struct iommu_domain *domain,
	478	+ unsigned long iova, struct scatterlist *sg,
	479	+ unsigned int nents, int prot);
374	480	extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
375		-extern phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
376		- dma_addr_t iova);
377		-extern bool iommu_is_iova_coherent(struct iommu_domain *domain,
378		- dma_addr_t iova);
379	481	extern void iommu_set_fault_handler(struct iommu_domain *domain,
380	482	iommu_fault_handler_t handler, void *token);
381	483
382	484	extern void iommu_get_resv_regions(struct device dev, struct list_head list);
383	485	extern void iommu_put_resv_regions(struct device dev, struct list_head list);
384		-extern int iommu_request_dm_for_dev(struct device *dev);
	486	+extern void generic_iommu_put_resv_regions(struct device *dev,
	487	+ struct list_head *list);
	488	+extern void iommu_set_default_passthrough(bool cmd_line);
	489	+extern void iommu_set_default_translated(bool cmd_line);
	490	+extern bool iommu_default_passthrough(void);
385	491	extern struct iommu_resv_region *
386	492	iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot,
387	493	enum iommu_resv_type type);
..	..	@@ -410,8 +516,18 @@
410	516	struct notifier_block *nb);
411	517	extern int iommu_group_unregister_notifier(struct iommu_group *group,
412	518	struct notifier_block *nb);
	519	+extern int iommu_register_device_fault_handler(struct device *dev,
	520	+ iommu_dev_fault_handler_t handler,
	521	+ void *data);
	522	+
	523	+extern int iommu_unregister_device_fault_handler(struct device *dev);
	524	+
	525	+extern int iommu_report_device_fault(struct device *dev,
	526	+ struct iommu_fault_event *evt);
	527	+extern int iommu_page_response(struct device *dev,
	528	+ struct iommu_page_response *msg);
	529	+
413	530	extern int iommu_group_id(struct iommu_group *group);
414		-extern struct iommu_group iommu_group_get_for_dev(struct device dev);
415	531	extern struct iommu_domain iommu_group_default_domain(struct iommu_group );
416	532
417	533	extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr,
..	..	@@ -425,79 +541,82 @@
425	541	int prot);
426	542	extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr);
427	543
428		-extern uint64_t iommu_iova_to_pte(struct iommu_domain *domain,
429		- dma_addr_t iova);
430		-
431	544	extern int report_iommu_fault(struct iommu_domain domain, struct device dev,
432	545	unsigned long iova, int flags);
433	546
434		-static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
	547	+static inline void iommu_flush_iotlb_all(struct iommu_domain *domain)
435	548	{
436	549	if (domain->ops->flush_iotlb_all)
437	550	domain->ops->flush_iotlb_all(domain);
438	551	}
439	552
440		-static inline void iommu_tlb_range_add(struct iommu_domain *domain,
441		- unsigned long iova, size_t size)
442		-{
443		- if (domain->ops->iotlb_range_add)
444		- domain->ops->iotlb_range_add(domain, iova, size);
445		-}
446		-
447		-static inline void iommu_tlb_sync(struct iommu_domain *domain)
	553	+static inline void iommu_iotlb_sync(struct iommu_domain *domain,
	554	+ struct iommu_iotlb_gather *iotlb_gather)
448	555	{
449	556	if (domain->ops->iotlb_sync)
450		- domain->ops->iotlb_sync(domain);
	557	+ domain->ops->iotlb_sync(domain, iotlb_gather);
	558	+
	559	+ iommu_iotlb_gather_init(iotlb_gather);
451	560	}
452	561
453		-extern void iommu_trigger_fault(struct iommu_domain *domain,
454		- unsigned long flags);
	562	+static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
	563	+ struct iommu_iotlb_gather *gather,
	564	+ unsigned long iova, size_t size)
	565	+{
	566	+ unsigned long start = iova, end = start + size - 1;
455	567
456		-extern unsigned long iommu_reg_read(struct iommu_domain *domain,
457		- unsigned long offset);
458		-extern void iommu_reg_write(struct iommu_domain *domain, unsigned long offset,
459		- unsigned long val);
	568	+ /*
	569	+ * If the new page is disjoint from the current range or is mapped at
	570	+ * a different granularity, then sync the TLB so that the gather
	571	+ * structure can be rewritten.
	572	+ */
	573	+ if (gather->pgsize != size \|\|
	574	+ end + 1 < gather->start \|\| start > gather->end + 1) {
	575	+ if (gather->pgsize)
	576	+ iommu_iotlb_sync(domain, gather);
	577	+ gather->pgsize = size;
	578	+ }
	579	+
	580	+ if (gather->end < end)
	581	+ gather->end = end;
	582	+
	583	+ if (gather->start > start)
	584	+ gather->start = start;
	585	+}
460	586
461	587	/* PCI device grouping function */
462	588	extern struct iommu_group pci_device_group(struct device dev);
463	589	/* Generic device grouping function */
464	590	extern struct iommu_group generic_device_group(struct device dev);
465		-extern void rk_iommu_mask_irq(struct device *dev);
466		-extern void rk_iommu_unmask_irq(struct device *dev);
467		-
468		-static inline void iommu_tlbiall(struct iommu_domain *domain)
469		-{
470		- if (domain->ops->tlbi_domain)
471		- domain->ops->tlbi_domain(domain);
472		-}
473		-
474		-static inline int iommu_enable_config_clocks(struct iommu_domain *domain)
475		-{
476		- if (domain->ops->enable_config_clocks)
477		- return domain->ops->enable_config_clocks(domain);
478		- return 0;
479		-}
480		-
481		-static inline void iommu_disable_config_clocks(struct iommu_domain *domain)
482		-{
483		- if (domain->ops->disable_config_clocks)
484		- domain->ops->disable_config_clocks(domain);
485		-}
	591	+/* FSL-MC device grouping function */
	592	+struct iommu_group fsl_mc_device_group(struct device dev);
486	593
487	594	/**
488	595	* struct iommu_fwspec - per-device IOMMU instance data
489	596	* @ops: ops for this device's IOMMU
490	597	* @iommu_fwnode: firmware handle for this device's IOMMU
491	598	* @iommu_priv: IOMMU driver private data for this device
	599	+ * @num_pasid_bits: number of PASID bits supported by this device
492	600	* @num_ids: number of associated device IDs
493	601	* @ids: IDs which this device may present to the IOMMU
494	602	*/
495	603	struct iommu_fwspec {
496	604	const struct iommu_ops *ops;
497	605	struct fwnode_handle *iommu_fwnode;
498		- void *iommu_priv;
	606	+ u32 flags;
	607	+ u32 num_pasid_bits;
499	608	unsigned int num_ids;
500		- u32 ids[1];
	609	+ u32 ids[];
	610	+};
	611	+
	612	+/* ATS is supported */
	613	+#define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0)
	614	+
	615	+/**
	616	+ * struct iommu_sva - handle to a device-mm bond
	617	+ */
	618	+struct iommu_sva {
	619	+ struct device *dev;
501	620	};
502	621
503	622	int iommu_fwspec_init(struct device dev, struct fwnode_handle iommu_fwnode,
..	..	@@ -505,7 +624,50 @@
505	624	void iommu_fwspec_free(struct device *dev);
506	625	int iommu_fwspec_add_ids(struct device dev, u32 ids, int num_ids);
507	626	const struct iommu_ops iommu_ops_from_fwnode(struct fwnode_handle fwnode);
508		-int iommu_is_available(struct device *dev);
	627	+
	628	+static inline struct iommu_fwspec dev_iommu_fwspec_get(struct device dev)
	629	+{
	630	+ if (dev->iommu)
	631	+ return dev->iommu->fwspec;
	632	+ else
	633	+ return NULL;
	634	+}
	635	+
	636	+static inline void dev_iommu_fwspec_set(struct device *dev,
	637	+ struct iommu_fwspec *fwspec)
	638	+{
	639	+ dev->iommu->fwspec = fwspec;
	640	+}
	641	+
	642	+static inline void dev_iommu_priv_get(struct device dev)
	643	+{
	644	+ if (dev->iommu)
	645	+ return dev->iommu->priv;
	646	+ else
	647	+ return NULL;
	648	+}
	649	+
	650	+static inline void dev_iommu_priv_set(struct device dev, void priv)
	651	+{
	652	+ dev->iommu->priv = priv;
	653	+}
	654	+
	655	+int iommu_probe_device(struct device *dev);
	656	+void iommu_release_device(struct device *dev);
	657	+
	658	+bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features f);
	659	+int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
	660	+int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
	661	+bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f);
	662	+int iommu_aux_attach_device(struct iommu_domain domain, struct device dev);
	663	+void iommu_aux_detach_device(struct iommu_domain domain, struct device dev);
	664	+int iommu_aux_get_pasid(struct iommu_domain domain, struct device dev);
	665	+
	666	+struct iommu_sva iommu_sva_bind_device(struct device dev,
	667	+ struct mm_struct *mm,
	668	+ void *drvdata);
	669	+void iommu_sva_unbind_device(struct iommu_sva *handle);
	670	+u32 iommu_sva_get_pasid(struct iommu_sva *handle);
509	671
510	672	#else /* CONFIG_IOMMU_API */
511	673
..	..	@@ -513,6 +675,8 @@
513	675	struct iommu_group {};
514	676	struct iommu_fwspec {};
515	677	struct iommu_device {};
	678	+struct iommu_fault_param {};
	679	+struct iommu_iotlb_gather {};
516	680
517	681	static inline bool iommu_present(struct bus_type *bus)
518	682	{
..	..	@@ -560,6 +724,13 @@
560	724	return -ENODEV;
561	725	}
562	726
	727	+static inline int iommu_map_atomic(struct iommu_domain *domain,
	728	+ unsigned long iova, phys_addr_t paddr,
	729	+ size_t size, int prot)
	730	+{
	731	+ return -ENODEV;
	732	+}
	733	+
563	734	static inline size_t iommu_unmap(struct iommu_domain *domain,
564	735	unsigned long iova, size_t size)
565	736	{
..	..	@@ -567,7 +738,8 @@
567	738	}
568	739
569	740	static inline size_t iommu_unmap_fast(struct iommu_domain *domain,
570		- unsigned long iova, int gfp_order)
	741	+ unsigned long iova, int gfp_order,
	742	+ struct iommu_iotlb_gather *iotlb_gather)
571	743	{
572	744	return 0;
573	745	}
..	..	@@ -579,16 +751,19 @@
579	751	return 0;
580	752	}
581	753
582		-static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
	754	+static inline size_t iommu_map_sg_atomic(struct iommu_domain *domain,
	755	+ unsigned long iova, struct scatterlist *sg,
	756	+ unsigned int nents, int prot)
	757	+{
	758	+ return 0;
	759	+}
	760	+
	761	+static inline void iommu_flush_iotlb_all(struct iommu_domain *domain)
583	762	{
584	763	}
585	764
586		-static inline void iommu_tlb_range_add(struct iommu_domain *domain,
587		- unsigned long iova, size_t size)
588		-{
589		-}
590		-
591		-static inline void iommu_tlb_sync(struct iommu_domain *domain)
	765	+static inline void iommu_iotlb_sync(struct iommu_domain *domain,
	766	+ struct iommu_iotlb_gather *iotlb_gather)
592	767	{
593	768	}
594	769
..	..	@@ -605,18 +780,6 @@
605	780	}
606	781
607	782	static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
608		-{
609		- return 0;
610		-}
611		-
612		-static inline phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
613		- dma_addr_t iova)
614		-{
615		- return 0;
616		-}
617		-
618		-static inline bool iommu_is_iova_coherent(struct iommu_domain *domain,
619		- dma_addr_t iova)
620	783	{
621	784	return 0;
622	785	}
..	..	@@ -642,9 +805,17 @@
642	805	return -ENODEV;
643	806	}
644	807
645		-static inline int iommu_request_dm_for_dev(struct device *dev)
	808	+static inline void iommu_set_default_passthrough(bool cmd_line)
646	809	{
647		- return -ENODEV;
	810	+}
	811	+
	812	+static inline void iommu_set_default_translated(bool cmd_line)
	813	+{
	814	+}
	815	+
	816	+static inline bool iommu_default_passthrough(void)
	817	+{
	818	+ return true;
648	819	}
649	820
650	821	static inline int iommu_attach_group(struct iommu_domain *domain,
..	..	@@ -718,6 +889,31 @@
718	889	return 0;
719	890	}
720	891
	892	+static inline
	893	+int iommu_register_device_fault_handler(struct device *dev,
	894	+ iommu_dev_fault_handler_t handler,
	895	+ void *data)
	896	+{
	897	+ return -ENODEV;
	898	+}
	899	+
	900	+static inline int iommu_unregister_device_fault_handler(struct device *dev)
	901	+{
	902	+ return 0;
	903	+}
	904	+
	905	+static inline
	906	+int iommu_report_device_fault(struct device dev, struct iommu_fault_event evt)
	907	+{
	908	+ return -ENODEV;
	909	+}
	910	+
	911	+static inline int iommu_page_response(struct device *dev,
	912	+ struct iommu_page_response *msg)
	913	+{
	914	+ return -ENODEV;
	915	+}
	916	+
721	917	static inline int iommu_group_id(struct iommu_group *group)
722	918	{
723	919	return -ENODEV;
..	..	@@ -755,6 +951,16 @@
755	951	return NULL;
756	952	}
757	953
	954	+static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
	955	+{
	956	+}
	957	+
	958	+static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
	959	+ struct iommu_iotlb_gather *gather,
	960	+ unsigned long iova, size_t size)
	961	+{
	962	+}
	963	+
758	964	static inline void iommu_device_unregister(struct iommu_device *iommu)
759	965	{
760	966	}
..	..	@@ -777,35 +983,6 @@
777	983	}
778	984
779	985	static inline void iommu_device_unlink(struct device dev, struct device link)
780		-{
781		-}
782		-
783		-static inline void iommu_trigger_fault(struct iommu_domain *domain,
784		- unsigned long flags)
785		-{
786		-}
787		-
788		-static inline unsigned long iommu_reg_read(struct iommu_domain *domain,
789		- unsigned long offset)
790		-{
791		- return 0;
792		-}
793		-
794		-static inline void iommu_reg_write(struct iommu_domain *domain,
795		- unsigned long val, unsigned long offset)
796		-{
797		-}
798		-
799		-static inline void iommu_tlbiall(struct iommu_domain *domain)
800		-{
801		-}
802		-
803		-static inline int iommu_enable_config_clocks(struct iommu_domain *domain)
804		-{
805		- return 0;
806		-}
807		-
808		-static inline void iommu_disable_config_clocks(struct iommu_domain *domain)
809	986	{
810	987	}
811	988
..	..	@@ -832,20 +1009,111 @@
832	1009	return NULL;
833	1010	}
834	1011
835		-static inline int iommu_is_available(struct device *dev)
	1012	+static inline bool
	1013	+iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat)
	1014	+{
	1015	+ return false;
	1016	+}
	1017	+
	1018	+static inline bool
	1019	+iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat)
	1020	+{
	1021	+ return false;
	1022	+}
	1023	+
	1024	+static inline int
	1025	+iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
836	1026	{
837	1027	return -ENODEV;
838	1028	}
839	1029
840		-static inline void rk_iommu_mask_irq(struct device *dev)
	1030	+static inline int
	1031	+iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
	1032	+{
	1033	+ return -ENODEV;
	1034	+}
	1035	+
	1036	+static inline int
	1037	+iommu_aux_attach_device(struct iommu_domain domain, struct device dev)
	1038	+{
	1039	+ return -ENODEV;
	1040	+}
	1041	+
	1042	+static inline void
	1043	+iommu_aux_detach_device(struct iommu_domain domain, struct device dev)
841	1044	{
842	1045	}
843	1046
844		-static inline void rk_iommu_unmask_irq(struct device *dev)
	1047	+static inline int
	1048	+iommu_aux_get_pasid(struct iommu_domain domain, struct device dev)
	1049	+{
	1050	+ return -ENODEV;
	1051	+}
	1052	+
	1053	+static inline struct iommu_sva *
	1054	+iommu_sva_bind_device(struct device dev, struct mm_struct mm, void *drvdata)
	1055	+{
	1056	+ return NULL;
	1057	+}
	1058	+
	1059	+static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
845	1060	{
846	1061	}
	1062	+
	1063	+static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
	1064	+{
	1065	+ return IOMMU_PASID_INVALID;
	1066	+}
	1067	+
	1068	+static inline int
	1069	+iommu_uapi_cache_invalidate(struct iommu_domain *domain,
	1070	+ struct device *dev,
	1071	+ struct iommu_cache_invalidate_info *inv_info)
	1072	+{
	1073	+ return -ENODEV;
	1074	+}
	1075	+
	1076	+static inline int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain,
	1077	+ struct device dev, void __user udata)
	1078	+{
	1079	+ return -ENODEV;
	1080	+}
	1081	+
	1082	+static inline int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain,
	1083	+ struct device dev, void __user udata)
	1084	+{
	1085	+ return -ENODEV;
	1086	+}
	1087	+
	1088	+static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
	1089	+ struct device *dev,
	1090	+ ioasid_t pasid)
	1091	+{
	1092	+ return -ENODEV;
	1093	+}
	1094	+
	1095	+static inline struct iommu_fwspec dev_iommu_fwspec_get(struct device dev)
	1096	+{
	1097	+ return NULL;
	1098	+}
847	1099	#endif /* CONFIG_IOMMU_API */
848	1100
	1101	+/**
	1102	+ * iommu_map_sgtable - Map the given buffer to the IOMMU domain
	1103	+ * @domain: The IOMMU domain to perform the mapping
	1104	+ * @iova: The start address to map the buffer
	1105	+ * @sgt: The sg_table object describing the buffer
	1106	+ * @prot: IOMMU protection bits
	1107	+ *
	1108	+ * Creates a mapping at @iova for the buffer described by a scatterlist
	1109	+ * stored in the given sg_table object in the provided IOMMU domain.
	1110	+ */
	1111	+static inline size_t iommu_map_sgtable(struct iommu_domain *domain,
	1112	+ unsigned long iova, struct sg_table *sgt, int prot)
	1113	+{
	1114	+ return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot);
	1115	+}
	1116	+
849	1117	#ifdef CONFIG_IOMMU_DEBUGFS
850	1118	extern struct dentry *iommu_debugfs_dir;
851	1119	void iommu_debugfs_setup(void);