~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,19 +1,7 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
1	2	/*
2	3	* Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3	4	* Author: Joerg Roedel <joerg.roedel@amd.com>
4		- *
5		- * This program is free software; you can redistribute it and/or modify it
6		- * under the terms of the GNU General Public License version 2 as published
7		- * by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program; if not, write to the Free Software
16		- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17	5	*/
18	6
19	7	#ifndef __LINUX_IOMMU_H
..	..	@@ -25,6 +13,8 @@
25	13	#include <linux/errno.h>
26	14	#include <linux/err.h>
27	15	#include <linux/of.h>
	16	+#include <linux/ioasid.h>
	17	+#include <uapi/linux/iommu.h>
28	18
29	19	#define IOMMU_READ (1 << 0)
30	20	#define IOMMU_WRITE (1 << 1)
..	..	@@ -41,14 +31,18 @@
41	31	* if the IOMMU page table format is equivalent.
42	32	*/
43	33	#define IOMMU_PRIV (1 << 5)
44		-/* Use upstream device's bus attribute */
45		-#define IOMMU_USE_UPSTREAM_HINT (1 << 6)
46		-
47		-/* Use upstream device's bus attribute with no write-allocate cache policy */
48		-#define IOMMU_USE_LLC_NWA (1 << 7)
49		-
50		-/* For shoting entire IOMMU tlb once */
51		-#define IOMMU_TLB_SHOT_ENTIRE (1 << 6)
	34	+/*
	35	+ * Allow caching in a transparent outer level of cache, also known as
	36	+ * the last-level or system cache, with a read/write allocation policy.
	37	+ * Does not depend on IOMMU_CACHE. Incompatible with IOMMU_SYS_CACHE_NWA.
	38	+ */
	39	+#define IOMMU_SYS_CACHE (1 << 6)
	40	+/*
	41	+ * Allow caching in a transparent outer level of cache, also known as
	42	+ * the last-level or system cache, with a read allocation policy.
	43	+ * Does not depend on IOMMU_CACHE. Incompatible with IOMMU_SYS_CACHE.
	44	+ */
	45	+#define IOMMU_SYS_CACHE_NWA (1 << 7)
52	46
53	47	struct iommu_ops;
54	48	struct iommu_group;
..	..	@@ -56,26 +50,21 @@
56	50	struct device;
57	51	struct iommu_domain;
58	52	struct notifier_block;
	53	+struct iommu_sva;
	54	+struct iommu_fault_event;
59	55
60	56	/* iommu fault flags */
61		-#define IOMMU_FAULT_READ (1 << 0)
62		-#define IOMMU_FAULT_WRITE (1 << 1)
63		-#define IOMMU_FAULT_TRANSLATION (1 << 2)
64		-#define IOMMU_FAULT_PERMISSION (1 << 3)
65		-#define IOMMU_FAULT_EXTERNAL (1 << 4)
66		-#define IOMMU_FAULT_TRANSACTION_STALLED (1 << 5)
	57	+#define IOMMU_FAULT_READ 0x0
	58	+#define IOMMU_FAULT_WRITE 0x1
67	59
68	60	typedef int (iommu_fault_handler_t)(struct iommu_domain ,
69	61	struct device , unsigned long, int, void );
	62	+typedef int (iommu_dev_fault_handler_t)(struct iommu_fault , void *);
70	63
71	64	struct iommu_domain_geometry {
72	65	dma_addr_t aperture_start; /* First address that can be mapped */
73	66	dma_addr_t aperture_end; /* Last address that can be mapped */
74	67	bool force_aperture; /* DMA only allowed in mappable range? */
75		-};
76		-
77		-struct iommu_pgtbl_info {
78		- void *ops;
79	68	};
80	69
81	70	/* Domain feature flags */
..	..	@@ -102,8 +91,6 @@
102	91	#define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING \| \
103	92	__IOMMU_DOMAIN_DMA_API)
104	93
105		-
106		-#define IOMMU_DOMAIN_NAME_LEN 32
107	94	struct iommu_domain {
108	95	unsigned type;
109	96	const struct iommu_ops *ops;
..	..	@@ -112,8 +99,6 @@
112	99	void *handler_token;
113	100	struct iommu_domain_geometry geometry;
114	101	void *iova_cookie;
115		- bool is_debug_domain;
116		- char name[IOMMU_DOMAIN_NAME_LEN];
117	102	};
118	103
119	104	enum iommu_cap {
..	..	@@ -134,11 +119,6 @@
134	119	* DOMAIN_ATTR_FSL_PAMUV1 corresponds to the above mentioned contraints.
135	120	* The caller can invoke iommu_domain_get_attr to check if the underlying
136	121	* iommu implementation supports these constraints.
137		- *
138		- * DOMAIN_ATTR_NO_CFRE
139		- * Some bus implementations may enter a bad state if iommu reports an error
140		- * on context fault. As context faults are not always fatal, this must be
141		- * avoided.
142	122	*/
143	123
144	124	enum iommu_attr {
..	..	@@ -149,27 +129,7 @@
149	129	DOMAIN_ATTR_FSL_PAMU_ENABLE,
150	130	DOMAIN_ATTR_FSL_PAMUV1,
151	131	DOMAIN_ATTR_NESTING, /* two stages of translation */
152		- DOMAIN_ATTR_PT_BASE_ADDR,
153		- DOMAIN_ATTR_CONTEXT_BANK,
154		- DOMAIN_ATTR_DYNAMIC,
155		- DOMAIN_ATTR_TTBR0,
156		- DOMAIN_ATTR_CONTEXTIDR,
157		- DOMAIN_ATTR_PROCID,
158		- DOMAIN_ATTR_NON_FATAL_FAULTS,
159		- DOMAIN_ATTR_S1_BYPASS,
160		- DOMAIN_ATTR_ATOMIC,
161		- DOMAIN_ATTR_SECURE_VMID,
162		- DOMAIN_ATTR_FAST,
163		- DOMAIN_ATTR_PGTBL_INFO,
164		- DOMAIN_ATTR_USE_UPSTREAM_HINT,
165		- DOMAIN_ATTR_EARLY_MAP,
166		- DOMAIN_ATTR_PAGE_TABLE_IS_COHERENT,
167		- DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT,
168		- DOMAIN_ATTR_BITMAP_IOVA_ALLOCATOR,
169		- DOMAIN_ATTR_USE_LLC_NWA,
170		- DOMAIN_ATTR_FAULT_MODEL_NO_CFRE,
171		- DOMAIN_ATTR_FAULT_MODEL_NO_STALL,
172		- DOMAIN_ATTR_FAULT_MODEL_HUPCF,
	132	+ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
173	133	DOMAIN_ATTR_MAX,
174	134	};
175	135
..	..	@@ -177,6 +137,12 @@
177	137	enum iommu_resv_type {
178	138	/* Memory regions which must be mapped 1:1 at all times */
179	139	IOMMU_RESV_DIRECT,
	140	+ /*
	141	+ * Memory regions which are advertised to be 1:1 but are
	142	+ * commonly considered relaxable in some conditions,
	143	+ * for instance in device assignment use case (USB, Graphics)
	144	+ */
	145	+ IOMMU_RESV_DIRECT_RELAXABLE,
180	146	/* Arbitrary "never map this or give it to a device" address ranges */
181	147	IOMMU_RESV_RESERVED,
182	148	/* Hardware MSI region (untranslated) */
..	..	@@ -201,9 +167,32 @@
201	167	enum iommu_resv_type type;
202	168	};
203	169
204		-extern struct dentry *iommu_debugfs_top;
	170	+/* Per device IOMMU features */
	171	+enum iommu_dev_features {
	172	+ IOMMU_DEV_FEAT_AUX, /* Aux-domain feature */
	173	+ IOMMU_DEV_FEAT_SVA, /* Shared Virtual Addresses */
	174	+};
	175	+
	176	+#define IOMMU_PASID_INVALID (-1U)
205	177
206	178	#ifdef CONFIG_IOMMU_API
	179	+
	180	+/**
	181	+ * struct iommu_iotlb_gather - Range information for a pending IOTLB flush
	182	+ *
	183	+ * @start: IOVA representing the start of the range to be flushed
	184	+ * @end: IOVA representing the end of the range to be flushed (inclusive)
	185	+ * @pgsize: The interval at which to perform the flush
	186	+ *
	187	+ * This structure is intended to be updated by multiple calls to the
	188	+ * ->unmap() function in struct iommu_ops before eventually being passed
	189	+ * into ->iotlb_sync().
	190	+ */
	191	+struct iommu_iotlb_gather {
	192	+ unsigned long start;
	193	+ unsigned long end;
	194	+ size_t pgsize;
	195	+};
207	196
208	197	/**
209	198	* struct iommu_ops - iommu ops and capabilities
..	..	@@ -213,17 +202,21 @@
213	202	* @attach_dev: attach device to an iommu domain
214	203	* @detach_dev: detach device from an iommu domain
215	204	* @map: map a physically contiguous memory region to an iommu domain
	205	+ * @map_pages: map a physically contiguous set of pages of the same size to
	206	+ * an iommu domain.
	207	+ * @map_sg: map a scatter-gather list of physically contiguous chunks to
	208	+ * an iommu domain.
216	209	* @unmap: unmap a physically contiguous memory region from an iommu domain
217		- * @map_sg: map a scatter-gather list of physically contiguous memory chunks
218		- * to an iommu domain
219		- * @flush_tlb_all: Synchronously flush all hardware TLBs for this domain
220		- * @tlb_range_add: Add a given iova range to the flush queue for this domain
221		- * @tlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
	210	+ * @unmap_pages: unmap a number of pages of the same size from an iommu domain
	211	+ * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
	212	+ * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
	213	+ * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
222	214	* queue
223	215	* @iova_to_phys: translate iova to physical address
224		- * @iova_to_phys_hard: translate iova to physical address using IOMMU hardware
225		- * @add_device: add device to iommu grouping
226		- * @remove_device: remove device from iommu grouping
	216	+ * @probe_device: Add device to iommu driver handling
	217	+ * @release_device: Remove device from iommu driver handling
	218	+ * @probe_finalize: Do final setup work after the device is added to an IOMMU
	219	+ * group and attached to the groups domain
227	220	* @device_group: find iommu group for a particular device
228	221	* @domain_get_attr: Query domain attributes
229	222	* @domain_set_attr: Change domain attributes
..	..	@@ -232,14 +225,27 @@
232	225	* @apply_resv_region: Temporary helper call-back for iova reserved ranges
233	226	* @domain_window_enable: Configure and enable a particular window for a domain
234	227	* @domain_window_disable: Disable a particular window for a domain
235		- * @domain_set_windows: Set the number of windows for a domain
236		- * @domain_get_windows: Return the number of windows for a domain
237	228	* @of_xlate: add OF master IDs to iommu grouping
	229	+ * @is_attach_deferred: Check if domain attach should be deferred from iommu
	230	+ * driver init to device driver init (default no)
	231	+ * @dev_has/enable/disable_feat: per device entries to check/enable/disable
	232	+ * iommu specific features.
	233	+ * @dev_feat_enabled: check enabled feature
	234	+ * @aux_attach/detach_dev: aux-domain specific attach/detach entries.
	235	+ * @aux_get_pasid: get the pasid given an aux-domain
	236	+ * @sva_bind: Bind process address space to device
	237	+ * @sva_unbind: Unbind process address space from device
	238	+ * @sva_get_pasid: Get PASID associated to a SVA handle
	239	+ * @page_response: handle page request response
	240	+ * @cache_invalidate: invalidate translation caches
	241	+ * @sva_bind_gpasid: bind guest pasid and mm
	242	+ * @sva_unbind_gpasid: unbind guest pasid and mm
	243	+ * @def_domain_type: device default domain type, return value:
	244	+ * - IOMMU_DOMAIN_IDENTITY: must use an identity domain
	245	+ * - IOMMU_DOMAIN_DMA: must use a dma domain
	246	+ * - 0: use the default setting
238	247	* @pgsize_bitmap: bitmap of all possible supported page sizes
239		- * @trigger_fault: trigger a fault on the device attached to an iommu domain
240		- * @tlbi_domain: Invalidate all TLBs covering an iommu domain
241		- * @enable_config_clocks: Enable all config clocks for this domain's IOMMU
242		- * @disable_config_clocks: Disable all config clocks for this domain's IOMMU
	248	+ * @owner: Driver module providing these ops
243	249	*/
244	250	struct iommu_ops {
245	251	bool (*capable)(enum iommu_cap);
..	..	@@ -251,20 +257,27 @@
251	257	int (attach_dev)(struct iommu_domain domain, struct device *dev);
252	258	void (detach_dev)(struct iommu_domain domain, struct device *dev);
253	259	int (map)(struct iommu_domain domain, unsigned long iova,
254		- phys_addr_t paddr, size_t size, int prot);
	260	+ phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
	261	+ int (map_pages)(struct iommu_domain domain, unsigned long iova,
	262	+ phys_addr_t paddr, size_t pgsize, size_t pgcount,
	263	+ int prot, gfp_t gfp, size_t *mapped);
	264	+ int (map_sg)(struct iommu_domain domain, unsigned long iova,
	265	+ struct scatterlist *sg, unsigned int nents, int prot,
	266	+ gfp_t gfp, size_t *mapped);
255	267	size_t (unmap)(struct iommu_domain domain, unsigned long iova,
256		- size_t size);
257		- size_t (map_sg)(struct iommu_domain domain, unsigned long iova,
258		- struct scatterlist *sg, unsigned int nents, int prot);
	268	+ size_t size, struct iommu_iotlb_gather *iotlb_gather);
	269	+ size_t (unmap_pages)(struct iommu_domain domain, unsigned long iova,
	270	+ size_t pgsize, size_t pgcount,
	271	+ struct iommu_iotlb_gather *iotlb_gather);
259	272	void (flush_iotlb_all)(struct iommu_domain domain);
260		- void (iotlb_range_add)(struct iommu_domain domain,
261		- unsigned long iova, size_t size);
262		- void (iotlb_sync)(struct iommu_domain domain);
	273	+ void (iotlb_sync_map)(struct iommu_domain domain, unsigned long iova,
	274	+ size_t size);
	275	+ void (iotlb_sync)(struct iommu_domain domain,
	276	+ struct iommu_iotlb_gather *iotlb_gather);
263	277	phys_addr_t (iova_to_phys)(struct iommu_domain domain, dma_addr_t iova);
264		- phys_addr_t (iova_to_phys_hard)(struct iommu_domain domain,
265		- dma_addr_t iova);
266		- int (add_device)(struct device dev);
267		- void (remove_device)(struct device dev);
	278	+ struct iommu_device (probe_device)(struct device *dev);
	279	+ void (release_device)(struct device dev);
	280	+ void (probe_finalize)(struct device dev);
268	281	struct iommu_group (device_group)(struct device *dev);
269	282	int (domain_get_attr)(struct iommu_domain domain,
270	283	enum iommu_attr attr, void *data);
..	..	@@ -282,22 +295,40 @@
282	295	int (domain_window_enable)(struct iommu_domain domain, u32 wnd_nr,
283	296	phys_addr_t paddr, u64 size, int prot);
284	297	void (domain_window_disable)(struct iommu_domain domain, u32 wnd_nr);
285		- /* Set the number of windows per domain */
286		- int (domain_set_windows)(struct iommu_domain domain, u32 w_count);
287		- /* Get the number of windows per domain */
288		- u32 (domain_get_windows)(struct iommu_domain domain);
289		- void (trigger_fault)(struct iommu_domain domain, unsigned long flags);
290		- void (tlbi_domain)(struct iommu_domain domain);
291		- int (enable_config_clocks)(struct iommu_domain domain);
292		- void (disable_config_clocks)(struct iommu_domain domain);
293		- uint64_t (iova_to_pte)(struct iommu_domain domain,
294		- dma_addr_t iova);
295	298
296	299	int (of_xlate)(struct device dev, struct of_phandle_args *args);
297	300	bool (is_attach_deferred)(struct iommu_domain domain, struct device *dev);
298	301
299		- bool (is_iova_coherent)(struct iommu_domain domain, dma_addr_t iova);
	302	+ /* Per device IOMMU features */
	303	+ bool (dev_has_feat)(struct device dev, enum iommu_dev_features f);
	304	+ bool (dev_feat_enabled)(struct device dev, enum iommu_dev_features f);
	305	+ int (dev_enable_feat)(struct device dev, enum iommu_dev_features f);
	306	+ int (dev_disable_feat)(struct device dev, enum iommu_dev_features f);
	307	+
	308	+ /* Aux-domain specific attach/detach entries */
	309	+ int (aux_attach_dev)(struct iommu_domain domain, struct device *dev);
	310	+ void (aux_detach_dev)(struct iommu_domain domain, struct device *dev);
	311	+ int (aux_get_pasid)(struct iommu_domain domain, struct device *dev);
	312	+
	313	+ struct iommu_sva (sva_bind)(struct device dev, struct mm_struct mm,
	314	+ void *drvdata);
	315	+ void (sva_unbind)(struct iommu_sva handle);
	316	+ u32 (sva_get_pasid)(struct iommu_sva handle);
	317	+
	318	+ int (page_response)(struct device dev,
	319	+ struct iommu_fault_event *evt,
	320	+ struct iommu_page_response *msg);
	321	+ int (cache_invalidate)(struct iommu_domain domain, struct device *dev,
	322	+ struct iommu_cache_invalidate_info *inv_info);
	323	+ int (sva_bind_gpasid)(struct iommu_domain domain,
	324	+ struct device dev, struct iommu_gpasid_bind_data data);
	325	+
	326	+ int (sva_unbind_gpasid)(struct device dev, u32 pasid);
	327	+
	328	+ int (def_domain_type)(struct device dev);
	329	+
300	330	unsigned long pgsize_bitmap;
	331	+ struct module *owner;
301	332	};
302	333
303	334	/**
..	..	@@ -314,6 +345,53 @@
314	345	struct device *dev;
315	346	};
316	347
	348	+/**
	349	+ * struct iommu_fault_event - Generic fault event
	350	+ *
	351	+ * Can represent recoverable faults such as a page requests or
	352	+ * unrecoverable faults such as DMA or IRQ remapping faults.
	353	+ *
	354	+ * @fault: fault descriptor
	355	+ * @list: pending fault event list, used for tracking responses
	356	+ */
	357	+struct iommu_fault_event {
	358	+ struct iommu_fault fault;
	359	+ struct list_head list;
	360	+};
	361	+
	362	+/**
	363	+ * struct iommu_fault_param - per-device IOMMU fault data
	364	+ * @handler: Callback function to handle IOMMU faults at device level
	365	+ * @data: handler private data
	366	+ * @faults: holds the pending faults which needs response
	367	+ * @lock: protect pending faults list
	368	+ */
	369	+struct iommu_fault_param {
	370	+ iommu_dev_fault_handler_t handler;
	371	+ void *data;
	372	+ struct list_head faults;
	373	+ struct mutex lock;
	374	+};
	375	+
	376	+/**
	377	+ * struct dev_iommu - Collection of per-device IOMMU data
	378	+ *
	379	+ * @fault_param: IOMMU detected device fault reporting data
	380	+ * @fwspec: IOMMU fwspec data
	381	+ * @iommu_dev: IOMMU device this device is linked to
	382	+ * @priv: IOMMU Driver private data
	383	+ *
	384	+ * TODO: migrate other per device data pointers under iommu_dev_data, e.g.
	385	+ * struct iommu_group *iommu_group;
	386	+ */
	387	+struct dev_iommu {
	388	+ struct mutex lock;
	389	+ struct iommu_fault_param *fault_param;
	390	+ struct iommu_fwspec *fwspec;
	391	+ struct iommu_device *iommu_dev;
	392	+ void *priv;
	393	+};
	394	+
317	395	int iommu_device_register(struct iommu_device *iommu);
318	396	void iommu_device_unregister(struct iommu_device *iommu);
319	397	int iommu_device_sysfs_add(struct iommu_device *iommu,
..	..	@@ -324,11 +402,18 @@
324	402	int iommu_device_link(struct iommu_device iommu, struct device link);
325	403	void iommu_device_unlink(struct iommu_device iommu, struct device link);
326	404
327		-static inline void iommu_device_set_ops(struct iommu_device *iommu,
328		- const struct iommu_ops *ops)
	405	+static inline void __iommu_device_set_ops(struct iommu_device *iommu,
	406	+ const struct iommu_ops *ops)
329	407	{
330	408	iommu->ops = ops;
331	409	}
	410	+
	411	+#define iommu_device_set_ops(iommu, ops) \
	412	+do { \
	413	+ struct iommu_ops __ops = (struct iommu_ops )(ops); \
	414	+ __ops->owner = THIS_MODULE; \
	415	+ __iommu_device_set_ops(iommu, __ops); \
	416	+} while (0)
332	417
333	418	static inline void iommu_device_set_fwnode(struct iommu_device *iommu,
334	419	struct fwnode_handle *fwnode)
..	..	@@ -341,6 +426,13 @@
341	426	return (struct iommu_device *)dev_get_drvdata(dev);
342	427	}
343	428
	429	+static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
	430	+{
	431	+ *gather = (struct iommu_iotlb_gather) {
	432	+ .start = ULONG_MAX,
	433	+ };
	434	+}
	435	+
344	436	#define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */
345	437	#define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */
346	438	#define IOMMU_GROUP_NOTIFY_BIND_DRIVER 3 /* Pre Driver bind */
..	..	@@ -349,6 +441,7 @@
349	441	#define IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER 6 /* Post Driver unbind */
350	442
351	443	extern int bus_set_iommu(struct bus_type bus, const struct iommu_ops ops);
	444	+extern int bus_iommu_probe(struct bus_type *bus);
352	445	extern bool iommu_present(struct bus_type *bus);
353	446	extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap);
354	447	extern struct iommu_domain iommu_domain_alloc(struct bus_type bus);
..	..	@@ -358,33 +451,43 @@
358	451	struct device *dev);
359	452	extern void iommu_detach_device(struct iommu_domain *domain,
360	453	struct device *dev);
	454	+extern int iommu_uapi_cache_invalidate(struct iommu_domain *domain,
	455	+ struct device *dev,
	456	+ void __user *uinfo);
	457	+
	458	+extern int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain,
	459	+ struct device dev, void __user udata);
	460	+extern int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain,
	461	+ struct device dev, void __user udata);
	462	+extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
	463	+ struct device *dev, ioasid_t pasid);
361	464	extern struct iommu_domain iommu_get_domain_for_dev(struct device dev);
362		-extern size_t iommu_pgsize(unsigned long pgsize_bitmap,
363		- unsigned long addr_merge, size_t size);
	465	+extern struct iommu_domain iommu_get_dma_domain(struct device dev);
364	466	extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
365	467	phys_addr_t paddr, size_t size, int prot);
	468	+extern int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
	469	+ phys_addr_t paddr, size_t size, int prot);
366	470	extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
367	471	size_t size);
368	472	extern size_t iommu_unmap_fast(struct iommu_domain *domain,
369		- unsigned long iova, size_t size);
	473	+ unsigned long iova, size_t size,
	474	+ struct iommu_iotlb_gather *iotlb_gather);
370	475	extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
371		- struct scatterlist *sg, unsigned int nents,
372		- int prot);
373		-extern size_t default_iommu_map_sg(struct iommu_domain *domain,
374		- unsigned long iova,
375		- struct scatterlist *sg, unsigned int nents,
376		- int prot);
	476	+ struct scatterlist *sg,unsigned int nents, int prot);
	477	+extern size_t iommu_map_sg_atomic(struct iommu_domain *domain,
	478	+ unsigned long iova, struct scatterlist *sg,
	479	+ unsigned int nents, int prot);
377	480	extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
378		-extern phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
379		- dma_addr_t iova);
380		-extern bool iommu_is_iova_coherent(struct iommu_domain *domain,
381		- dma_addr_t iova);
382	481	extern void iommu_set_fault_handler(struct iommu_domain *domain,
383	482	iommu_fault_handler_t handler, void *token);
384	483
385	484	extern void iommu_get_resv_regions(struct device dev, struct list_head list);
386	485	extern void iommu_put_resv_regions(struct device dev, struct list_head list);
387		-extern int iommu_request_dm_for_dev(struct device *dev);
	486	+extern void generic_iommu_put_resv_regions(struct device *dev,
	487	+ struct list_head *list);
	488	+extern void iommu_set_default_passthrough(bool cmd_line);
	489	+extern void iommu_set_default_translated(bool cmd_line);
	490	+extern bool iommu_default_passthrough(void);
388	491	extern struct iommu_resv_region *
389	492	iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot,
390	493	enum iommu_resv_type type);
..	..	@@ -413,8 +516,18 @@
413	516	struct notifier_block *nb);
414	517	extern int iommu_group_unregister_notifier(struct iommu_group *group,
415	518	struct notifier_block *nb);
	519	+extern int iommu_register_device_fault_handler(struct device *dev,
	520	+ iommu_dev_fault_handler_t handler,
	521	+ void *data);
	522	+
	523	+extern int iommu_unregister_device_fault_handler(struct device *dev);
	524	+
	525	+extern int iommu_report_device_fault(struct device *dev,
	526	+ struct iommu_fault_event *evt);
	527	+extern int iommu_page_response(struct device *dev,
	528	+ struct iommu_page_response *msg);
	529	+
416	530	extern int iommu_group_id(struct iommu_group *group);
417		-extern struct iommu_group iommu_group_get_for_dev(struct device dev);
418	531	extern struct iommu_domain iommu_group_default_domain(struct iommu_group );
419	532
420	533	extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr,
..	..	@@ -428,79 +541,82 @@
428	541	int prot);
429	542	extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr);
430	543
431		-extern uint64_t iommu_iova_to_pte(struct iommu_domain *domain,
432		- dma_addr_t iova);
433		-
434	544	extern int report_iommu_fault(struct iommu_domain domain, struct device dev,
435	545	unsigned long iova, int flags);
436	546
437		-static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
	547	+static inline void iommu_flush_iotlb_all(struct iommu_domain *domain)
438	548	{
439	549	if (domain->ops->flush_iotlb_all)
440	550	domain->ops->flush_iotlb_all(domain);
441	551	}
442	552
443		-static inline void iommu_tlb_range_add(struct iommu_domain *domain,
444		- unsigned long iova, size_t size)
445		-{
446		- if (domain->ops->iotlb_range_add)
447		- domain->ops->iotlb_range_add(domain, iova, size);
448		-}
449		-
450		-static inline void iommu_tlb_sync(struct iommu_domain *domain)
	553	+static inline void iommu_iotlb_sync(struct iommu_domain *domain,
	554	+ struct iommu_iotlb_gather *iotlb_gather)
451	555	{
452	556	if (domain->ops->iotlb_sync)
453		- domain->ops->iotlb_sync(domain);
	557	+ domain->ops->iotlb_sync(domain, iotlb_gather);
	558	+
	559	+ iommu_iotlb_gather_init(iotlb_gather);
454	560	}
455	561
456		-extern void iommu_trigger_fault(struct iommu_domain *domain,
457		- unsigned long flags);
	562	+static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
	563	+ struct iommu_iotlb_gather *gather,
	564	+ unsigned long iova, size_t size)
	565	+{
	566	+ unsigned long start = iova, end = start + size - 1;
458	567
459		-extern unsigned long iommu_reg_read(struct iommu_domain *domain,
460		- unsigned long offset);
461		-extern void iommu_reg_write(struct iommu_domain *domain, unsigned long offset,
462		- unsigned long val);
	568	+ /*
	569	+ * If the new page is disjoint from the current range or is mapped at
	570	+ * a different granularity, then sync the TLB so that the gather
	571	+ * structure can be rewritten.
	572	+ */
	573	+ if (gather->pgsize != size \|\|
	574	+ end + 1 < gather->start \|\| start > gather->end + 1) {
	575	+ if (gather->pgsize)
	576	+ iommu_iotlb_sync(domain, gather);
	577	+ gather->pgsize = size;
	578	+ }
	579	+
	580	+ if (gather->end < end)
	581	+ gather->end = end;
	582	+
	583	+ if (gather->start > start)
	584	+ gather->start = start;
	585	+}
463	586
464	587	/* PCI device grouping function */
465	588	extern struct iommu_group pci_device_group(struct device dev);
466	589	/* Generic device grouping function */
467	590	extern struct iommu_group generic_device_group(struct device dev);
468		-extern void rk_iommu_mask_irq(struct device *dev);
469		-extern void rk_iommu_unmask_irq(struct device *dev);
470		-
471		-static inline void iommu_tlbiall(struct iommu_domain *domain)
472		-{
473		- if (domain->ops->tlbi_domain)
474		- domain->ops->tlbi_domain(domain);
475		-}
476		-
477		-static inline int iommu_enable_config_clocks(struct iommu_domain *domain)
478		-{
479		- if (domain->ops->enable_config_clocks)
480		- return domain->ops->enable_config_clocks(domain);
481		- return 0;
482		-}
483		-
484		-static inline void iommu_disable_config_clocks(struct iommu_domain *domain)
485		-{
486		- if (domain->ops->disable_config_clocks)
487		- domain->ops->disable_config_clocks(domain);
488		-}
	591	+/* FSL-MC device grouping function */
	592	+struct iommu_group fsl_mc_device_group(struct device dev);
489	593
490	594	/**
491	595	* struct iommu_fwspec - per-device IOMMU instance data
492	596	* @ops: ops for this device's IOMMU
493	597	* @iommu_fwnode: firmware handle for this device's IOMMU
494	598	* @iommu_priv: IOMMU driver private data for this device
	599	+ * @num_pasid_bits: number of PASID bits supported by this device
495	600	* @num_ids: number of associated device IDs
496	601	* @ids: IDs which this device may present to the IOMMU
497	602	*/
498	603	struct iommu_fwspec {
499	604	const struct iommu_ops *ops;
500	605	struct fwnode_handle *iommu_fwnode;
501		- void *iommu_priv;
	606	+ u32 flags;
	607	+ u32 num_pasid_bits;
502	608	unsigned int num_ids;
503		- u32 ids[1];
	609	+ u32 ids[];
	610	+};
	611	+
	612	+/* ATS is supported */
	613	+#define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0)
	614	+
	615	+/**
	616	+ * struct iommu_sva - handle to a device-mm bond
	617	+ */
	618	+struct iommu_sva {
	619	+ struct device *dev;
504	620	};
505	621
506	622	int iommu_fwspec_init(struct device dev, struct fwnode_handle iommu_fwnode,
..	..	@@ -508,7 +624,50 @@
508	624	void iommu_fwspec_free(struct device *dev);
509	625	int iommu_fwspec_add_ids(struct device dev, u32 ids, int num_ids);
510	626	const struct iommu_ops iommu_ops_from_fwnode(struct fwnode_handle fwnode);
511		-int iommu_is_available(struct device *dev);
	627	+
	628	+static inline struct iommu_fwspec dev_iommu_fwspec_get(struct device dev)
	629	+{
	630	+ if (dev->iommu)
	631	+ return dev->iommu->fwspec;
	632	+ else
	633	+ return NULL;
	634	+}
	635	+
	636	+static inline void dev_iommu_fwspec_set(struct device *dev,
	637	+ struct iommu_fwspec *fwspec)
	638	+{
	639	+ dev->iommu->fwspec = fwspec;
	640	+}
	641	+
	642	+static inline void dev_iommu_priv_get(struct device dev)
	643	+{
	644	+ if (dev->iommu)
	645	+ return dev->iommu->priv;
	646	+ else
	647	+ return NULL;
	648	+}
	649	+
	650	+static inline void dev_iommu_priv_set(struct device dev, void priv)
	651	+{
	652	+ dev->iommu->priv = priv;
	653	+}
	654	+
	655	+int iommu_probe_device(struct device *dev);
	656	+void iommu_release_device(struct device *dev);
	657	+
	658	+bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features f);
	659	+int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
	660	+int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
	661	+bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f);
	662	+int iommu_aux_attach_device(struct iommu_domain domain, struct device dev);
	663	+void iommu_aux_detach_device(struct iommu_domain domain, struct device dev);
	664	+int iommu_aux_get_pasid(struct iommu_domain domain, struct device dev);
	665	+
	666	+struct iommu_sva iommu_sva_bind_device(struct device dev,
	667	+ struct mm_struct *mm,
	668	+ void *drvdata);
	669	+void iommu_sva_unbind_device(struct iommu_sva *handle);
	670	+u32 iommu_sva_get_pasid(struct iommu_sva *handle);
512	671
513	672	#else /* CONFIG_IOMMU_API */
514	673
..	..	@@ -516,6 +675,8 @@
516	675	struct iommu_group {};
517	676	struct iommu_fwspec {};
518	677	struct iommu_device {};
	678	+struct iommu_fault_param {};
	679	+struct iommu_iotlb_gather {};
519	680
520	681	static inline bool iommu_present(struct bus_type *bus)
521	682	{
..	..	@@ -563,6 +724,13 @@
563	724	return -ENODEV;
564	725	}
565	726
	727	+static inline int iommu_map_atomic(struct iommu_domain *domain,
	728	+ unsigned long iova, phys_addr_t paddr,
	729	+ size_t size, int prot)
	730	+{
	731	+ return -ENODEV;
	732	+}
	733	+
566	734	static inline size_t iommu_unmap(struct iommu_domain *domain,
567	735	unsigned long iova, size_t size)
568	736	{
..	..	@@ -570,7 +738,8 @@
570	738	}
571	739
572	740	static inline size_t iommu_unmap_fast(struct iommu_domain *domain,
573		- unsigned long iova, int gfp_order)
	741	+ unsigned long iova, int gfp_order,
	742	+ struct iommu_iotlb_gather *iotlb_gather)
574	743	{
575	744	return 0;
576	745	}
..	..	@@ -582,16 +751,19 @@
582	751	return 0;
583	752	}
584	753
585		-static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
	754	+static inline size_t iommu_map_sg_atomic(struct iommu_domain *domain,
	755	+ unsigned long iova, struct scatterlist *sg,
	756	+ unsigned int nents, int prot)
	757	+{
	758	+ return 0;
	759	+}
	760	+
	761	+static inline void iommu_flush_iotlb_all(struct iommu_domain *domain)
586	762	{
587	763	}
588	764
589		-static inline void iommu_tlb_range_add(struct iommu_domain *domain,
590		- unsigned long iova, size_t size)
591		-{
592		-}
593		-
594		-static inline void iommu_tlb_sync(struct iommu_domain *domain)
	765	+static inline void iommu_iotlb_sync(struct iommu_domain *domain,
	766	+ struct iommu_iotlb_gather *iotlb_gather)
595	767	{
596	768	}
597	769
..	..	@@ -608,18 +780,6 @@
608	780	}
609	781
610	782	static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
611		-{
612		- return 0;
613		-}
614		-
615		-static inline phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
616		- dma_addr_t iova)
617		-{
618		- return 0;
619		-}
620		-
621		-static inline bool iommu_is_iova_coherent(struct iommu_domain *domain,
622		- dma_addr_t iova)
623	783	{
624	784	return 0;
625	785	}
..	..	@@ -645,9 +805,17 @@
645	805	return -ENODEV;
646	806	}
647	807
648		-static inline int iommu_request_dm_for_dev(struct device *dev)
	808	+static inline void iommu_set_default_passthrough(bool cmd_line)
649	809	{
650		- return -ENODEV;
	810	+}
	811	+
	812	+static inline void iommu_set_default_translated(bool cmd_line)
	813	+{
	814	+}
	815	+
	816	+static inline bool iommu_default_passthrough(void)
	817	+{
	818	+ return true;
651	819	}
652	820
653	821	static inline int iommu_attach_group(struct iommu_domain *domain,
..	..	@@ -721,6 +889,31 @@
721	889	return 0;
722	890	}
723	891
	892	+static inline
	893	+int iommu_register_device_fault_handler(struct device *dev,
	894	+ iommu_dev_fault_handler_t handler,
	895	+ void *data)
	896	+{
	897	+ return -ENODEV;
	898	+}
	899	+
	900	+static inline int iommu_unregister_device_fault_handler(struct device *dev)
	901	+{
	902	+ return 0;
	903	+}
	904	+
	905	+static inline
	906	+int iommu_report_device_fault(struct device dev, struct iommu_fault_event evt)
	907	+{
	908	+ return -ENODEV;
	909	+}
	910	+
	911	+static inline int iommu_page_response(struct device *dev,
	912	+ struct iommu_page_response *msg)
	913	+{
	914	+ return -ENODEV;
	915	+}
	916	+
724	917	static inline int iommu_group_id(struct iommu_group *group)
725	918	{
726	919	return -ENODEV;
..	..	@@ -758,6 +951,16 @@
758	951	return NULL;
759	952	}
760	953
	954	+static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
	955	+{
	956	+}
	957	+
	958	+static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
	959	+ struct iommu_iotlb_gather *gather,
	960	+ unsigned long iova, size_t size)
	961	+{
	962	+}
	963	+
761	964	static inline void iommu_device_unregister(struct iommu_device *iommu)
762	965	{
763	966	}
..	..	@@ -780,35 +983,6 @@
780	983	}
781	984
782	985	static inline void iommu_device_unlink(struct device dev, struct device link)
783		-{
784		-}
785		-
786		-static inline void iommu_trigger_fault(struct iommu_domain *domain,
787		- unsigned long flags)
788		-{
789		-}
790		-
791		-static inline unsigned long iommu_reg_read(struct iommu_domain *domain,
792		- unsigned long offset)
793		-{
794		- return 0;
795		-}
796		-
797		-static inline void iommu_reg_write(struct iommu_domain *domain,
798		- unsigned long val, unsigned long offset)
799		-{
800		-}
801		-
802		-static inline void iommu_tlbiall(struct iommu_domain *domain)
803		-{
804		-}
805		-
806		-static inline int iommu_enable_config_clocks(struct iommu_domain *domain)
807		-{
808		- return 0;
809		-}
810		-
811		-static inline void iommu_disable_config_clocks(struct iommu_domain *domain)
812	986	{
813	987	}
814	988
..	..	@@ -835,20 +1009,111 @@
835	1009	return NULL;
836	1010	}
837	1011
838		-static inline int iommu_is_available(struct device *dev)
	1012	+static inline bool
	1013	+iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat)
	1014	+{
	1015	+ return false;
	1016	+}
	1017	+
	1018	+static inline bool
	1019	+iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat)
	1020	+{
	1021	+ return false;
	1022	+}
	1023	+
	1024	+static inline int
	1025	+iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
839	1026	{
840	1027	return -ENODEV;
841	1028	}
842	1029
843		-static inline void rk_iommu_mask_irq(struct device *dev)
	1030	+static inline int
	1031	+iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
	1032	+{
	1033	+ return -ENODEV;
	1034	+}
	1035	+
	1036	+static inline int
	1037	+iommu_aux_attach_device(struct iommu_domain domain, struct device dev)
	1038	+{
	1039	+ return -ENODEV;
	1040	+}
	1041	+
	1042	+static inline void
	1043	+iommu_aux_detach_device(struct iommu_domain domain, struct device dev)
844	1044	{
845	1045	}
846	1046
847		-static inline void rk_iommu_unmask_irq(struct device *dev)
	1047	+static inline int
	1048	+iommu_aux_get_pasid(struct iommu_domain domain, struct device dev)
	1049	+{
	1050	+ return -ENODEV;
	1051	+}
	1052	+
	1053	+static inline struct iommu_sva *
	1054	+iommu_sva_bind_device(struct device dev, struct mm_struct mm, void *drvdata)
	1055	+{
	1056	+ return NULL;
	1057	+}
	1058	+
	1059	+static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
848	1060	{
849	1061	}
	1062	+
	1063	+static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
	1064	+{
	1065	+ return IOMMU_PASID_INVALID;
	1066	+}
	1067	+
	1068	+static inline int
	1069	+iommu_uapi_cache_invalidate(struct iommu_domain *domain,
	1070	+ struct device *dev,
	1071	+ struct iommu_cache_invalidate_info *inv_info)
	1072	+{
	1073	+ return -ENODEV;
	1074	+}
	1075	+
	1076	+static inline int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain,
	1077	+ struct device dev, void __user udata)
	1078	+{
	1079	+ return -ENODEV;
	1080	+}
	1081	+
	1082	+static inline int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain,
	1083	+ struct device dev, void __user udata)
	1084	+{
	1085	+ return -ENODEV;
	1086	+}
	1087	+
	1088	+static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
	1089	+ struct device *dev,
	1090	+ ioasid_t pasid)
	1091	+{
	1092	+ return -ENODEV;
	1093	+}
	1094	+
	1095	+static inline struct iommu_fwspec dev_iommu_fwspec_get(struct device dev)
	1096	+{
	1097	+ return NULL;
	1098	+}
850	1099	#endif /* CONFIG_IOMMU_API */
851	1100
	1101	+/**
	1102	+ * iommu_map_sgtable - Map the given buffer to the IOMMU domain
	1103	+ * @domain: The IOMMU domain to perform the mapping
	1104	+ * @iova: The start address to map the buffer
	1105	+ * @sgt: The sg_table object describing the buffer
	1106	+ * @prot: IOMMU protection bits
	1107	+ *
	1108	+ * Creates a mapping at @iova for the buffer described by a scatterlist
	1109	+ * stored in the given sg_table object in the provided IOMMU domain.
	1110	+ */
	1111	+static inline size_t iommu_map_sgtable(struct iommu_domain *domain,
	1112	+ unsigned long iova, struct sg_table *sgt, int prot)
	1113	+{
	1114	+ return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot);
	1115	+}
	1116	+
852	1117	#ifdef CONFIG_IOMMU_DEBUGFS
853	1118	extern struct dentry *iommu_debugfs_dir;
854	1119	void iommu_debugfs_setup(void);