~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,28 +1,18 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3	4	* Author: Joerg Roedel <jroedel@suse.de>
4		- *
5		- * This program is free software; you can redistribute it and/or modify it
6		- * under the terms of the GNU General Public License version 2 as published
7		- * by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope that it will be useful,
10		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
11		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12		- * GNU General Public License for more details.
13		- *
14		- * You should have received a copy of the GNU General Public License
15		- * along with this program; if not, write to the Free Software
16		- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17	5	*/
18	6
19	7	#define pr_fmt(fmt) "iommu: " fmt
20	8
21	9	#include <linux/device.h>
22	10	#include <linux/kernel.h>
	11	+#include <linux/bits.h>
23	12	#include <linux/bug.h>
24	13	#include <linux/types.h>
25		-#include <linux/module.h>
	14	+#include <linux/init.h>
	15	+#include <linux/export.h>
26	16	#include <linux/slab.h>
27	17	#include <linux/errno.h>
28	18	#include <linux/iommu.h>
..	..	@@ -32,19 +22,16 @@
32	22	#include <linux/pci.h>
33	23	#include <linux/bitops.h>
34	24	#include <linux/property.h>
	25	+#include <linux/fsl/mc.h>
	26	+#include <linux/module.h>
35	27	#include <trace/events/iommu.h>
36	28
37	29	static struct kset *iommu_group_kset;
38	30	static DEFINE_IDA(iommu_group_ida);
39		-#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH
40		-static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
41		-#else
42		-static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
43		-#endif
44	31
45		-struct iommu_callback_data {
46		- const struct iommu_ops *ops;
47		-};
	32	+static unsigned int iommu_def_domain_type __read_mostly;
	33	+static bool iommu_dma_strict __read_mostly = true;
	34	+static u32 iommu_cmd_line __read_mostly;
48	35
49	36	struct iommu_group {
50	37	struct kobject kobj;
..	..	@@ -58,6 +45,7 @@
58	45	int id;
59	46	struct iommu_domain *default_domain;
60	47	struct iommu_domain *domain;
	48	+ struct list_head entry;
61	49	};
62	50
63	51	struct group_device {
..	..	@@ -74,11 +62,38 @@
74	62	};
75	63
76	64	static const char * const iommu_group_resv_type_string[] = {
77		- [IOMMU_RESV_DIRECT] = "direct",
78		- [IOMMU_RESV_RESERVED] = "reserved",
79		- [IOMMU_RESV_MSI] = "msi",
80		- [IOMMU_RESV_SW_MSI] = "msi",
	65	+ [IOMMU_RESV_DIRECT] = "direct",
	66	+ [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable",
	67	+ [IOMMU_RESV_RESERVED] = "reserved",
	68	+ [IOMMU_RESV_MSI] = "msi",
	69	+ [IOMMU_RESV_SW_MSI] = "msi",
81	70	};
	71	+
	72	+#define IOMMU_CMD_LINE_DMA_API BIT(0)
	73	+
	74	+static void iommu_set_cmd_line_dma_api(void)
	75	+{
	76	+ iommu_cmd_line \|= IOMMU_CMD_LINE_DMA_API;
	77	+}
	78	+
	79	+static bool iommu_cmd_line_dma_api(void)
	80	+{
	81	+ return !!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API);
	82	+}
	83	+
	84	+static int iommu_alloc_default_domain(struct iommu_group *group,
	85	+ struct device *dev);
	86	+static struct iommu_domain __iommu_domain_alloc(struct bus_type bus,
	87	+ unsigned type);
	88	+static int __iommu_attach_device(struct iommu_domain *domain,
	89	+ struct device *dev);
	90	+static int __iommu_attach_group(struct iommu_domain *domain,
	91	+ struct iommu_group *group);
	92	+static void __iommu_detach_group(struct iommu_domain *domain,
	93	+ struct iommu_group *group);
	94	+static int iommu_create_device_direct_mappings(struct iommu_group *group,
	95	+ struct device *dev);
	96	+static struct iommu_group iommu_group_get_for_dev(struct device dev);
82	97
83	98	#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
84	99	struct iommu_group_attribute iommu_group_attr_##_name = \
..	..	@@ -92,12 +107,55 @@
92	107	static LIST_HEAD(iommu_device_list);
93	108	static DEFINE_SPINLOCK(iommu_device_lock);
94	109
	110	+/*
	111	+ * Use a function instead of an array here because the domain-type is a
	112	+ * bit-field, so an array would waste memory.
	113	+ */
	114	+static const char *iommu_domain_type_str(unsigned int t)
	115	+{
	116	+ switch (t) {
	117	+ case IOMMU_DOMAIN_BLOCKED:
	118	+ return "Blocked";
	119	+ case IOMMU_DOMAIN_IDENTITY:
	120	+ return "Passthrough";
	121	+ case IOMMU_DOMAIN_UNMANAGED:
	122	+ return "Unmanaged";
	123	+ case IOMMU_DOMAIN_DMA:
	124	+ return "Translated";
	125	+ default:
	126	+ return "Unknown";
	127	+ }
	128	+}
	129	+
	130	+static int __init iommu_subsys_init(void)
	131	+{
	132	+ bool cmd_line = iommu_cmd_line_dma_api();
	133	+
	134	+ if (!cmd_line) {
	135	+ if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
	136	+ iommu_set_default_passthrough(false);
	137	+ else
	138	+ iommu_set_default_translated(false);
	139	+
	140	+ if (iommu_default_passthrough() && mem_encrypt_active()) {
	141	+ pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
	142	+ iommu_set_default_translated(false);
	143	+ }
	144	+ }
	145	+
	146	+ pr_info("Default domain type: %s %s\n",
	147	+ iommu_domain_type_str(iommu_def_domain_type),
	148	+ cmd_line ? "(set via kernel command line)" : "");
	149	+
	150	+ return 0;
	151	+}
	152	+subsys_initcall(iommu_subsys_init);
	153	+
95	154	int iommu_device_register(struct iommu_device *iommu)
96	155	{
97	156	spin_lock(&iommu_device_lock);
98	157	list_add_tail(&iommu->list, &iommu_device_list);
99	158	spin_unlock(&iommu_device_lock);
100		-
101	159	return 0;
102	160	}
103	161	EXPORT_SYMBOL_GPL(iommu_device_register);
..	..	@@ -110,14 +168,167 @@
110	168	}
111	169	EXPORT_SYMBOL_GPL(iommu_device_unregister);
112	170
113		-static struct iommu_domain __iommu_domain_alloc(struct bus_type bus,
114		- unsigned type);
115		-static int __iommu_attach_device(struct iommu_domain *domain,
116		- struct device *dev);
117		-static int __iommu_attach_group(struct iommu_domain *domain,
118		- struct iommu_group *group);
119		-static void __iommu_detach_group(struct iommu_domain *domain,
120		- struct iommu_group *group);
	171	+static struct dev_iommu dev_iommu_get(struct device dev)
	172	+{
	173	+ struct dev_iommu *param = dev->iommu;
	174	+
	175	+ if (param)
	176	+ return param;
	177	+
	178	+ param = kzalloc(sizeof(*param), GFP_KERNEL);
	179	+ if (!param)
	180	+ return NULL;
	181	+
	182	+ mutex_init(&param->lock);
	183	+ dev->iommu = param;
	184	+ return param;
	185	+}
	186	+
	187	+static void dev_iommu_free(struct device *dev)
	188	+{
	189	+ struct dev_iommu *param = dev->iommu;
	190	+
	191	+ dev->iommu = NULL;
	192	+ if (param->fwspec) {
	193	+ fwnode_handle_put(param->fwspec->iommu_fwnode);
	194	+ kfree(param->fwspec);
	195	+ }
	196	+ kfree(param);
	197	+}
	198	+
	199	+static int __iommu_probe_device(struct device dev, struct list_head group_list)
	200	+{
	201	+ const struct iommu_ops *ops = dev->bus->iommu_ops;
	202	+ struct iommu_device *iommu_dev;
	203	+ struct iommu_group *group;
	204	+ static DEFINE_MUTEX(iommu_probe_device_lock);
	205	+ int ret;
	206	+
	207	+ if (!ops)
	208	+ return -ENODEV;
	209	+ /*
	210	+ * Serialise to avoid races between IOMMU drivers registering in
	211	+ * parallel and/or the "replay" calls from ACPI/OF code via client
	212	+ * driver probe. Once the latter have been cleaned up we should
	213	+ * probably be able to use device_lock() here to minimise the scope,
	214	+ * but for now enforcing a simple global ordering is fine.
	215	+ */
	216	+ mutex_lock(&iommu_probe_device_lock);
	217	+ if (!dev_iommu_get(dev)) {
	218	+ ret = -ENOMEM;
	219	+ goto err_unlock;
	220	+ }
	221	+
	222	+ if (!try_module_get(ops->owner)) {
	223	+ ret = -EINVAL;
	224	+ goto err_free;
	225	+ }
	226	+
	227	+ iommu_dev = ops->probe_device(dev);
	228	+ if (IS_ERR(iommu_dev)) {
	229	+ ret = PTR_ERR(iommu_dev);
	230	+ goto out_module_put;
	231	+ }
	232	+
	233	+ dev->iommu->iommu_dev = iommu_dev;
	234	+
	235	+ group = iommu_group_get_for_dev(dev);
	236	+ if (IS_ERR(group)) {
	237	+ ret = PTR_ERR(group);
	238	+ goto out_release;
	239	+ }
	240	+
	241	+ mutex_lock(&group->mutex);
	242	+ if (group_list && !group->default_domain && list_empty(&group->entry))
	243	+ list_add_tail(&group->entry, group_list);
	244	+ mutex_unlock(&group->mutex);
	245	+ iommu_group_put(group);
	246	+
	247	+ mutex_unlock(&iommu_probe_device_lock);
	248	+ iommu_device_link(iommu_dev, dev);
	249	+
	250	+ return 0;
	251	+
	252	+out_release:
	253	+ ops->release_device(dev);
	254	+
	255	+out_module_put:
	256	+ module_put(ops->owner);
	257	+
	258	+err_free:
	259	+ dev_iommu_free(dev);
	260	+
	261	+err_unlock:
	262	+ mutex_unlock(&iommu_probe_device_lock);
	263	+
	264	+ return ret;
	265	+}
	266	+
	267	+int iommu_probe_device(struct device *dev)
	268	+{
	269	+ const struct iommu_ops *ops = dev->bus->iommu_ops;
	270	+ struct iommu_group *group;
	271	+ int ret;
	272	+
	273	+ ret = __iommu_probe_device(dev, NULL);
	274	+ if (ret)
	275	+ goto err_out;
	276	+
	277	+ group = iommu_group_get(dev);
	278	+ if (!group)
	279	+ goto err_release;
	280	+
	281	+ /*
	282	+ * Try to allocate a default domain - needs support from the
	283	+ * IOMMU driver. There are still some drivers which don't
	284	+ * support default domains, so the return value is not yet
	285	+ * checked.
	286	+ */
	287	+ mutex_lock(&group->mutex);
	288	+ iommu_alloc_default_domain(group, dev);
	289	+
	290	+ if (group->default_domain) {
	291	+ ret = __iommu_attach_device(group->default_domain, dev);
	292	+ if (ret) {
	293	+ mutex_unlock(&group->mutex);
	294	+ iommu_group_put(group);
	295	+ goto err_release;
	296	+ }
	297	+ }
	298	+
	299	+ iommu_create_device_direct_mappings(group, dev);
	300	+
	301	+ mutex_unlock(&group->mutex);
	302	+ iommu_group_put(group);
	303	+
	304	+ if (ops->probe_finalize)
	305	+ ops->probe_finalize(dev);
	306	+
	307	+ return 0;
	308	+
	309	+err_release:
	310	+ iommu_release_device(dev);
	311	+
	312	+err_out:
	313	+ return ret;
	314	+
	315	+}
	316	+
	317	+void iommu_release_device(struct device *dev)
	318	+{
	319	+ const struct iommu_ops *ops = dev->bus->iommu_ops;
	320	+
	321	+ if (!dev->iommu)
	322	+ return;
	323	+
	324	+ iommu_device_unlink(dev->iommu->iommu_dev, dev);
	325	+
	326	+ ops->release_device(dev);
	327	+
	328	+ iommu_group_remove_device(dev);
	329	+ module_put(ops->owner);
	330	+ dev_iommu_free(dev);
	331	+}
121	332
122	333	static int __init iommu_set_def_domain_type(char *str)
123	334	{
..	..	@@ -128,10 +339,20 @@
128	339	if (ret)
129	340	return ret;
130	341
131		- iommu_def_domain_type = pt ? IOMMU_DOMAIN_IDENTITY : IOMMU_DOMAIN_DMA;
	342	+ if (pt)
	343	+ iommu_set_default_passthrough(true);
	344	+ else
	345	+ iommu_set_default_translated(true);
	346	+
132	347	return 0;
133	348	}
134	349	early_param("iommu.passthrough", iommu_set_def_domain_type);
	350	+
	351	+static int __init iommu_dma_setup(char *str)
	352	+{
	353	+ return kstrtobool(str, &iommu_dma_strict);
	354	+}
	355	+early_param("iommu.strict", iommu_dma_setup);
135	356
136	357	static ssize_t iommu_group_attr_show(struct kobject *kobj,
137	358	struct attribute __attr, char buf)
..	..	@@ -186,60 +407,58 @@
186	407	* @new: new region to insert
187	408	* @regions: list of regions
188	409	*
189		- * The new element is sorted by address with respect to the other
190		- * regions of the same type. In case it overlaps with another
191		- * region of the same type, regions are merged. In case it
192		- * overlaps with another region of different type, regions are
193		- * not merged.
	410	+ * Elements are sorted by start address and overlapping segments
	411	+ * of the same type are merged.
194	412	*/
195	413	static int iommu_insert_resv_region(struct iommu_resv_region *new,
196	414	struct list_head *regions)
197	415	{
198		- struct iommu_resv_region *region;
199		- phys_addr_t start = new->start;
200		- phys_addr_t end = new->start + new->length - 1;
201		- struct list_head *pos = regions->next;
	416	+ struct iommu_resv_region iter, tmp, nr, top;
	417	+ LIST_HEAD(stack);
202	418
203		- while (pos != regions) {
204		- struct iommu_resv_region *entry =
205		- list_entry(pos, struct iommu_resv_region, list);
206		- phys_addr_t a = entry->start;
207		- phys_addr_t b = entry->start + entry->length - 1;
208		- int type = entry->type;
209		-
210		- if (end < a) {
211		- goto insert;
212		- } else if (start > b) {
213		- pos = pos->next;
214		- } else if ((start >= a) && (end <= b)) {
215		- if (new->type == type)
216		- return 0;
217		- else
218		- pos = pos->next;
219		- } else {
220		- if (new->type == type) {
221		- phys_addr_t new_start = min(a, start);
222		- phys_addr_t new_end = max(b, end);
223		- int ret;
224		-
225		- list_del(&entry->list);
226		- entry->start = new_start;
227		- entry->length = new_end - new_start + 1;
228		- ret = iommu_insert_resv_region(entry, regions);
229		- kfree(entry);
230		- return ret;
231		- } else {
232		- pos = pos->next;
233		- }
234		- }
235		- }
236		-insert:
237		- region = iommu_alloc_resv_region(new->start, new->length,
238		- new->prot, new->type);
239		- if (!region)
	419	+ nr = iommu_alloc_resv_region(new->start, new->length,
	420	+ new->prot, new->type);
	421	+ if (!nr)
240	422	return -ENOMEM;
241	423
242		- list_add_tail(&region->list, pos);
	424	+ /* First add the new element based on start address sorting */
	425	+ list_for_each_entry(iter, regions, list) {
	426	+ if (nr->start < iter->start \|\|
	427	+ (nr->start == iter->start && nr->type <= iter->type))
	428	+ break;
	429	+ }
	430	+ list_add_tail(&nr->list, &iter->list);
	431	+
	432	+ /* Merge overlapping segments of type nr->type in @regions, if any */
	433	+ list_for_each_entry_safe(iter, tmp, regions, list) {
	434	+ phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
	435	+
	436	+ /* no merge needed on elements of different types than @new */
	437	+ if (iter->type != new->type) {
	438	+ list_move_tail(&iter->list, &stack);
	439	+ continue;
	440	+ }
	441	+
	442	+ /* look for the last stack element of same type as @iter */
	443	+ list_for_each_entry_reverse(top, &stack, list)
	444	+ if (top->type == iter->type)
	445	+ goto check_overlap;
	446	+
	447	+ list_move_tail(&iter->list, &stack);
	448	+ continue;
	449	+
	450	+check_overlap:
	451	+ top_end = top->start + top->length - 1;
	452	+
	453	+ if (iter->start > top_end + 1) {
	454	+ list_move_tail(&iter->list, &stack);
	455	+ } else {
	456	+ top->length = max(top_end, iter_end) - top->start + 1;
	457	+ list_del(&iter->list);
	458	+ kfree(iter);
	459	+ }
	460	+ }
	461	+ list_splice(&stack, regions);
243	462	return 0;
244	463	}
245	464
..	..	@@ -381,6 +600,7 @@
381	600	group->kobj.kset = iommu_group_kset;
382	601	mutex_init(&group->mutex);
383	602	INIT_LIST_HEAD(&group->devices);
	603	+ INIT_LIST_HEAD(&group->entry);
384	604	BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
385	605
386	606	ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL);
..	..	@@ -522,8 +742,8 @@
522	742	}
523	743	EXPORT_SYMBOL_GPL(iommu_group_set_name);
524	744
525		-static int iommu_group_create_direct_mappings(struct iommu_group *group,
526		- struct device *dev)
	745	+static int iommu_create_device_direct_mappings(struct iommu_group *group,
	746	+ struct device *dev)
527	747	{
528	748	struct iommu_domain *domain = group->default_domain;
529	749	struct iommu_resv_region *entry;
..	..	@@ -551,7 +771,8 @@
551	771	start = ALIGN(entry->start, pg_size);
552	772	end = ALIGN(entry->start + entry->length, pg_size);
553	773
554		- if (entry->type != IOMMU_RESV_DIRECT)
	774	+ if (entry->type != IOMMU_RESV_DIRECT &&
	775	+ entry->type != IOMMU_RESV_DIRECT_RELAXABLE)
555	776	continue;
556	777
557	778	for (addr = start; addr < end; addr += pg_size) {
..	..	@@ -568,12 +789,21 @@
568	789
569	790	}
570	791
571		- iommu_flush_tlb_all(domain);
	792	+ iommu_flush_iotlb_all(domain);
572	793
573	794	out:
574	795	iommu_put_resv_regions(dev, &mappings);
575	796
576	797	return ret;
	798	+}
	799	+
	800	+static bool iommu_is_attach_deferred(struct iommu_domain *domain,
	801	+ struct device *dev)
	802	+{
	803	+ if (domain->ops->is_attach_deferred)
	804	+ return domain->ops->is_attach_deferred(domain, dev);
	805	+
	806	+ return false;
577	807	}
578	808
579	809	/**
..	..	@@ -626,11 +856,9 @@
626	856
627	857	dev->iommu_group = group;
628	858
629		- iommu_group_create_direct_mappings(group, dev);
630		-
631	859	mutex_lock(&group->mutex);
632	860	list_add_tail(&device->list, &group->devices);
633		- if (group->domain)
	861	+ if (group->domain && !iommu_is_attach_deferred(group->domain, dev))
634	862	ret = __iommu_attach_device(group->domain, dev);
635	863	mutex_unlock(&group->mutex);
636	864	if (ret)
..	..	@@ -642,7 +870,7 @@
642	870
643	871	trace_add_device_to_group(group->id, dev);
644	872
645		- pr_info("Adding device %s to group %d\n", dev_name(dev), group->id);
	873	+ dev_info(dev, "Adding to iommu group %d\n", group->id);
646	874
647	875	return 0;
648	876
..	..	@@ -659,7 +887,7 @@
659	887	sysfs_remove_link(&dev->kobj, "iommu_group");
660	888	err_free_device:
661	889	kfree(device);
662		- pr_err("Failed to add device %s to group %d: %d\n", dev_name(dev), group->id, ret);
	890	+ dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
663	891	return ret;
664	892	}
665	893	EXPORT_SYMBOL_GPL(iommu_group_add_device);
..	..	@@ -676,7 +904,10 @@
676	904	struct iommu_group *group = dev->iommu_group;
677	905	struct group_device tmp_device, device = NULL;
678	906
679		- pr_info("Removing device %s from group %d\n", dev_name(dev), group->id);
	907	+ if (!group)
	908	+ return;
	909	+
	910	+ dev_info(dev, "Removing from iommu group %d\n", group->id);
680	911
681	912	/* Pre-notify listeners that a device is being removed. */
682	913	blocking_notifier_call_chain(&group->notifier,
..	..	@@ -833,6 +1064,217 @@
833	1064	return blocking_notifier_chain_unregister(&group->notifier, nb);
834	1065	}
835	1066	EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier);
	1067	+
	1068	+/**
	1069	+ * iommu_register_device_fault_handler() - Register a device fault handler
	1070	+ * @dev: the device
	1071	+ * @handler: the fault handler
	1072	+ * @data: private data passed as argument to the handler
	1073	+ *
	1074	+ * When an IOMMU fault event is received, this handler gets called with the
	1075	+ * fault event and data as argument. The handler should return 0 on success. If
	1076	+ * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also
	1077	+ * complete the fault by calling iommu_page_response() with one of the following
	1078	+ * response code:
	1079	+ * - IOMMU_PAGE_RESP_SUCCESS: retry the translation
	1080	+ * - IOMMU_PAGE_RESP_INVALID: terminate the fault
	1081	+ * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting
	1082	+ * page faults if possible.
	1083	+ *
	1084	+ * Return 0 if the fault handler was installed successfully, or an error.
	1085	+ */
	1086	+int iommu_register_device_fault_handler(struct device *dev,
	1087	+ iommu_dev_fault_handler_t handler,
	1088	+ void *data)
	1089	+{
	1090	+ struct dev_iommu *param = dev->iommu;
	1091	+ int ret = 0;
	1092	+
	1093	+ if (!param)
	1094	+ return -EINVAL;
	1095	+
	1096	+ mutex_lock(&param->lock);
	1097	+ /* Only allow one fault handler registered for each device */
	1098	+ if (param->fault_param) {
	1099	+ ret = -EBUSY;
	1100	+ goto done_unlock;
	1101	+ }
	1102	+
	1103	+ get_device(dev);
	1104	+ param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL);
	1105	+ if (!param->fault_param) {
	1106	+ put_device(dev);
	1107	+ ret = -ENOMEM;
	1108	+ goto done_unlock;
	1109	+ }
	1110	+ param->fault_param->handler = handler;
	1111	+ param->fault_param->data = data;
	1112	+ mutex_init(&param->fault_param->lock);
	1113	+ INIT_LIST_HEAD(&param->fault_param->faults);
	1114	+
	1115	+done_unlock:
	1116	+ mutex_unlock(&param->lock);
	1117	+
	1118	+ return ret;
	1119	+}
	1120	+EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler);
	1121	+
	1122	+/**
	1123	+ * iommu_unregister_device_fault_handler() - Unregister the device fault handler
	1124	+ * @dev: the device
	1125	+ *
	1126	+ * Remove the device fault handler installed with
	1127	+ * iommu_register_device_fault_handler().
	1128	+ *
	1129	+ * Return 0 on success, or an error.
	1130	+ */
	1131	+int iommu_unregister_device_fault_handler(struct device *dev)
	1132	+{
	1133	+ struct dev_iommu *param = dev->iommu;
	1134	+ int ret = 0;
	1135	+
	1136	+ if (!param)
	1137	+ return -EINVAL;
	1138	+
	1139	+ mutex_lock(&param->lock);
	1140	+
	1141	+ if (!param->fault_param)
	1142	+ goto unlock;
	1143	+
	1144	+ /* we cannot unregister handler if there are pending faults */
	1145	+ if (!list_empty(&param->fault_param->faults)) {
	1146	+ ret = -EBUSY;
	1147	+ goto unlock;
	1148	+ }
	1149	+
	1150	+ kfree(param->fault_param);
	1151	+ param->fault_param = NULL;
	1152	+ put_device(dev);
	1153	+unlock:
	1154	+ mutex_unlock(&param->lock);
	1155	+
	1156	+ return ret;
	1157	+}
	1158	+EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler);
	1159	+
	1160	+/**
	1161	+ * iommu_report_device_fault() - Report fault event to device driver
	1162	+ * @dev: the device
	1163	+ * @evt: fault event data
	1164	+ *
	1165	+ * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ
	1166	+ * handler. When this function fails and the fault is recoverable, it is the
	1167	+ * caller's responsibility to complete the fault.
	1168	+ *
	1169	+ * Return 0 on success, or an error.
	1170	+ */
	1171	+int iommu_report_device_fault(struct device dev, struct iommu_fault_event evt)
	1172	+{
	1173	+ struct dev_iommu *param = dev->iommu;
	1174	+ struct iommu_fault_event *evt_pending = NULL;
	1175	+ struct iommu_fault_param *fparam;
	1176	+ int ret = 0;
	1177	+
	1178	+ if (!param \|\| !evt)
	1179	+ return -EINVAL;
	1180	+
	1181	+ /* we only report device fault if there is a handler registered */
	1182	+ mutex_lock(&param->lock);
	1183	+ fparam = param->fault_param;
	1184	+ if (!fparam \|\| !fparam->handler) {
	1185	+ ret = -EINVAL;
	1186	+ goto done_unlock;
	1187	+ }
	1188	+
	1189	+ if (evt->fault.type == IOMMU_FAULT_PAGE_REQ &&
	1190	+ (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) {
	1191	+ evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event),
	1192	+ GFP_KERNEL);
	1193	+ if (!evt_pending) {
	1194	+ ret = -ENOMEM;
	1195	+ goto done_unlock;
	1196	+ }
	1197	+ mutex_lock(&fparam->lock);
	1198	+ list_add_tail(&evt_pending->list, &fparam->faults);
	1199	+ mutex_unlock(&fparam->lock);
	1200	+ }
	1201	+
	1202	+ ret = fparam->handler(&evt->fault, fparam->data);
	1203	+ if (ret && evt_pending) {
	1204	+ mutex_lock(&fparam->lock);
	1205	+ list_del(&evt_pending->list);
	1206	+ mutex_unlock(&fparam->lock);
	1207	+ kfree(evt_pending);
	1208	+ }
	1209	+done_unlock:
	1210	+ mutex_unlock(&param->lock);
	1211	+ return ret;
	1212	+}
	1213	+EXPORT_SYMBOL_GPL(iommu_report_device_fault);
	1214	+
	1215	+int iommu_page_response(struct device *dev,
	1216	+ struct iommu_page_response *msg)
	1217	+{
	1218	+ bool needs_pasid;
	1219	+ int ret = -EINVAL;
	1220	+ struct iommu_fault_event *evt;
	1221	+ struct iommu_fault_page_request *prm;
	1222	+ struct dev_iommu *param = dev->iommu;
	1223	+ bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID;
	1224	+ struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
	1225	+
	1226	+ if (!domain \|\| !domain->ops->page_response)
	1227	+ return -ENODEV;
	1228	+
	1229	+ if (!param \|\| !param->fault_param)
	1230	+ return -EINVAL;
	1231	+
	1232	+ if (msg->version != IOMMU_PAGE_RESP_VERSION_1 \|\|
	1233	+ msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID)
	1234	+ return -EINVAL;
	1235	+
	1236	+ /* Only send response if there is a fault report pending */
	1237	+ mutex_lock(&param->fault_param->lock);
	1238	+ if (list_empty(&param->fault_param->faults)) {
	1239	+ dev_warn_ratelimited(dev, "no pending PRQ, drop response\n");
	1240	+ goto done_unlock;
	1241	+ }
	1242	+ /*
	1243	+ * Check if we have a matching page request pending to respond,
	1244	+ * otherwise return -EINVAL
	1245	+ */
	1246	+ list_for_each_entry(evt, &param->fault_param->faults, list) {
	1247	+ prm = &evt->fault.prm;
	1248	+ if (prm->grpid != msg->grpid)
	1249	+ continue;
	1250	+
	1251	+ /*
	1252	+ * If the PASID is required, the corresponding request is
	1253	+ * matched using the group ID, the PASID valid bit and the PASID
	1254	+ * value. Otherwise only the group ID matches request and
	1255	+ * response.
	1256	+ */
	1257	+ needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
	1258	+ if (needs_pasid && (!has_pasid \|\| msg->pasid != prm->pasid))
	1259	+ continue;
	1260	+
	1261	+ if (!needs_pasid && has_pasid) {
	1262	+ /* No big deal, just clear it. */
	1263	+ msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID;
	1264	+ msg->pasid = 0;
	1265	+ }
	1266	+
	1267	+ ret = domain->ops->page_response(dev, evt, msg);
	1268	+ list_del(&evt->list);
	1269	+ kfree(evt);
	1270	+ break;
	1271	+ }
	1272	+
	1273	+done_unlock:
	1274	+ mutex_unlock(&param->fault_param->lock);
	1275	+ return ret;
	1276	+}
	1277	+EXPORT_SYMBOL_GPL(iommu_page_response);
836	1278
837	1279	/**
838	1280	* iommu_group_id - Return ID for a group
..	..	@@ -1032,6 +1474,74 @@
1032	1474	}
1033	1475	EXPORT_SYMBOL_GPL(pci_device_group);
1034	1476
	1477	+/* Get the IOMMU group for device on fsl-mc bus */
	1478	+struct iommu_group fsl_mc_device_group(struct device dev)
	1479	+{
	1480	+ struct device *cont_dev = fsl_mc_cont_dev(dev);
	1481	+ struct iommu_group *group;
	1482	+
	1483	+ group = iommu_group_get(cont_dev);
	1484	+ if (!group)
	1485	+ group = iommu_group_alloc();
	1486	+ return group;
	1487	+}
	1488	+EXPORT_SYMBOL_GPL(fsl_mc_device_group);
	1489	+
	1490	+static int iommu_get_def_domain_type(struct device *dev)
	1491	+{
	1492	+ const struct iommu_ops *ops = dev->bus->iommu_ops;
	1493	+ unsigned int type = 0;
	1494	+
	1495	+ if (ops->def_domain_type)
	1496	+ type = ops->def_domain_type(dev);
	1497	+
	1498	+ return (type == 0) ? iommu_def_domain_type : type;
	1499	+}
	1500	+
	1501	+static int iommu_group_alloc_default_domain(struct bus_type *bus,
	1502	+ struct iommu_group *group,
	1503	+ unsigned int type)
	1504	+{
	1505	+ struct iommu_domain *dom;
	1506	+
	1507	+ dom = __iommu_domain_alloc(bus, type);
	1508	+ if (!dom && type != IOMMU_DOMAIN_DMA) {
	1509	+ dom = __iommu_domain_alloc(bus, IOMMU_DOMAIN_DMA);
	1510	+ if (dom)
	1511	+ pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
	1512	+ type, group->name);
	1513	+ }
	1514	+
	1515	+ if (!dom)
	1516	+ return -ENOMEM;
	1517	+
	1518	+ group->default_domain = dom;
	1519	+ if (!group->domain)
	1520	+ group->domain = dom;
	1521	+
	1522	+ if (!iommu_dma_strict) {
	1523	+ int attr = 1;
	1524	+ iommu_domain_set_attr(dom,
	1525	+ DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
	1526	+ &attr);
	1527	+ }
	1528	+
	1529	+ return 0;
	1530	+}
	1531	+
	1532	+static int iommu_alloc_default_domain(struct iommu_group *group,
	1533	+ struct device *dev)
	1534	+{
	1535	+ unsigned int type;
	1536	+
	1537	+ if (group->default_domain)
	1538	+ return 0;
	1539	+
	1540	+ type = iommu_get_def_domain_type(dev);
	1541	+
	1542	+ return iommu_group_alloc_default_domain(dev->bus, group, type);
	1543	+}
	1544	+
1035	1545	/**
1036	1546	* iommu_group_get_for_dev - Find or create the IOMMU group for a device
1037	1547	* @dev: target device
..	..	@@ -1042,7 +1552,7 @@
1042	1552	* to the returned IOMMU group, which will already include the provided
1043	1553	* device. The reference should be released with iommu_group_put().
1044	1554	*/
1045		-struct iommu_group iommu_group_get_for_dev(struct device dev)
	1555	+static struct iommu_group iommu_group_get_for_dev(struct device dev)
1046	1556	{
1047	1557	const struct iommu_ops *ops = dev->bus->iommu_ops;
1048	1558	struct iommu_group *group;
..	..	@@ -1062,60 +1572,37 @@
1062	1572	if (IS_ERR(group))
1063	1573	return group;
1064	1574
1065		- /*
1066		- * Try to allocate a default domain - needs support from the
1067		- * IOMMU driver.
1068		- */
1069		- if (!group->default_domain) {
1070		- struct iommu_domain *dom;
1071		-
1072		- dom = __iommu_domain_alloc(dev->bus, iommu_def_domain_type);
1073		- if (!dom && iommu_def_domain_type != IOMMU_DOMAIN_DMA) {
1074		- dev_warn(dev,
1075		- "failed to allocate default IOMMU domain of type %u; falling back to IOMMU_DOMAIN_DMA",
1076		- iommu_def_domain_type);
1077		- dom = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_DMA);
1078		- }
1079		-
1080		- group->default_domain = dom;
1081		- if (!group->domain)
1082		- group->domain = dom;
1083		- }
1084		-
1085	1575	ret = iommu_group_add_device(group, dev);
1086		- if (ret) {
1087		- iommu_group_put(group);
1088		- return ERR_PTR(ret);
1089		- }
	1576	+ if (ret)
	1577	+ goto out_put_group;
1090	1578
1091	1579	return group;
	1580	+
	1581	+out_put_group:
	1582	+ iommu_group_put(group);
	1583	+
	1584	+ return ERR_PTR(ret);
1092	1585	}
1093		-EXPORT_SYMBOL_GPL(iommu_group_get_for_dev);
1094	1586
1095	1587	struct iommu_domain iommu_group_default_domain(struct iommu_group group)
1096	1588	{
1097	1589	return group->default_domain;
1098	1590	}
1099		-EXPORT_SYMBOL_GPL(iommu_group_default_domain);
1100	1591
1101		-static int add_iommu_group(struct device dev, void data)
	1592	+static int probe_iommu_group(struct device dev, void data)
1102	1593	{
1103		- struct iommu_callback_data *cb = data;
1104		- const struct iommu_ops *ops = cb->ops;
	1594	+ struct list_head *group_list = data;
	1595	+ struct iommu_group *group;
1105	1596	int ret;
1106	1597
1107		- if (!ops->add_device)
	1598	+ /* Device is probed already if in a group */
	1599	+ group = iommu_group_get(dev);
	1600	+ if (group) {
	1601	+ iommu_group_put(group);
1108	1602	return 0;
	1603	+ }
1109	1604
1110		- WARN_ON(dev->iommu_group);
1111		-
1112		- ret = ops->add_device(dev);
1113		-
1114		- /*
1115		- * We ignore -ENODEV errors for now, as they just mean that the
1116		- * device is not translated by an IOMMU. We still care about
1117		- * other errors and fail to initialize when they happen.
1118		- */
	1605	+ ret = __iommu_probe_device(dev, group_list);
1119	1606	if (ret == -ENODEV)
1120	1607	ret = 0;
1121	1608
..	..	@@ -1124,11 +1611,7 @@
1124	1611
1125	1612	static int remove_iommu_group(struct device dev, void data)
1126	1613	{
1127		- struct iommu_callback_data *cb = data;
1128		- const struct iommu_ops *ops = cb->ops;
1129		-
1130		- if (ops->remove_device && dev->iommu_group)
1131		- ops->remove_device(dev);
	1614	+ iommu_release_device(dev);
1132	1615
1133	1616	return 0;
1134	1617	}
..	..	@@ -1136,27 +1619,22 @@
1136	1619	static int iommu_bus_notifier(struct notifier_block *nb,
1137	1620	unsigned long action, void *data)
1138	1621	{
1139		- struct device *dev = data;
1140		- const struct iommu_ops *ops = dev->bus->iommu_ops;
1141		- struct iommu_group *group;
1142	1622	unsigned long group_action = 0;
	1623	+ struct device *dev = data;
	1624	+ struct iommu_group *group;
1143	1625
1144	1626	/*
1145	1627	* ADD/DEL call into iommu driver ops if provided, which may
1146	1628	* result in ADD/DEL notifiers to group->notifier
1147	1629	*/
1148	1630	if (action == BUS_NOTIFY_ADD_DEVICE) {
1149		- if (ops->add_device) {
1150		- int ret;
	1631	+ int ret;
1151	1632
1152		- ret = ops->add_device(dev);
1153		- return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1154		- }
	1633	+ ret = iommu_probe_device(dev);
	1634	+ return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1155	1635	} else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
1156		- if (ops->remove_device && dev->iommu_group) {
1157		- ops->remove_device(dev);
1158		- return 0;
1159		- }
	1636	+ iommu_release_device(dev);
	1637	+ return NOTIFY_OK;
1160	1638	}
1161	1639
1162	1640	/*
..	..	@@ -1190,13 +1668,152 @@
1190	1668	return 0;
1191	1669	}
1192	1670
	1671	+struct __group_domain_type {
	1672	+ struct device *dev;
	1673	+ unsigned int type;
	1674	+};
	1675	+
	1676	+static int probe_get_default_domain_type(struct device dev, void data)
	1677	+{
	1678	+ const struct iommu_ops *ops = dev->bus->iommu_ops;
	1679	+ struct __group_domain_type *gtype = data;
	1680	+ unsigned int type = 0;
	1681	+
	1682	+ if (ops->def_domain_type)
	1683	+ type = ops->def_domain_type(dev);
	1684	+
	1685	+ if (type) {
	1686	+ if (gtype->type && gtype->type != type) {
	1687	+ dev_warn(dev, "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n",
	1688	+ iommu_domain_type_str(type),
	1689	+ dev_name(gtype->dev),
	1690	+ iommu_domain_type_str(gtype->type));
	1691	+ gtype->type = 0;
	1692	+ }
	1693	+
	1694	+ if (!gtype->dev) {
	1695	+ gtype->dev = dev;
	1696	+ gtype->type = type;
	1697	+ }
	1698	+ }
	1699	+
	1700	+ return 0;
	1701	+}
	1702	+
	1703	+static void probe_alloc_default_domain(struct bus_type *bus,
	1704	+ struct iommu_group *group)
	1705	+{
	1706	+ struct __group_domain_type gtype;
	1707	+
	1708	+ memset(&gtype, 0, sizeof(gtype));
	1709	+
	1710	+ /* Ask for default domain requirements of all devices in the group */
	1711	+ __iommu_group_for_each_dev(group, &gtype,
	1712	+ probe_get_default_domain_type);
	1713	+
	1714	+ if (!gtype.type)
	1715	+ gtype.type = iommu_def_domain_type;
	1716	+
	1717	+ iommu_group_alloc_default_domain(bus, group, gtype.type);
	1718	+
	1719	+}
	1720	+
	1721	+static int iommu_group_do_dma_attach(struct device dev, void data)
	1722	+{
	1723	+ struct iommu_domain *domain = data;
	1724	+ int ret = 0;
	1725	+
	1726	+ if (!iommu_is_attach_deferred(domain, dev))
	1727	+ ret = __iommu_attach_device(domain, dev);
	1728	+
	1729	+ return ret;
	1730	+}
	1731	+
	1732	+static int __iommu_group_dma_attach(struct iommu_group *group)
	1733	+{
	1734	+ return __iommu_group_for_each_dev(group, group->default_domain,
	1735	+ iommu_group_do_dma_attach);
	1736	+}
	1737	+
	1738	+static int iommu_group_do_probe_finalize(struct device dev, void data)
	1739	+{
	1740	+ struct iommu_domain *domain = data;
	1741	+
	1742	+ if (domain->ops->probe_finalize)
	1743	+ domain->ops->probe_finalize(dev);
	1744	+
	1745	+ return 0;
	1746	+}
	1747	+
	1748	+static void __iommu_group_dma_finalize(struct iommu_group *group)
	1749	+{
	1750	+ __iommu_group_for_each_dev(group, group->default_domain,
	1751	+ iommu_group_do_probe_finalize);
	1752	+}
	1753	+
	1754	+static int iommu_do_create_direct_mappings(struct device dev, void data)
	1755	+{
	1756	+ struct iommu_group *group = data;
	1757	+
	1758	+ iommu_create_device_direct_mappings(group, dev);
	1759	+
	1760	+ return 0;
	1761	+}
	1762	+
	1763	+static int iommu_group_create_direct_mappings(struct iommu_group *group)
	1764	+{
	1765	+ return __iommu_group_for_each_dev(group, group,
	1766	+ iommu_do_create_direct_mappings);
	1767	+}
	1768	+
	1769	+int bus_iommu_probe(struct bus_type *bus)
	1770	+{
	1771	+ struct iommu_group group, next;
	1772	+ LIST_HEAD(group_list);
	1773	+ int ret;
	1774	+
	1775	+ /*
	1776	+ * This code-path does not allocate the default domain when
	1777	+ * creating the iommu group, so do it after the groups are
	1778	+ * created.
	1779	+ */
	1780	+ ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group);
	1781	+ if (ret)
	1782	+ return ret;
	1783	+
	1784	+ list_for_each_entry_safe(group, next, &group_list, entry) {
	1785	+ mutex_lock(&group->mutex);
	1786	+
	1787	+ /* Remove item from the list */
	1788	+ list_del_init(&group->entry);
	1789	+
	1790	+ /* Try to allocate default domain */
	1791	+ probe_alloc_default_domain(bus, group);
	1792	+
	1793	+ if (!group->default_domain) {
	1794	+ mutex_unlock(&group->mutex);
	1795	+ continue;
	1796	+ }
	1797	+
	1798	+ iommu_group_create_direct_mappings(group);
	1799	+
	1800	+ ret = __iommu_group_dma_attach(group);
	1801	+
	1802	+ mutex_unlock(&group->mutex);
	1803	+
	1804	+ if (ret)
	1805	+ break;
	1806	+
	1807	+ __iommu_group_dma_finalize(group);
	1808	+ }
	1809	+
	1810	+ return ret;
	1811	+}
	1812	+
1193	1813	static int iommu_bus_init(struct bus_type bus, const struct iommu_ops ops)
1194	1814	{
1195		- int err;
1196	1815	struct notifier_block *nb;
1197		- struct iommu_callback_data cb = {
1198		- .ops = ops,
1199		- };
	1816	+ int err;
1200	1817
1201	1818	nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
1202	1819	if (!nb)
..	..	@@ -1208,15 +1825,16 @@
1208	1825	if (err)
1209	1826	goto out_free;
1210	1827
1211		- err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
	1828	+ err = bus_iommu_probe(bus);
1212	1829	if (err)
1213	1830	goto out_err;
	1831	+
1214	1832
1215	1833	return 0;
1216	1834
1217	1835	out_err:
1218	1836	/* Clean up */
1219		- bus_for_each_dev(bus, NULL, &cb, remove_iommu_group);
	1837	+ bus_for_each_dev(bus, NULL, NULL, remove_iommu_group);
1220	1838	bus_unregister_notifier(bus, nb);
1221	1839
1222	1840	out_free:
..	..	@@ -1241,6 +1859,11 @@
1241	1859	int bus_set_iommu(struct bus_type bus, const struct iommu_ops ops)
1242	1860	{
1243	1861	int err;
	1862	+
	1863	+ if (ops == NULL) {
	1864	+ bus->iommu_ops = NULL;
	1865	+ return 0;
	1866	+ }
1244	1867
1245	1868	if (bus->iommu_ops != NULL)
1246	1869	return -EBUSY;
..	..	@@ -1310,8 +1933,6 @@
1310	1933	domain->type = type;
1311	1934	/* Assume all sizes by default; the driver may override this later */
1312	1935	domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
1313		- domain->is_debug_domain = false;
1314		- memset(domain->name, 0, IOMMU_DOMAIN_NAME_LEN);
1315	1936
1316	1937	return domain;
1317	1938	}
..	..	@@ -1333,28 +1954,12 @@
1333	1954	{
1334	1955	int ret;
1335	1956
1336		- /* Hack for disable iommu */
1337		- if (!domain) {
1338		- ret = dev->bus->iommu_ops->attach_dev(domain, dev);
1339		- return ret;
1340		- }
1341		-
1342		- if ((domain->ops->is_attach_deferred != NULL) &&
1343		- domain->ops->is_attach_deferred(domain, dev))
1344		- return 0;
1345		-
1346	1957	if (unlikely(domain->ops->attach_dev == NULL))
1347	1958	return -ENODEV;
1348	1959
1349	1960	ret = domain->ops->attach_dev(domain, dev);
1350		- if (!ret) {
	1961	+ if (!ret)
1351	1962	trace_attach_device_to_domain(dev);
1352		-
1353		- if (!strnlen(domain->name, IOMMU_DOMAIN_NAME_LEN)) {
1354		- strlcpy(domain->name, dev_name(dev),
1355		- IOMMU_DOMAIN_NAME_LEN);
1356		- }
1357		- }
1358	1963	return ret;
1359	1964	}
1360	1965
..	..	@@ -1388,11 +1993,220 @@
1388	1993	}
1389	1994	EXPORT_SYMBOL_GPL(iommu_attach_device);
1390	1995
	1996	+/*
	1997	+ * Check flags and other user provided data for valid combinations. We also
	1998	+ * make sure no reserved fields or unused flags are set. This is to ensure
	1999	+ * not breaking userspace in the future when these fields or flags are used.
	2000	+ */
	2001	+static int iommu_check_cache_invl_data(struct iommu_cache_invalidate_info *info)
	2002	+{
	2003	+ u32 mask;
	2004	+ int i;
	2005	+
	2006	+ if (info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1)
	2007	+ return -EINVAL;
	2008	+
	2009	+ mask = (1 << IOMMU_CACHE_INV_TYPE_NR) - 1;
	2010	+ if (info->cache & ~mask)
	2011	+ return -EINVAL;
	2012	+
	2013	+ if (info->granularity >= IOMMU_INV_GRANU_NR)
	2014	+ return -EINVAL;
	2015	+
	2016	+ switch (info->granularity) {
	2017	+ case IOMMU_INV_GRANU_ADDR:
	2018	+ if (info->cache & IOMMU_CACHE_INV_TYPE_PASID)
	2019	+ return -EINVAL;
	2020	+
	2021	+ mask = IOMMU_INV_ADDR_FLAGS_PASID \|
	2022	+ IOMMU_INV_ADDR_FLAGS_ARCHID \|
	2023	+ IOMMU_INV_ADDR_FLAGS_LEAF;
	2024	+
	2025	+ if (info->granu.addr_info.flags & ~mask)
	2026	+ return -EINVAL;
	2027	+ break;
	2028	+ case IOMMU_INV_GRANU_PASID:
	2029	+ mask = IOMMU_INV_PASID_FLAGS_PASID \|
	2030	+ IOMMU_INV_PASID_FLAGS_ARCHID;
	2031	+ if (info->granu.pasid_info.flags & ~mask)
	2032	+ return -EINVAL;
	2033	+
	2034	+ break;
	2035	+ case IOMMU_INV_GRANU_DOMAIN:
	2036	+ if (info->cache & IOMMU_CACHE_INV_TYPE_DEV_IOTLB)
	2037	+ return -EINVAL;
	2038	+ break;
	2039	+ default:
	2040	+ return -EINVAL;
	2041	+ }
	2042	+
	2043	+ /* Check reserved padding fields */
	2044	+ for (i = 0; i < sizeof(info->padding); i++) {
	2045	+ if (info->padding[i])
	2046	+ return -EINVAL;
	2047	+ }
	2048	+
	2049	+ return 0;
	2050	+}
	2051	+
	2052	+int iommu_uapi_cache_invalidate(struct iommu_domain domain, struct device dev,
	2053	+ void __user *uinfo)
	2054	+{
	2055	+ struct iommu_cache_invalidate_info inv_info = { 0 };
	2056	+ u32 minsz;
	2057	+ int ret;
	2058	+
	2059	+ if (unlikely(!domain->ops->cache_invalidate))
	2060	+ return -ENODEV;
	2061	+
	2062	+ /*
	2063	+ * No new spaces can be added before the variable sized union, the
	2064	+ * minimum size is the offset to the union.
	2065	+ */
	2066	+ minsz = offsetof(struct iommu_cache_invalidate_info, granu);
	2067	+
	2068	+ /* Copy minsz from user to get flags and argsz */
	2069	+ if (copy_from_user(&inv_info, uinfo, minsz))
	2070	+ return -EFAULT;
	2071	+
	2072	+ /* Fields before the variable size union are mandatory */
	2073	+ if (inv_info.argsz < minsz)
	2074	+ return -EINVAL;
	2075	+
	2076	+ /* PASID and address granu require additional info beyond minsz */
	2077	+ if (inv_info.granularity == IOMMU_INV_GRANU_PASID &&
	2078	+ inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.pasid_info))
	2079	+ return -EINVAL;
	2080	+
	2081	+ if (inv_info.granularity == IOMMU_INV_GRANU_ADDR &&
	2082	+ inv_info.argsz < offsetofend(struct iommu_cache_invalidate_info, granu.addr_info))
	2083	+ return -EINVAL;
	2084	+
	2085	+ /*
	2086	+ * User might be using a newer UAPI header which has a larger data
	2087	+ * size, we shall support the existing flags within the current
	2088	+ * size. Copy the remaining user data _after_ minsz but not more
	2089	+ * than the current kernel supported size.
	2090	+ */
	2091	+ if (copy_from_user((void *)&inv_info + minsz, uinfo + minsz,
	2092	+ min_t(u32, inv_info.argsz, sizeof(inv_info)) - minsz))
	2093	+ return -EFAULT;
	2094	+
	2095	+ /* Now the argsz is validated, check the content */
	2096	+ ret = iommu_check_cache_invl_data(&inv_info);
	2097	+ if (ret)
	2098	+ return ret;
	2099	+
	2100	+ return domain->ops->cache_invalidate(domain, dev, &inv_info);
	2101	+}
	2102	+EXPORT_SYMBOL_GPL(iommu_uapi_cache_invalidate);
	2103	+
	2104	+static int iommu_check_bind_data(struct iommu_gpasid_bind_data *data)
	2105	+{
	2106	+ u64 mask;
	2107	+ int i;
	2108	+
	2109	+ if (data->version != IOMMU_GPASID_BIND_VERSION_1)
	2110	+ return -EINVAL;
	2111	+
	2112	+ /* Check the range of supported formats */
	2113	+ if (data->format >= IOMMU_PASID_FORMAT_LAST)
	2114	+ return -EINVAL;
	2115	+
	2116	+ /* Check all flags */
	2117	+ mask = IOMMU_SVA_GPASID_VAL;
	2118	+ if (data->flags & ~mask)
	2119	+ return -EINVAL;
	2120	+
	2121	+ /* Check reserved padding fields */
	2122	+ for (i = 0; i < sizeof(data->padding); i++) {
	2123	+ if (data->padding[i])
	2124	+ return -EINVAL;
	2125	+ }
	2126	+
	2127	+ return 0;
	2128	+}
	2129	+
	2130	+static int iommu_sva_prepare_bind_data(void __user *udata,
	2131	+ struct iommu_gpasid_bind_data *data)
	2132	+{
	2133	+ u32 minsz;
	2134	+
	2135	+ /*
	2136	+ * No new spaces can be added before the variable sized union, the
	2137	+ * minimum size is the offset to the union.
	2138	+ */
	2139	+ minsz = offsetof(struct iommu_gpasid_bind_data, vendor);
	2140	+
	2141	+ /* Copy minsz from user to get flags and argsz */
	2142	+ if (copy_from_user(data, udata, minsz))
	2143	+ return -EFAULT;
	2144	+
	2145	+ /* Fields before the variable size union are mandatory */
	2146	+ if (data->argsz < minsz)
	2147	+ return -EINVAL;
	2148	+ /*
	2149	+ * User might be using a newer UAPI header, we shall let IOMMU vendor
	2150	+ * driver decide on what size it needs. Since the guest PASID bind data
	2151	+ * can be vendor specific, larger argsz could be the result of extension
	2152	+ * for one vendor but it should not affect another vendor.
	2153	+ * Copy the remaining user data _after_ minsz
	2154	+ */
	2155	+ if (copy_from_user((void *)data + minsz, udata + minsz,
	2156	+ min_t(u32, data->argsz, sizeof(*data)) - minsz))
	2157	+ return -EFAULT;
	2158	+
	2159	+ return iommu_check_bind_data(data);
	2160	+}
	2161	+
	2162	+int iommu_uapi_sva_bind_gpasid(struct iommu_domain domain, struct device dev,
	2163	+ void __user *udata)
	2164	+{
	2165	+ struct iommu_gpasid_bind_data data = { 0 };
	2166	+ int ret;
	2167	+
	2168	+ if (unlikely(!domain->ops->sva_bind_gpasid))
	2169	+ return -ENODEV;
	2170	+
	2171	+ ret = iommu_sva_prepare_bind_data(udata, &data);
	2172	+ if (ret)
	2173	+ return ret;
	2174	+
	2175	+ return domain->ops->sva_bind_gpasid(domain, dev, &data);
	2176	+}
	2177	+EXPORT_SYMBOL_GPL(iommu_uapi_sva_bind_gpasid);
	2178	+
	2179	+int iommu_sva_unbind_gpasid(struct iommu_domain domain, struct device dev,
	2180	+ ioasid_t pasid)
	2181	+{
	2182	+ if (unlikely(!domain->ops->sva_unbind_gpasid))
	2183	+ return -ENODEV;
	2184	+
	2185	+ return domain->ops->sva_unbind_gpasid(dev, pasid);
	2186	+}
	2187	+EXPORT_SYMBOL_GPL(iommu_sva_unbind_gpasid);
	2188	+
	2189	+int iommu_uapi_sva_unbind_gpasid(struct iommu_domain domain, struct device dev,
	2190	+ void __user *udata)
	2191	+{
	2192	+ struct iommu_gpasid_bind_data data = { 0 };
	2193	+ int ret;
	2194	+
	2195	+ if (unlikely(!domain->ops->sva_bind_gpasid))
	2196	+ return -ENODEV;
	2197	+
	2198	+ ret = iommu_sva_prepare_bind_data(udata, &data);
	2199	+ if (ret)
	2200	+ return ret;
	2201	+
	2202	+ return iommu_sva_unbind_gpasid(domain, dev, data.hpasid);
	2203	+}
	2204	+EXPORT_SYMBOL_GPL(iommu_uapi_sva_unbind_gpasid);
	2205	+
1391	2206	static void __iommu_detach_device(struct iommu_domain *domain,
1392	2207	struct device *dev)
1393	2208	{
1394		- if ((domain->ops->is_attach_deferred != NULL) &&
1395		- domain->ops->is_attach_deferred(domain, dev))
	2209	+ if (iommu_is_attach_deferred(domain, dev))
1396	2210	return;
1397	2211
1398	2212	if (unlikely(domain->ops->detach_dev == NULL))
..	..	@@ -1443,7 +2257,16 @@
1443	2257	EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
1444	2258
1445	2259	/*
1446		- * IOMMU groups are really the natrual working unit of the IOMMU, but
	2260	+ * For IOMMU_DOMAIN_DMA implementations which already provide their own
	2261	+ * guarantees that the group and its default domain are valid and correct.
	2262	+ */
	2263	+struct iommu_domain iommu_get_dma_domain(struct device dev)
	2264	+{
	2265	+ return dev->iommu_group->default_domain;
	2266	+}
	2267	+
	2268	+/*
	2269	+ * IOMMU groups are really the natural working unit of the IOMMU, but
1447	2270	* the IOMMU API works on domains and devices. Bridge that gap by
1448	2271	* iterating over the devices in a group. Ideally we'd have a single
1449	2272	* device which represents the requestor ID of the group, but we also
..	..	@@ -1463,6 +2286,9 @@
1463	2286	struct iommu_group *group)
1464	2287	{
1465	2288	int ret;
	2289	+
	2290	+ if (group->default_domain && group->domain != group->default_domain)
	2291	+ return -EBUSY;
1466	2292
1467	2293	ret = __iommu_group_for_each_dev(group, domain,
1468	2294	iommu_group_do_attach_device);
..	..	@@ -1493,18 +2319,28 @@
1493	2319	return 0;
1494	2320	}
1495	2321
1496		-/*
1497		- * Although upstream implements detaching the default_domain as a noop,
1498		- * the "SID switch" secure usecase require complete removal of SIDS/SMRS
1499		- * from HLOS iommu registers.
1500		- */
1501	2322	static void __iommu_detach_group(struct iommu_domain *domain,
1502	2323	struct iommu_group *group)
1503	2324	{
1504		- __iommu_group_for_each_dev(group, domain,
	2325	+ int ret;
	2326	+
	2327	+ if (!group->default_domain) {
	2328	+ __iommu_group_for_each_dev(group, domain,
1505	2329	iommu_group_do_detach_device);
1506		- group->domain = NULL;
1507		- return;
	2330	+ group->domain = NULL;
	2331	+ return;
	2332	+ }
	2333	+
	2334	+ if (group->domain == group->default_domain)
	2335	+ return;
	2336	+
	2337	+ /* Detach by re-attaching to the default domain */
	2338	+ ret = __iommu_group_for_each_dev(group, group->default_domain,
	2339	+ iommu_group_do_attach_device);
	2340	+ if (ret != 0)
	2341	+ WARN_ON(1);
	2342	+ else
	2343	+ group->domain = group->default_domain;
1508	2344	}
1509	2345
1510	2346	void iommu_detach_group(struct iommu_domain domain, struct iommu_group group)
..	..	@@ -1524,78 +2360,96 @@
1524	2360	}
1525	2361	EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
1526	2362
1527		-phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
1528		- dma_addr_t iova)
	2363	+static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
	2364	+ phys_addr_t paddr, size_t size, size_t *count)
1529	2365	{
1530		- if (unlikely(domain->ops->iova_to_phys_hard == NULL))
1531		- return 0;
	2366	+ unsigned int pgsize_idx, pgsize_idx_next;
	2367	+ unsigned long pgsizes;
	2368	+ size_t offset, pgsize, pgsize_next;
	2369	+ unsigned long addr_merge = paddr \| iova;
1532	2370
1533		- return domain->ops->iova_to_phys_hard(domain, iova);
1534		-}
	2371	+ /* Page sizes supported by the hardware and small enough for @size */
	2372	+ pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0);
1535	2373
1536		-uint64_t iommu_iova_to_pte(struct iommu_domain *domain,
1537		- dma_addr_t iova)
1538		-{
1539		- if (unlikely(domain->ops->iova_to_pte == NULL))
1540		- return 0;
	2374	+ /* Constrain the page sizes further based on the maximum alignment */
	2375	+ if (likely(addr_merge))
	2376	+ pgsizes &= GENMASK(__ffs(addr_merge), 0);
1541	2377
1542		- return domain->ops->iova_to_pte(domain, iova);
1543		-}
	2378	+ /* Make sure we have at least one suitable page size */
	2379	+ BUG_ON(!pgsizes);
1544	2380
1545		-bool iommu_is_iova_coherent(struct iommu_domain *domain, dma_addr_t iova)
1546		-{
1547		- if (unlikely(domain->ops->is_iova_coherent == NULL))
1548		- return 0;
	2381	+ /* Pick the biggest page size remaining */
	2382	+ pgsize_idx = __fls(pgsizes);
	2383	+ pgsize = BIT(pgsize_idx);
	2384	+ if (!count)
	2385	+ return pgsize;
1549	2386
1550		- return domain->ops->is_iova_coherent(domain, iova);
1551		-}
1552	2387
1553		-size_t iommu_pgsize(unsigned long pgsize_bitmap,
1554		- unsigned long addr_merge, size_t size)
1555		-{
1556		- unsigned int pgsize_idx;
1557		- size_t pgsize;
	2388	+ /* Find the next biggest support page size, if it exists */
	2389	+ pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
	2390	+ if (!pgsizes)
	2391	+ goto out_set_count;
1558	2392
1559		- /* Max page size that still fits into 'size' */
1560		- pgsize_idx = __fls(size);
	2393	+ pgsize_idx_next = __ffs(pgsizes);
	2394	+ pgsize_next = BIT(pgsize_idx_next);
1561	2395
1562		- /* need to consider alignment requirements ? */
1563		- if (likely(addr_merge)) {
1564		- /* Max page size allowed by address */
1565		- unsigned int align_pgsize_idx = __ffs(addr_merge);
1566		- pgsize_idx = min(pgsize_idx, align_pgsize_idx);
1567		- }
	2396	+ /*
	2397	+ * There's no point trying a bigger page size unless the virtual
	2398	+ * and physical addresses are similarly offset within the larger page.
	2399	+ */
	2400	+ if ((iova ^ paddr) & (pgsize_next - 1))
	2401	+ goto out_set_count;
1568	2402
1569		- /* build a mask of acceptable page sizes */
1570		- pgsize = (1UL << (pgsize_idx + 1)) - 1;
	2403	+ /* Calculate the offset to the next page size alignment boundary */
	2404	+ offset = pgsize_next - (addr_merge & (pgsize_next - 1));
1571	2405
1572		- /* throw away page sizes not supported by the hardware */
1573		- pgsize &= pgsize_bitmap;
	2406	+ /*
	2407	+ * If size is big enough to accommodate the larger page, reduce
	2408	+ * the number of smaller pages.
	2409	+ */
	2410	+ if (offset + pgsize_next <= size)
	2411	+ size = offset;
1574	2412
1575		- /* make sure we're still sane */
1576		- if (!pgsize) {
1577		- pr_err("invalid pgsize/addr/size! 0x%lx 0x%lx 0x%zx\n",
1578		- pgsize_bitmap, addr_merge, size);
1579		- BUG();
1580		- }
1581		-
1582		- /* pick the biggest page */
1583		- pgsize_idx = __fls(pgsize);
1584		- pgsize = 1UL << pgsize_idx;
1585		-
	2413	+out_set_count:
	2414	+ *count = size >> pgsize_idx;
1586	2415	return pgsize;
1587	2416	}
1588	2417
1589		-int iommu_map(struct iommu_domain *domain, unsigned long iova,
1590		- phys_addr_t paddr, size_t size, int prot)
	2418	+static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova,
	2419	+ phys_addr_t paddr, size_t size, int prot,
	2420	+ gfp_t gfp, size_t *mapped)
1591	2421	{
	2422	+ const struct iommu_ops *ops = domain->ops;
	2423	+ size_t pgsize, count;
	2424	+ int ret;
	2425	+
	2426	+ pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
	2427	+
	2428	+ pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
	2429	+ iova, &paddr, pgsize, count);
	2430	+
	2431	+ if (ops->map_pages) {
	2432	+ ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
	2433	+ gfp, mapped);
	2434	+ } else {
	2435	+ ret = ops->map(domain, iova, paddr, pgsize, prot, gfp);
	2436	+ *mapped = ret ? 0 : pgsize;
	2437	+ }
	2438	+
	2439	+ return ret;
	2440	+}
	2441	+
	2442	+static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
	2443	+ phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
	2444	+{
	2445	+ const struct iommu_ops *ops = domain->ops;
1592	2446	unsigned long orig_iova = iova;
1593	2447	unsigned int min_pagesz;
1594	2448	size_t orig_size = size;
1595	2449	phys_addr_t orig_paddr = paddr;
1596	2450	int ret = 0;
1597	2451
1598		- if (unlikely(domain->ops->map == NULL \|\|
	2452	+ if (unlikely(!(ops->map \|\| ops->map_pages) \|\|
1599	2453	domain->pgsize_bitmap == 0UL))
1600	2454	return -ENODEV;
1601	2455
..	..	@@ -1619,41 +2473,83 @@
1619	2473	pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
1620	2474
1621	2475	while (size) {
1622		- size_t pgsize = iommu_pgsize(domain->pgsize_bitmap,
1623		- iova \| paddr, size);
	2476	+ size_t mapped = 0;
1624	2477
1625		- pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
1626		- iova, &paddr, pgsize);
	2478	+ ret = __iommu_map_pages(domain, iova, paddr, size, prot, gfp,
	2479	+ &mapped);
	2480	+ /*
	2481	+ * Some pages may have been mapped, even if an error occurred,
	2482	+ * so we should account for those so they can be unmapped.
	2483	+ */
	2484	+ size -= mapped;
1627	2485
1628		- ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
1629	2486	if (ret)
1630	2487	break;
1631	2488
1632		- iova += pgsize;
1633		- paddr += pgsize;
1634		- size -= pgsize;
	2489	+ iova += mapped;
	2490	+ paddr += mapped;
1635	2491	}
1636	2492
1637	2493	/* unroll mapping in case something went wrong */
1638	2494	if (ret)
1639	2495	iommu_unmap(domain, orig_iova, orig_size - size);
1640	2496	else
1641		- trace_map(domain, orig_iova, orig_paddr, orig_size, prot);
	2497	+ trace_map(orig_iova, orig_paddr, orig_size);
1642	2498
1643	2499	return ret;
1644	2500	}
	2501	+
	2502	+static int _iommu_map(struct iommu_domain *domain, unsigned long iova,
	2503	+ phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
	2504	+{
	2505	+ const struct iommu_ops *ops = domain->ops;
	2506	+ int ret;
	2507	+
	2508	+ ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
	2509	+ if (ret == 0 && ops->iotlb_sync_map)
	2510	+ ops->iotlb_sync_map(domain, iova, size);
	2511	+
	2512	+ return ret;
	2513	+}
	2514	+
	2515	+int iommu_map(struct iommu_domain *domain, unsigned long iova,
	2516	+ phys_addr_t paddr, size_t size, int prot)
	2517	+{
	2518	+ might_sleep();
	2519	+ return _iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL);
	2520	+}
1645	2521	EXPORT_SYMBOL_GPL(iommu_map);
	2522	+
	2523	+int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
	2524	+ phys_addr_t paddr, size_t size, int prot)
	2525	+{
	2526	+ return _iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC);
	2527	+}
	2528	+EXPORT_SYMBOL_GPL(iommu_map_atomic);
	2529	+
	2530	+static size_t __iommu_unmap_pages(struct iommu_domain *domain,
	2531	+ unsigned long iova, size_t size,
	2532	+ struct iommu_iotlb_gather *iotlb_gather)
	2533	+{
	2534	+ const struct iommu_ops *ops = domain->ops;
	2535	+ size_t pgsize, count;
	2536	+
	2537	+ pgsize = iommu_pgsize(domain, iova, iova, size, &count);
	2538	+ return ops->unmap_pages ?
	2539	+ ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather) :
	2540	+ ops->unmap(domain, iova, pgsize, iotlb_gather);
	2541	+}
1646	2542
1647	2543	static size_t __iommu_unmap(struct iommu_domain *domain,
1648	2544	unsigned long iova, size_t size,
1649		- bool sync)
	2545	+ struct iommu_iotlb_gather *iotlb_gather)
1650	2546	{
1651	2547	const struct iommu_ops *ops = domain->ops;
1652	2548	size_t unmapped_page, unmapped = 0;
1653	2549	unsigned long orig_iova = iova;
1654	2550	unsigned int min_pagesz;
1655	2551
1656		- if (unlikely(ops->unmap == NULL \|\|
	2552	+ if (unlikely(!(ops->unmap \|\| ops->unmap_pages) \|\|
1657	2553	domain->pgsize_bitmap == 0UL))
1658	2554	return 0;
1659	2555
..	..	@@ -1681,14 +2577,11 @@
1681	2577	* or we hit an area that isn't mapped.
1682	2578	*/
1683	2579	while (unmapped < size) {
1684		- size_t pgsize = iommu_pgsize(domain->pgsize_bitmap, iova, size - unmapped);
1685		-
1686		- unmapped_page = ops->unmap(domain, iova, pgsize);
	2580	+ unmapped_page = __iommu_unmap_pages(domain, iova,
	2581	+ size - unmapped,
	2582	+ iotlb_gather);
1687	2583	if (!unmapped_page)
1688	2584	break;
1689		-
1690		- if (sync && ops->iotlb_range_add)
1691		- ops->iotlb_range_add(domain, iova, pgsize);
1692	2585
1693	2586	pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
1694	2587	iova, unmapped_page);
..	..	@@ -1697,71 +2590,81 @@
1697	2590	unmapped += unmapped_page;
1698	2591	}
1699	2592
1700		- if (sync && ops->iotlb_sync)
1701		- ops->iotlb_sync(domain);
1702		-
1703		- trace_unmap(domain, orig_iova, size, unmapped);
	2593	+ trace_unmap(orig_iova, size, unmapped);
1704	2594	return unmapped;
1705	2595	}
1706	2596
1707	2597	size_t iommu_unmap(struct iommu_domain *domain,
1708	2598	unsigned long iova, size_t size)
1709	2599	{
1710		- return __iommu_unmap(domain, iova, size, true);
	2600	+ struct iommu_iotlb_gather iotlb_gather;
	2601	+ size_t ret;
	2602	+
	2603	+ iommu_iotlb_gather_init(&iotlb_gather);
	2604	+ ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
	2605	+ iommu_iotlb_sync(domain, &iotlb_gather);
	2606	+
	2607	+ return ret;
1711	2608	}
1712	2609	EXPORT_SYMBOL_GPL(iommu_unmap);
1713	2610
1714	2611	size_t iommu_unmap_fast(struct iommu_domain *domain,
1715		- unsigned long iova, size_t size)
	2612	+ unsigned long iova, size_t size,
	2613	+ struct iommu_iotlb_gather *iotlb_gather)
1716	2614	{
1717		- return __iommu_unmap(domain, iova, size, false);
	2615	+ return __iommu_unmap(domain, iova, size, iotlb_gather);
1718	2616	}
1719	2617	EXPORT_SYMBOL_GPL(iommu_unmap_fast);
1720	2618
1721		-size_t iommu_map_sg(struct iommu_domain *domain,
1722		- unsigned long iova, struct scatterlist *sg,
1723		- unsigned int nents, int prot)
	2619	+static size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
	2620	+ struct scatterlist *sg, unsigned int nents, int prot,
	2621	+ gfp_t gfp)
1724	2622	{
1725		- size_t mapped;
1726		-
1727		- mapped = domain->ops->map_sg(domain, iova, sg, nents, prot);
1728		- trace_map_sg(domain, iova, mapped, prot);
1729		- return mapped;
1730		-}
1731		-EXPORT_SYMBOL(iommu_map_sg);
1732		-
1733		-size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
1734		- struct scatterlist *sg, unsigned int nents, int prot)
1735		-{
1736		- struct scatterlist *s;
1737		- size_t mapped = 0;
1738		- unsigned int i, min_pagesz;
	2623	+ const struct iommu_ops *ops = domain->ops;
	2624	+ size_t len = 0, mapped = 0;
	2625	+ phys_addr_t start;
	2626	+ unsigned int i = 0;
1739	2627	int ret;
1740	2628
1741		- if (unlikely(domain->pgsize_bitmap == 0UL))
1742		- return 0;
	2629	+ if (ops->map_sg) {
	2630	+ ret = ops->map_sg(domain, iova, sg, nents, prot, gfp, &mapped);
1743	2631
1744		- min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
	2632	+ if (ops->iotlb_sync_map)
	2633	+ ops->iotlb_sync_map(domain, iova, mapped);
1745	2634
1746		- for_each_sg(sg, s, nents, i) {
1747		- phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
1748		-
1749		- /*
1750		- * We are mapping on IOMMU page boundaries, so offset within
1751		- * the page must be 0. However, the IOMMU may support pages
1752		- * smaller than PAGE_SIZE, so s->offset may still represent
1753		- * an offset of that boundary within the CPU page.
1754		- */
1755		- if (!IS_ALIGNED(s->offset, min_pagesz))
1756		- goto out_err;
1757		-
1758		- ret = iommu_map(domain, iova + mapped, phys, s->length, prot);
1759	2635	if (ret)
1760	2636	goto out_err;
1761	2637
1762		- mapped += s->length;
	2638	+ return mapped;
1763	2639	}
1764	2640
	2641	+ while (i <= nents) {
	2642	+ phys_addr_t s_phys = sg_phys(sg);
	2643	+
	2644	+ if (len && s_phys != start + len) {
	2645	+ ret = __iommu_map(domain, iova + mapped, start,
	2646	+ len, prot, gfp);
	2647	+
	2648	+ if (ret)
	2649	+ goto out_err;
	2650	+
	2651	+ mapped += len;
	2652	+ len = 0;
	2653	+ }
	2654	+
	2655	+ if (len) {
	2656	+ len += sg->length;
	2657	+ } else {
	2658	+ len = sg->length;
	2659	+ start = s_phys;
	2660	+ }
	2661	+
	2662	+ if (++i < nents)
	2663	+ sg = sg_next(sg);
	2664	+ }
	2665	+
	2666	+ if (ops->iotlb_sync_map)
	2667	+ ops->iotlb_sync_map(domain, iova, mapped);
1765	2668	return mapped;
1766	2669
1767	2670	out_err:
..	..	@@ -1771,7 +2674,21 @@
1771	2674	return 0;
1772	2675
1773	2676	}
1774		-EXPORT_SYMBOL_GPL(default_iommu_map_sg);
	2677	+
	2678	+size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
	2679	+ struct scatterlist *sg, unsigned int nents, int prot)
	2680	+{
	2681	+ might_sleep();
	2682	+ return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL);
	2683	+}
	2684	+EXPORT_SYMBOL_GPL(iommu_map_sg);
	2685	+
	2686	+size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova,
	2687	+ struct scatterlist *sg, unsigned int nents, int prot)
	2688	+{
	2689	+ return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC);
	2690	+}
	2691	+EXPORT_SYMBOL_GPL(iommu_map_sg_atomic);
1775	2692
1776	2693	int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
1777	2694	phys_addr_t paddr, u64 size, int prot)
..	..	@@ -1835,9 +2752,6 @@
1835	2752	}
1836	2753	EXPORT_SYMBOL_GPL(report_iommu_fault);
1837	2754
1838		-struct dentry *iommu_debugfs_top;
1839		-EXPORT_SYMBOL_GPL(iommu_debugfs_top);
1840		-
1841	2755	static int __init iommu_init(void)
1842	2756	{
1843	2757	iommu_group_kset = kset_create_and_add("iommu_groups",
..	..	@@ -1856,7 +2770,6 @@
1856	2770	struct iommu_domain_geometry *geometry;
1857	2771	bool *paging;
1858	2772	int ret = 0;
1859		- u32 *count;
1860	2773
1861	2774	switch (attr) {
1862	2775	case DOMAIN_ATTR_GEOMETRY:
..	..	@@ -1867,15 +2780,6 @@
1867	2780	case DOMAIN_ATTR_PAGING:
1868	2781	paging = data;
1869	2782	*paging = (domain->pgsize_bitmap != 0UL);
1870		- break;
1871		- case DOMAIN_ATTR_WINDOWS:
1872		- count = data;
1873		-
1874		- if (domain->ops->domain_get_windows != NULL)
1875		- *count = domain->ops->domain_get_windows(domain);
1876		- else
1877		- ret = -ENODEV;
1878		-
1879	2783	break;
1880	2784	default:
1881	2785	if (!domain->ops->domain_get_attr)
..	..	@@ -1892,18 +2796,8 @@
1892	2796	enum iommu_attr attr, void *data)
1893	2797	{
1894	2798	int ret = 0;
1895		- u32 *count;
1896	2799
1897	2800	switch (attr) {
1898		- case DOMAIN_ATTR_WINDOWS:
1899		- count = data;
1900		-
1901		- if (domain->ops->domain_set_windows != NULL)
1902		- ret = domain->ops->domain_set_windows(domain, *count);
1903		- else
1904		- ret = -ENODEV;
1905		-
1906		- break;
1907	2801	default:
1908	2802	if (domain->ops->domain_set_attr == NULL)
1909	2803	return -EINVAL;
..	..	@@ -1932,19 +2826,23 @@
1932	2826	}
1933	2827
1934	2828	/**
1935		- * iommu_trigger_fault() - trigger an IOMMU fault
1936		- * @domain: iommu domain
	2829	+ * generic_iommu_put_resv_regions - Reserved region driver helper
	2830	+ * @dev: device for which to free reserved regions
	2831	+ * @list: reserved region list for device
1937	2832	*
1938		- * Triggers a fault on the device to which this domain is attached.
1939		- *
1940		- * This function should only be used for debugging purposes, for obvious
1941		- * reasons.
	2833	+ * IOMMU drivers can use this to implement their .put_resv_regions() callback
	2834	+ * for simple reservations. Memory allocated for each reserved region will be
	2835	+ * freed. If an IOMMU driver allocates additional resources per region, it is
	2836	+ * going to have to implement a custom callback.
1942	2837	*/
1943		-void iommu_trigger_fault(struct iommu_domain *domain, unsigned long flags)
	2838	+void generic_iommu_put_resv_regions(struct device dev, struct list_head list)
1944	2839	{
1945		- if (domain->ops->trigger_fault)
1946		- domain->ops->trigger_fault(domain, flags);
	2840	+ struct iommu_resv_region entry, next;
	2841	+
	2842	+ list_for_each_entry_safe(entry, next, list, list)
	2843	+ kfree(entry);
1947	2844	}
	2845	+EXPORT_SYMBOL(generic_iommu_put_resv_regions);
1948	2846
1949	2847	struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
1950	2848	size_t length, int prot,
..	..	@@ -1965,58 +2863,27 @@
1965	2863	}
1966	2864	EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
1967	2865
1968		-/* Request that a device is direct mapped by the IOMMU */
1969		-int iommu_request_dm_for_dev(struct device *dev)
	2866	+void iommu_set_default_passthrough(bool cmd_line)
1970	2867	{
1971		- struct iommu_domain *dm_domain;
1972		- struct iommu_group *group;
1973		- int ret;
	2868	+ if (cmd_line)
	2869	+ iommu_set_cmd_line_dma_api();
1974	2870
1975		- /* Device must already be in a group before calling this function */
1976		- group = iommu_group_get(dev);
1977		- if (!group)
1978		- return -EINVAL;
1979		-
1980		- mutex_lock(&group->mutex);
1981		-
1982		- /* Check if the default domain is already direct mapped */
1983		- ret = 0;
1984		- if (group->default_domain &&
1985		- group->default_domain->type == IOMMU_DOMAIN_IDENTITY)
1986		- goto out;
1987		-
1988		- /* Don't change mappings of existing devices */
1989		- ret = -EBUSY;
1990		- if (iommu_group_device_count(group) != 1)
1991		- goto out;
1992		-
1993		- /* Allocate a direct mapped domain */
1994		- ret = -ENOMEM;
1995		- dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY);
1996		- if (!dm_domain)
1997		- goto out;
1998		-
1999		- /* Attach the device to the domain */
2000		- ret = __iommu_attach_group(dm_domain, group);
2001		- if (ret) {
2002		- iommu_domain_free(dm_domain);
2003		- goto out;
2004		- }
2005		-
2006		- /* Make the direct mapped domain the default for this group */
2007		- if (group->default_domain)
2008		- iommu_domain_free(group->default_domain);
2009		- group->default_domain = dm_domain;
2010		-
2011		- pr_info("Using direct mapping for device %s\n", dev_name(dev));
2012		-
2013		- ret = 0;
2014		-out:
2015		- mutex_unlock(&group->mutex);
2016		- iommu_group_put(group);
2017		-
2018		- return ret;
	2871	+ iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
2019	2872	}
	2873	+
	2874	+void iommu_set_default_translated(bool cmd_line)
	2875	+{
	2876	+ if (cmd_line)
	2877	+ iommu_set_cmd_line_dma_api();
	2878	+
	2879	+ iommu_def_domain_type = IOMMU_DOMAIN_DMA;
	2880	+}
	2881	+
	2882	+bool iommu_default_passthrough(void)
	2883	+{
	2884	+ return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
	2885	+}
	2886	+EXPORT_SYMBOL_GPL(iommu_default_passthrough);
2020	2887
2021	2888	const struct iommu_ops iommu_ops_from_fwnode(struct fwnode_handle fwnode)
2022	2889	{
..	..	@@ -2036,57 +2903,256 @@
2036	2903	int iommu_fwspec_init(struct device dev, struct fwnode_handle iommu_fwnode,
2037	2904	const struct iommu_ops *ops)
2038	2905	{
2039		- struct iommu_fwspec *fwspec = dev->iommu_fwspec;
	2906	+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2040	2907
2041	2908	if (fwspec)
2042	2909	return ops == fwspec->ops ? 0 : -EINVAL;
2043	2910
2044		- fwspec = kzalloc(sizeof(*fwspec), GFP_KERNEL);
	2911	+ if (!dev_iommu_get(dev))
	2912	+ return -ENOMEM;
	2913	+
	2914	+ /* Preallocate for the overwhelmingly common case of 1 ID */
	2915	+ fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
2045	2916	if (!fwspec)
2046	2917	return -ENOMEM;
2047	2918
2048	2919	of_node_get(to_of_node(iommu_fwnode));
2049	2920	fwspec->iommu_fwnode = iommu_fwnode;
2050	2921	fwspec->ops = ops;
2051		- dev->iommu_fwspec = fwspec;
	2922	+ dev_iommu_fwspec_set(dev, fwspec);
2052	2923	return 0;
2053	2924	}
2054	2925	EXPORT_SYMBOL_GPL(iommu_fwspec_init);
2055	2926
2056	2927	void iommu_fwspec_free(struct device *dev)
2057	2928	{
2058		- struct iommu_fwspec *fwspec = dev->iommu_fwspec;
	2929	+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2059	2930
2060	2931	if (fwspec) {
2061	2932	fwnode_handle_put(fwspec->iommu_fwnode);
2062	2933	kfree(fwspec);
2063		- dev->iommu_fwspec = NULL;
	2934	+ dev_iommu_fwspec_set(dev, NULL);
2064	2935	}
2065	2936	}
2066	2937	EXPORT_SYMBOL_GPL(iommu_fwspec_free);
2067	2938
2068	2939	int iommu_fwspec_add_ids(struct device dev, u32 ids, int num_ids)
2069	2940	{
2070		- struct iommu_fwspec *fwspec = dev->iommu_fwspec;
2071		- size_t size;
2072		- int i;
	2941	+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
	2942	+ int i, new_num;
2073	2943
2074	2944	if (!fwspec)
2075	2945	return -EINVAL;
2076	2946
2077		- size = offsetof(struct iommu_fwspec, ids[fwspec->num_ids + num_ids]);
2078		- if (size > sizeof(*fwspec)) {
2079		- fwspec = krealloc(dev->iommu_fwspec, size, GFP_KERNEL);
	2947	+ new_num = fwspec->num_ids + num_ids;
	2948	+ if (new_num > 1) {
	2949	+ fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
	2950	+ GFP_KERNEL);
2080	2951	if (!fwspec)
2081	2952	return -ENOMEM;
2082	2953
2083		- dev->iommu_fwspec = fwspec;
	2954	+ dev_iommu_fwspec_set(dev, fwspec);
2084	2955	}
2085	2956
2086	2957	for (i = 0; i < num_ids; i++)
2087	2958	fwspec->ids[fwspec->num_ids + i] = ids[i];
2088	2959
2089		- fwspec->num_ids += num_ids;
	2960	+ fwspec->num_ids = new_num;
2090	2961	return 0;
2091	2962	}
2092	2963	EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
	2964	+
	2965	+/*
	2966	+ * Per device IOMMU features.
	2967	+ */
	2968	+bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat)
	2969	+{
	2970	+ const struct iommu_ops *ops = dev->bus->iommu_ops;
	2971	+
	2972	+ if (ops && ops->dev_has_feat)
	2973	+ return ops->dev_has_feat(dev, feat);
	2974	+
	2975	+ return false;
	2976	+}
	2977	+EXPORT_SYMBOL_GPL(iommu_dev_has_feature);
	2978	+
	2979	+int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
	2980	+{
	2981	+ if (dev->iommu && dev->iommu->iommu_dev) {
	2982	+ const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;
	2983	+
	2984	+ if (ops->dev_enable_feat)
	2985	+ return ops->dev_enable_feat(dev, feat);
	2986	+ }
	2987	+
	2988	+ return -ENODEV;
	2989	+}
	2990	+EXPORT_SYMBOL_GPL(iommu_dev_enable_feature);
	2991	+
	2992	+/*
	2993	+ * The device drivers should do the necessary cleanups before calling this.
	2994	+ * For example, before disabling the aux-domain feature, the device driver
	2995	+ * should detach all aux-domains. Otherwise, this will return -EBUSY.
	2996	+ */
	2997	+int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
	2998	+{
	2999	+ if (dev->iommu && dev->iommu->iommu_dev) {
	3000	+ const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;
	3001	+
	3002	+ if (ops->dev_disable_feat)
	3003	+ return ops->dev_disable_feat(dev, feat);
	3004	+ }
	3005	+
	3006	+ return -EBUSY;
	3007	+}
	3008	+EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
	3009	+
	3010	+bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat)
	3011	+{
	3012	+ if (dev->iommu && dev->iommu->iommu_dev) {
	3013	+ const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;
	3014	+
	3015	+ if (ops->dev_feat_enabled)
	3016	+ return ops->dev_feat_enabled(dev, feat);
	3017	+ }
	3018	+
	3019	+ return false;
	3020	+}
	3021	+EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled);
	3022	+
	3023	+/*
	3024	+ * Aux-domain specific attach/detach.
	3025	+ *
	3026	+ * Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns
	3027	+ * true. Also, as long as domains are attached to a device through this
	3028	+ * interface, any tries to call iommu_attach_device() should fail
	3029	+ * (iommu_detach_device() can't fail, so we fail when trying to re-attach).
	3030	+ * This should make us safe against a device being attached to a guest as a
	3031	+ * whole while there are still pasid users on it (aux and sva).
	3032	+ */
	3033	+int iommu_aux_attach_device(struct iommu_domain domain, struct device dev)
	3034	+{
	3035	+ int ret = -ENODEV;
	3036	+
	3037	+ if (domain->ops->aux_attach_dev)
	3038	+ ret = domain->ops->aux_attach_dev(domain, dev);
	3039	+
	3040	+ if (!ret)
	3041	+ trace_attach_device_to_domain(dev);
	3042	+
	3043	+ return ret;
	3044	+}
	3045	+EXPORT_SYMBOL_GPL(iommu_aux_attach_device);
	3046	+
	3047	+void iommu_aux_detach_device(struct iommu_domain domain, struct device dev)
	3048	+{
	3049	+ if (domain->ops->aux_detach_dev) {
	3050	+ domain->ops->aux_detach_dev(domain, dev);
	3051	+ trace_detach_device_from_domain(dev);
	3052	+ }
	3053	+}
	3054	+EXPORT_SYMBOL_GPL(iommu_aux_detach_device);
	3055	+
	3056	+int iommu_aux_get_pasid(struct iommu_domain domain, struct device dev)
	3057	+{
	3058	+ int ret = -ENODEV;
	3059	+
	3060	+ if (domain->ops->aux_get_pasid)
	3061	+ ret = domain->ops->aux_get_pasid(domain, dev);
	3062	+
	3063	+ return ret;
	3064	+}
	3065	+EXPORT_SYMBOL_GPL(iommu_aux_get_pasid);
	3066	+
	3067	+/**
	3068	+ * iommu_sva_bind_device() - Bind a process address space to a device
	3069	+ * @dev: the device
	3070	+ * @mm: the mm to bind, caller must hold a reference to it
	3071	+ *
	3072	+ * Create a bond between device and address space, allowing the device to access
	3073	+ * the mm using the returned PASID. If a bond already exists between @device and
	3074	+ * @mm, it is returned and an additional reference is taken. Caller must call
	3075	+ * iommu_sva_unbind_device() to release each reference.
	3076	+ *
	3077	+ * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
	3078	+ * initialize the required SVA features.
	3079	+ *
	3080	+ * On error, returns an ERR_PTR value.
	3081	+ */
	3082	+struct iommu_sva *
	3083	+iommu_sva_bind_device(struct device dev, struct mm_struct mm, void *drvdata)
	3084	+{
	3085	+ struct iommu_group *group;
	3086	+ struct iommu_sva *handle = ERR_PTR(-EINVAL);
	3087	+ const struct iommu_ops *ops = dev->bus->iommu_ops;
	3088	+
	3089	+ if (!ops \|\| !ops->sva_bind)
	3090	+ return ERR_PTR(-ENODEV);
	3091	+
	3092	+ group = iommu_group_get(dev);
	3093	+ if (!group)
	3094	+ return ERR_PTR(-ENODEV);
	3095	+
	3096	+ /* Ensure device count and domain don't change while we're binding */
	3097	+ mutex_lock(&group->mutex);
	3098	+
	3099	+ /*
	3100	+ * To keep things simple, SVA currently doesn't support IOMMU groups
	3101	+ * with more than one device. Existing SVA-capable systems are not
	3102	+ * affected by the problems that required IOMMU groups (lack of ACS
	3103	+ * isolation, device ID aliasing and other hardware issues).
	3104	+ */
	3105	+ if (iommu_group_device_count(group) != 1)
	3106	+ goto out_unlock;
	3107	+
	3108	+ handle = ops->sva_bind(dev, mm, drvdata);
	3109	+
	3110	+out_unlock:
	3111	+ mutex_unlock(&group->mutex);
	3112	+ iommu_group_put(group);
	3113	+
	3114	+ return handle;
	3115	+}
	3116	+EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
	3117	+
	3118	+/**
	3119	+ * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
	3120	+ * @handle: the handle returned by iommu_sva_bind_device()
	3121	+ *
	3122	+ * Put reference to a bond between device and address space. The device should
	3123	+ * not be issuing any more transaction for this PASID. All outstanding page
	3124	+ * requests for this PASID must have been flushed to the IOMMU.
	3125	+ *
	3126	+ * Returns 0 on success, or an error value
	3127	+ */
	3128	+void iommu_sva_unbind_device(struct iommu_sva *handle)
	3129	+{
	3130	+ struct iommu_group *group;
	3131	+ struct device *dev = handle->dev;
	3132	+ const struct iommu_ops *ops = dev->bus->iommu_ops;
	3133	+
	3134	+ if (!ops \|\| !ops->sva_unbind)
	3135	+ return;
	3136	+
	3137	+ group = iommu_group_get(dev);
	3138	+ if (!group)
	3139	+ return;
	3140	+
	3141	+ mutex_lock(&group->mutex);
	3142	+ ops->sva_unbind(handle);
	3143	+ mutex_unlock(&group->mutex);
	3144	+
	3145	+ iommu_group_put(group);
	3146	+}
	3147	+EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
	3148	+
	3149	+u32 iommu_sva_get_pasid(struct iommu_sva *handle)
	3150	+{
	3151	+ const struct iommu_ops *ops = handle->dev->bus->iommu_ops;
	3152	+
	3153	+ if (!ops \|\| !ops->sva_get_pasid)
	3154	+ return IOMMU_PASID_INVALID;
	3155	+
	3156	+ return ops->sva_get_pasid(handle);
	3157	+}
	3158	+EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);