forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 958e46acc8e900e8569dd467c1af9b8d2d019394 
 kernel/arch/powerpc/platforms/pseries/iommu.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
34  *
....@@ -7,21 +8,6 @@
78  * Copyright (C) 2006 Olof Johansson <olof@lixom.net>
89  *
910  * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
10
- *
11
- *
12
- * This program is free software; you can redistribute it and/or modify
13
- * it under the terms of the GNU General Public License as published by
14
- * the Free Software Foundation; either version 2 of the License, or
15
- * (at your option) any later version.
16
- *
17
- * This program is distributed in the hope that it will be useful,
18
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
19
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20
- * GNU General Public License for more details.
21
- *
22
- * You should have received a copy of the GNU General Public License
23
- * along with this program; if not, write to the Free Software
24
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
2511  */
2612 
2713 #include <linux/init.h>
....@@ -38,6 +24,7 @@
3824 #include <linux/of.h>
3925 #include <linux/iommu.h>
4026 #include <linux/rculist.h>
27
+#include <linux/local_lock.h>
4128 #include <asm/io.h>
4229 #include <asm/prom.h>
4330 #include <asm/rtas.h>
....@@ -53,11 +40,24 @@
5340 
5441 #include "pseries.h"
5542 
43
+enum {
44
+	DDW_QUERY_PE_DMA_WIN  = 0,
45
+	DDW_CREATE_PE_DMA_WIN = 1,
46
+	DDW_REMOVE_PE_DMA_WIN = 2,
47
+
48
+	DDW_APPLICABLE_SIZE
49
+};
50
+
51
+enum {
52
+	DDW_EXT_SIZE = 0,
53
+	DDW_EXT_RESET_DMA_WIN = 1,
54
+	DDW_EXT_QUERY_OUT_SIZE = 2
55
+};
56
+
5657 static struct iommu_table_group *iommu_pseries_alloc_group(int node)
5758 {
5859 	struct iommu_table_group *table_group;
5960 	struct iommu_table *tbl;
60
-	struct iommu_table_group_link *tgl;
6161 
6262 	table_group = kzalloc_node(sizeof(struct iommu_table_group), GFP_KERNEL,
6363 			   node);
....@@ -68,22 +68,13 @@
6868 	if (!tbl)
6969 		goto free_group;
7070 
71
-	tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL,
72
-			node);
73
-	if (!tgl)
74
-		goto free_table;
75
-
7671 	INIT_LIST_HEAD_RCU(&tbl->it_group_list);
7772 	kref_init(&tbl->it_kref);
78
-	tgl->table_group = table_group;
79
-	list_add_rcu(&tgl->next, &tbl->it_group_list);
8073 
8174 	table_group->tables[0] = tbl;
8275 
8376 	return table_group;
8477 
85
-free_table:
86
-	kfree(tbl);
8778 free_group:
8879 	kfree(table_group);
8980 	return NULL;
....@@ -93,23 +84,12 @@
9384 		const char *node_name)
9485 {
9586 	struct iommu_table *tbl;
96
-#ifdef CONFIG_IOMMU_API
97
-	struct iommu_table_group_link *tgl;
98
-#endif
9987 
10088 	if (!table_group)
10189 		return;
10290 
10391 	tbl = table_group->tables[0];
10492 #ifdef CONFIG_IOMMU_API
105
-	tgl = list_first_entry_or_null(&tbl->it_group_list,
106
-			struct iommu_table_group_link, next);
107
-
108
-	WARN_ON_ONCE(!tgl);
109
-	if (tgl) {
110
-		list_del_rcu(&tgl->next);
111
-		kfree(tgl);
112
-	}
11393 	if (table_group->group) {
11494 		iommu_group_put(table_group->group);
11595 		BUG_ON(table_group->group);
....@@ -126,7 +106,7 @@
126106 			      unsigned long attrs)
127107 {
128108 	u64 proto_tce;
129
-	__be64 *tcep, *tces;
109
+	__be64 *tcep;
130110 	u64 rpn;
131111 
132112 	proto_tce = TCE_PCI_READ; // Read allowed
....@@ -134,7 +114,7 @@
134114 	if (direction != DMA_TO_DEVICE)
135115 		proto_tce |= TCE_PCI_WRITE;
136116 
137
-	tces = tcep = ((__be64 *)tbl->it_base) + index;
117
+	tcep = ((__be64 *)tbl->it_base) + index;
138118 
139119 	while (npages--) {
140120 		/* can't move this out since we might cross MEMBLOCK boundary */
....@@ -150,9 +130,9 @@
150130 
151131 static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
152132 {
153
-	__be64 *tcep, *tces;
133
+	__be64 *tcep;
154134 
155
-	tces = tcep = ((__be64 *)tbl->it_base) + index;
135
+	tcep = ((__be64 *)tbl->it_base) + index;
156136 
157137 	while (npages--)
158138 		*(tcep++) = 0;
....@@ -211,7 +191,13 @@
211191 	return ret;
212192 }
213193 
214
-static DEFINE_PER_CPU(__be64 *, tce_page);
194
+struct tce_page {
195
+	__be64 * page;
196
+	local_lock_t lock;
197
+};
198
+static DEFINE_PER_CPU(struct tce_page, tce_page) = {
199
+	.lock = INIT_LOCAL_LOCK(lock),
200
+};
215201 
216202 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
217203 				     long npages, unsigned long uaddr,
....@@ -227,15 +213,16 @@
227213 	int ret = 0;
228214 	unsigned long flags;
229215 
230
-	if ((npages == 1) || !firmware_has_feature(FW_FEATURE_MULTITCE)) {
216
+	if ((npages == 1) || !firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
231217 		return tce_build_pSeriesLP(tbl->it_index, tcenum,
232218 					   tbl->it_page_shift, npages, uaddr,
233219 		                           direction, attrs);
234220 	}
235221 
236
-	local_irq_save(flags);	/* to protect tcep and the page behind it */
222
+	/* to protect tcep and the page behind it */
223
+	local_lock_irqsave(&tce_page.lock, flags);
237224 
238
-	tcep = __this_cpu_read(tce_page);
225
+	tcep = __this_cpu_read(tce_page.page);
239226 
240227 	/* This is safe to do since interrupts are off when we're called
241228 	 * from iommu_alloc{,_sg}()
....@@ -244,12 +231,12 @@
244231 		tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
245232 		/* If allocation fails, fall back to the loop implementation */
246233 		if (!tcep) {
247
-			local_irq_restore(flags);
234
+			local_unlock_irqrestore(&tce_page.lock, flags);
248235 			return tce_build_pSeriesLP(tbl->it_index, tcenum,
249236 					tbl->it_page_shift,
250237 					npages, uaddr, direction, attrs);
251238 		}
252
-		__this_cpu_write(tce_page, tcep);
239
+		__this_cpu_write(tce_page.page, tcep);
253240 	}
254241 
255242 	rpn = __pa(uaddr) >> TCE_SHIFT;
....@@ -279,7 +266,7 @@
279266 		tcenum += limit;
280267 	} while (npages > 0 && !rc);
281268 
282
-	local_irq_restore(flags);
269
+	local_unlock_irqrestore(&tce_page.lock, flags);
283270 
284271 	if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
285272 		ret = (int)rc;
....@@ -321,7 +308,7 @@
321308 {
322309 	u64 rc;
323310 
324
-	if (!firmware_has_feature(FW_FEATURE_MULTITCE))
311
+	if (!firmware_has_feature(FW_FEATURE_STUFF_TCE))
325312 		return tce_free_pSeriesLP(tbl->it_index, tcenum, npages);
326313 
327314 	rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);
....@@ -369,7 +356,7 @@
369356 /* Dynamic DMA Window support */
370357 struct ddw_query_response {
371358 	u32 windows_available;
372
-	u32 largest_available_block;
359
+	u64 largest_available_block;
373360 	u32 page_size;
374361 	u32 migration_capable;
375362 };
....@@ -437,7 +424,7 @@
437424 	u64 rc = 0;
438425 	long l, limit;
439426 
440
-	if (!firmware_has_feature(FW_FEATURE_MULTITCE)) {
427
+	if (!firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
441428 		unsigned long tceshift = be32_to_cpu(maprange->tce_shift);
442429 		unsigned long dmastart = (start_pfn << PAGE_SHIFT) +
443430 				be64_to_cpu(maprange->dma_base);
....@@ -450,16 +437,17 @@
450437 				DMA_BIDIRECTIONAL, 0);
451438 	}
452439 
453
-	local_irq_disable();	/* to protect tcep and the page behind it */
454
-	tcep = __this_cpu_read(tce_page);
440
+	/* to protect tcep and the page behind it */
441
+	local_lock_irq(&tce_page.lock);
442
+	tcep = __this_cpu_read(tce_page.page);
455443 
456444 	if (!tcep) {
457445 		tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
458446 		if (!tcep) {
459
-			local_irq_enable();
447
+			local_unlock_irq(&tce_page.lock);
460448 			return -ENOMEM;
461449 		}
462
-		__this_cpu_write(tce_page, tcep);
450
+		__this_cpu_write(tce_page.page, tcep);
463451 	}
464452 
465453 	proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
....@@ -502,7 +490,7 @@
502490 
503491 	/* error cleanup: caller will clear whole range */
504492 
505
-	local_irq_enable();
493
+	local_unlock_irq(&tce_page.lock);
506494 	return rc;
507495 }
508496 
....@@ -659,8 +647,7 @@
659647 
660648 	iommu_table_setparms(pci->phb, dn, tbl);
661649 	tbl->it_ops = &iommu_table_pseries_ops;
662
-	iommu_init_table(tbl, pci->phb->node);
663
-	iommu_register_group(pci->table_group, pci_domain_nr(bus), 0);
650
+	iommu_init_table(tbl, pci->phb->node, 0, 0);
664651 
665652 	/* Divide the rest (1.75GB) among the children */
666653 	pci->phb->dma_window_size = 0x80000000ul;
....@@ -672,7 +659,8 @@
672659 
673660 #ifdef CONFIG_IOMMU_API
674661 static int tce_exchange_pseries(struct iommu_table *tbl, long index, unsigned
675
-				long *tce, enum dma_data_direction *direction)
662
+				long *tce, enum dma_data_direction *direction,
663
+				bool realmode)
676664 {
677665 	long rc;
678666 	unsigned long ioba = (unsigned long) index << tbl->it_page_shift;
....@@ -700,7 +688,7 @@
700688 struct iommu_table_ops iommu_table_lpar_multi_ops = {
701689 	.set = tce_buildmulti_pSeriesLP,
702690 #ifdef CONFIG_IOMMU_API
703
-	.exchange = tce_exchange_pseries,
691
+	.xchg_no_kill = tce_exchange_pseries,
704692 #endif
705693 	.clear = tce_freemulti_pSeriesLP,
706694 	.get = tce_get_pSeriesLP
....@@ -741,7 +729,7 @@
741729 		iommu_table_setparms_lpar(ppci->phb, pdn, tbl,
742730 				ppci->table_group, dma_window);
743731 		tbl->it_ops = &iommu_table_lpar_multi_ops;
744
-		iommu_init_table(tbl, ppci->phb->node);
732
+		iommu_init_table(tbl, ppci->phb->node, 0, 0);
745733 		iommu_register_group(ppci->table_group,
746734 				pci_domain_nr(bus), 0);
747735 		pr_debug("  created table: %p\n", ppci->table_group);
....@@ -770,11 +758,8 @@
770758 		tbl = PCI_DN(dn)->table_group->tables[0];
771759 		iommu_table_setparms(phb, dn, tbl);
772760 		tbl->it_ops = &iommu_table_pseries_ops;
773
-		iommu_init_table(tbl, phb->node);
774
-		iommu_register_group(PCI_DN(dn)->table_group,
775
-				pci_domain_nr(phb->bus), 0);
761
+		iommu_init_table(tbl, phb->node, 0, 0);
776762 		set_iommu_table_base(&dev->dev, tbl);
777
-		iommu_add_device(&dev->dev);
778763 		return;
779764 	}
780765 
....@@ -785,11 +770,10 @@
785770 	while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL)
786771 		dn = dn->parent;
787772 
788
-	if (dn && PCI_DN(dn)) {
773
+	if (dn && PCI_DN(dn))
789774 		set_iommu_table_base(&dev->dev,
790775 				PCI_DN(dn)->table_group->tables[0]);
791
-		iommu_add_device(&dev->dev);
792
-	} else
776
+	else
793777 		printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
794778 		       pci_name(dev));
795779 }
....@@ -806,25 +790,14 @@
806790 
807791 early_param("disable_ddw", disable_ddw_setup);
808792 
809
-static void remove_ddw(struct device_node *np, bool remove_prop)
793
+static void remove_dma_window(struct device_node *np, u32 *ddw_avail,
794
+			      struct property *win)
810795 {
811796 	struct dynamic_dma_window_prop *dwp;
812
-	struct property *win64;
813
-	u32 ddw_avail[3];
814797 	u64 liobn;
815
-	int ret = 0;
798
+	int ret;
816799 
817
-	ret = of_property_read_u32_array(np, "ibm,ddw-applicable",
818
-					 &ddw_avail[0], 3);
819
-
820
-	win64 = of_find_property(np, DIRECT64_PROPNAME, NULL);
821
-	if (!win64)
822
-		return;
823
-
824
-	if (ret || win64->length < sizeof(*dwp))
825
-		goto delprop;
826
-
827
-	dwp = win64->value;
800
+	dwp = win->value;
828801 	liobn = (u64)be32_to_cpu(dwp->liobn);
829802 
830803 	/* clear the whole window, note the arg is in kernel pages */
....@@ -837,19 +810,39 @@
837810 		pr_debug("%pOF successfully cleared tces in window.\n",
838811 			 np);
839812 
840
-	ret = rtas_call(ddw_avail[2], 1, 1, NULL, liobn);
813
+	ret = rtas_call(ddw_avail[DDW_REMOVE_PE_DMA_WIN], 1, 1, NULL, liobn);
841814 	if (ret)
842815 		pr_warn("%pOF: failed to remove direct window: rtas returned "
843816 			"%d to ibm,remove-pe-dma-window(%x) %llx\n",
844
-			np, ret, ddw_avail[2], liobn);
817
+			np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
845818 	else
846819 		pr_debug("%pOF: successfully removed direct window: rtas returned "
847820 			"%d to ibm,remove-pe-dma-window(%x) %llx\n",
848
-			np, ret, ddw_avail[2], liobn);
821
+			np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
822
+}
849823 
850
-delprop:
851
-	if (remove_prop)
852
-		ret = of_remove_property(np, win64);
824
+static void remove_ddw(struct device_node *np, bool remove_prop)
825
+{
826
+	struct property *win;
827
+	u32 ddw_avail[DDW_APPLICABLE_SIZE];
828
+	int ret = 0;
829
+
830
+	ret = of_property_read_u32_array(np, "ibm,ddw-applicable",
831
+					 &ddw_avail[0], DDW_APPLICABLE_SIZE);
832
+	if (ret)
833
+		return;
834
+
835
+	win = of_find_property(np, DIRECT64_PROPNAME, NULL);
836
+	if (!win)
837
+		return;
838
+
839
+	if (win->length >= sizeof(struct dynamic_dma_window_prop))
840
+		remove_dma_window(np, ddw_avail, win);
841
+
842
+	if (!remove_prop)
843
+		return;
844
+
845
+	ret = of_remove_property(np, win);
853846 	if (ret)
854847 		pr_warn("%pOF: failed to remove direct window property: %d\n",
855848 			np, ret);
....@@ -908,14 +901,62 @@
908901 }
909902 machine_arch_initcall(pseries, find_existing_ddw_windows);
910903 
904
+/**
905
+ * ddw_read_ext - Get the value of an DDW extension
906
+ * @np:		device node from which the extension value is to be read.
907
+ * @extnum:	index number of the extension.
908
+ * @value:	pointer to return value, modified when extension is available.
909
+ *
910
+ * Checks if "ibm,ddw-extensions" exists for this node, and get the value
911
+ * on index 'extnum'.
912
+ * It can be used only to check if a property exists, passing value == NULL.
913
+ *
914
+ * Returns:
915
+ *	0 if extension successfully read
916
+ *	-EINVAL if the "ibm,ddw-extensions" does not exist,
917
+ *	-ENODATA if "ibm,ddw-extensions" does not have a value, and
918
+ *	-EOVERFLOW if "ibm,ddw-extensions" does not contain this extension.
919
+ */
920
+static inline int ddw_read_ext(const struct device_node *np, int extnum,
921
+			       u32 *value)
922
+{
923
+	static const char propname[] = "ibm,ddw-extensions";
924
+	u32 count;
925
+	int ret;
926
+
927
+	ret = of_property_read_u32_index(np, propname, DDW_EXT_SIZE, &count);
928
+	if (ret)
929
+		return ret;
930
+
931
+	if (count < extnum)
932
+		return -EOVERFLOW;
933
+
934
+	if (!value)
935
+		value = &count;
936
+
937
+	return of_property_read_u32_index(np, propname, extnum, value);
938
+}
939
+
911940 static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
912
-			struct ddw_query_response *query)
941
+		     struct ddw_query_response *query,
942
+		     struct device_node *parent)
913943 {
914944 	struct device_node *dn;
915945 	struct pci_dn *pdn;
916
-	u32 cfg_addr;
946
+	u32 cfg_addr, ext_query, query_out[5];
917947 	u64 buid;
918
-	int ret;
948
+	int ret, out_sz;
949
+
950
+	/*
951
+	 * From LoPAR level 2.8, "ibm,ddw-extensions" index 3 can rule how many
952
+	 * output parameters ibm,query-pe-dma-windows will have, ranging from
953
+	 * 5 to 6.
954
+	 */
955
+	ret = ddw_read_ext(parent, DDW_EXT_QUERY_OUT_SIZE, &ext_query);
956
+	if (!ret && ext_query == 1)
957
+		out_sz = 6;
958
+	else
959
+		out_sz = 5;
919960 
920961 	/*
921962 	 * Get the config address and phb buid of the PE window.
....@@ -928,11 +969,28 @@
928969 	buid = pdn->phb->buid;
929970 	cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
930971 
931
-	ret = rtas_call(ddw_avail[0], 3, 5, (u32 *)query,
932
-		  cfg_addr, BUID_HI(buid), BUID_LO(buid));
933
-	dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x"
934
-		" returned %d\n", ddw_avail[0], cfg_addr, BUID_HI(buid),
935
-		BUID_LO(buid), ret);
972
+	ret = rtas_call(ddw_avail[DDW_QUERY_PE_DMA_WIN], 3, out_sz, query_out,
973
+			cfg_addr, BUID_HI(buid), BUID_LO(buid));
974
+	dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x returned %d\n",
975
+		 ddw_avail[DDW_QUERY_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
976
+		 BUID_LO(buid), ret);
977
+
978
+	switch (out_sz) {
979
+	case 5:
980
+		query->windows_available = query_out[0];
981
+		query->largest_available_block = query_out[1];
982
+		query->page_size = query_out[2];
983
+		query->migration_capable = query_out[3];
984
+		break;
985
+	case 6:
986
+		query->windows_available = query_out[0];
987
+		query->largest_available_block = ((u64)query_out[1] << 32) |
988
+						 query_out[2];
989
+		query->page_size = query_out[3];
990
+		query->migration_capable = query_out[4];
991
+		break;
992
+	}
993
+
936994 	return ret;
937995 }
938996 
....@@ -959,15 +1017,16 @@
9591017 
9601018 	do {
9611019 		/* extra outputs are LIOBN and dma-addr (hi, lo) */
962
-		ret = rtas_call(ddw_avail[1], 5, 4, (u32 *)create,
963
-				cfg_addr, BUID_HI(buid), BUID_LO(buid),
964
-				page_shift, window_shift);
1020
+		ret = rtas_call(ddw_avail[DDW_CREATE_PE_DMA_WIN], 5, 4,
1021
+				(u32 *)create, cfg_addr, BUID_HI(buid),
1022
+				BUID_LO(buid), page_shift, window_shift);
9651023 	} while (rtas_busy_delay(ret));
9661024 	dev_info(&dev->dev,
9671025 		"ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
968
-		"(liobn = 0x%x starting addr = %x %x)\n", ddw_avail[1],
969
-		 cfg_addr, BUID_HI(buid), BUID_LO(buid), page_shift,
970
-		 window_shift, ret, create->liobn, create->addr_hi, create->addr_lo);
1026
+		"(liobn = 0x%x starting addr = %x %x)\n",
1027
+		 ddw_avail[DDW_CREATE_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1028
+		 BUID_LO(buid), page_shift, window_shift, ret, create->liobn,
1029
+		 create->addr_hi, create->addr_lo);
9711030 
9721031 	return ret;
9731032 }
....@@ -979,6 +1038,66 @@
9791038 
9801039 static LIST_HEAD(failed_ddw_pdn_list);
9811040 
1041
+static phys_addr_t ddw_memory_hotplug_max(void)
1042
+{
1043
+	phys_addr_t max_addr = memory_hotplug_max();
1044
+	struct device_node *memory;
1045
+
1046
+	for_each_node_by_type(memory, "memory") {
1047
+		unsigned long start, size;
1048
+		int n_mem_addr_cells, n_mem_size_cells, len;
1049
+		const __be32 *memcell_buf;
1050
+
1051
+		memcell_buf = of_get_property(memory, "reg", &len);
1052
+		if (!memcell_buf || len <= 0)
1053
+			continue;
1054
+
1055
+		n_mem_addr_cells = of_n_addr_cells(memory);
1056
+		n_mem_size_cells = of_n_size_cells(memory);
1057
+
1058
+		start = of_read_number(memcell_buf, n_mem_addr_cells);
1059
+		memcell_buf += n_mem_addr_cells;
1060
+		size = of_read_number(memcell_buf, n_mem_size_cells);
1061
+		memcell_buf += n_mem_size_cells;
1062
+
1063
+		max_addr = max_t(phys_addr_t, max_addr, start + size);
1064
+	}
1065
+
1066
+	return max_addr;
1067
+}
1068
+
1069
+/*
1070
+ * Platforms supporting the DDW option starting with LoPAR level 2.7 implement
1071
+ * ibm,ddw-extensions, which carries the rtas token for
1072
+ * ibm,reset-pe-dma-windows.
1073
+ * That rtas-call can be used to restore the default DMA window for the device.
1074
+ */
1075
+static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn)
1076
+{
1077
+	int ret;
1078
+	u32 cfg_addr, reset_dma_win;
1079
+	u64 buid;
1080
+	struct device_node *dn;
1081
+	struct pci_dn *pdn;
1082
+
1083
+	ret = ddw_read_ext(par_dn, DDW_EXT_RESET_DMA_WIN, &reset_dma_win);
1084
+	if (ret)
1085
+		return;
1086
+
1087
+	dn = pci_device_to_OF_node(dev);
1088
+	pdn = PCI_DN(dn);
1089
+	buid = pdn->phb->buid;
1090
+	cfg_addr = (pdn->busno << 16) | (pdn->devfn << 8);
1091
+
1092
+	ret = rtas_call(reset_dma_win, 3, 1, NULL, cfg_addr, BUID_HI(buid),
1093
+			BUID_LO(buid));
1094
+	if (ret)
1095
+		dev_info(&dev->dev,
1096
+			 "ibm,reset-pe-dma-windows(%x) %x %x %x returned %d ",
1097
+			 reset_dma_win, cfg_addr, BUID_HI(buid), BUID_LO(buid),
1098
+			 ret);
1099
+}
1100
+
9821101 /*
9831102  * If the PE supports dynamic dma windows, and there is space for a table
9841103  * that can map all pages in a linear offset, then setup such a table,
....@@ -988,7 +1107,7 @@
9881107  * pdn: the parent pe node with the ibm,dma_window property
9891108  * Future: also check if we can remap the base window for our base page size
9901109  *
991
- * returns the dma offset for use by dma_set_mask
1110
+ * returns the dma offset for use by the direct mapped DMA code.
9921111  */
9931112 static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn)
9941113 {
....@@ -998,11 +1117,12 @@
9981117 	int page_shift;
9991118 	u64 dma_addr, max_addr;
10001119 	struct device_node *dn;
1001
-	u32 ddw_avail[3];
1120
+	u32 ddw_avail[DDW_APPLICABLE_SIZE];
10021121 	struct direct_window *window;
10031122 	struct property *win64;
10041123 	struct dynamic_dma_window_prop *ddwprop;
10051124 	struct failed_ddw_pdn *fpdn;
1125
+	bool default_win_removed = false;
10061126 
10071127 	mutex_lock(&direct_window_init_mutex);
10081128 
....@@ -1031,7 +1151,7 @@
10311151 	 * the property is actually in the parent, not the PE
10321152 	 */
10331153 	ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable",
1034
-					 &ddw_avail[0], 3);
1154
+					 &ddw_avail[0], DDW_APPLICABLE_SIZE);
10351155 	if (ret)
10361156 		goto out_failed;
10371157 
....@@ -1042,18 +1162,42 @@
10421162 	 * of page sizes: supported and supported for migrate-dma.
10431163 	 */
10441164 	dn = pci_device_to_OF_node(dev);
1045
-	ret = query_ddw(dev, ddw_avail, &query);
1165
+	ret = query_ddw(dev, ddw_avail, &query, pdn);
10461166 	if (ret != 0)
10471167 		goto out_failed;
10481168 
1169
+	/*
1170
+	 * If there is no window available, remove the default DMA window,
1171
+	 * if it's present. This will make all the resources available to the
1172
+	 * new DDW window.
1173
+	 * If anything fails after this, we need to restore it, so also check
1174
+	 * for extensions presence.
1175
+	 */
10491176 	if (query.windows_available == 0) {
1050
-		/*
1051
-		 * no additional windows are available for this device.
1052
-		 * We might be able to reallocate the existing window,
1053
-		 * trading in for a larger page size.
1054
-		 */
1055
-		dev_dbg(&dev->dev, "no free dynamic windows");
1056
-		goto out_failed;
1177
+		struct property *default_win;
1178
+		int reset_win_ext;
1179
+
1180
+		default_win = of_find_property(pdn, "ibm,dma-window", NULL);
1181
+		if (!default_win)
1182
+			goto out_failed;
1183
+
1184
+		reset_win_ext = ddw_read_ext(pdn, DDW_EXT_RESET_DMA_WIN, NULL);
1185
+		if (reset_win_ext)
1186
+			goto out_failed;
1187
+
1188
+		remove_dma_window(pdn, ddw_avail, default_win);
1189
+		default_win_removed = true;
1190
+
1191
+		/* Query again, to check if the window is available */
1192
+		ret = query_ddw(dev, ddw_avail, &query, pdn);
1193
+		if (ret != 0)
1194
+			goto out_failed;
1195
+
1196
+		if (query.windows_available == 0) {
1197
+			/* no windows are available for this device. */
1198
+			dev_dbg(&dev->dev, "no free dynamic windows");
1199
+			goto out_failed;
1200
+		}
10571201 	}
10581202 	if (query.page_size & 4) {
10591203 		page_shift = 24; /* 16MB */
....@@ -1068,9 +1212,9 @@
10681212 	}
10691213 	/* verify the window * number of ptes will map the partition */
10701214 	/* check largest block * page size > max memory hotplug addr */
1071
-	max_addr = memory_hotplug_max();
1215
+	max_addr = ddw_memory_hotplug_max();
10721216 	if (query.largest_available_block < (max_addr >> page_shift)) {
1073
-		dev_dbg(&dev->dev, "can't map partition max 0x%llx with %u "
1217
+		dev_dbg(&dev->dev, "can't map partition max 0x%llx with %llu "
10741218 			  "%llu-sized pages\n", max_addr,  query.largest_available_block,
10751219 			  1ULL << page_shift);
10761220 		goto out_failed;
....@@ -1144,6 +1288,8 @@
11441288 	kfree(win64);
11451289 
11461290 out_failed:
1291
+	if (default_win_removed)
1292
+		reset_dma_window(dev, pdn);
11471293 
11481294 	fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL);
11491295 	if (!fpdn)
....@@ -1196,7 +1342,7 @@
11961342 		iommu_table_setparms_lpar(pci->phb, pdn, tbl,
11971343 				pci->table_group, dma_window);
11981344 		tbl->it_ops = &iommu_table_lpar_multi_ops;
1199
-		iommu_init_table(tbl, pci->phb->node);
1345
+		iommu_init_table(tbl, pci->phb->node, 0, 0);
12001346 		iommu_register_group(pci->table_group,
12011347 				pci_domain_nr(pci->phb->bus), 0);
12021348 		pr_debug("  created table: %p\n", pci->table_group);
....@@ -1205,90 +1351,40 @@
12051351 	}
12061352 
12071353 	set_iommu_table_base(&dev->dev, pci->table_group->tables[0]);
1208
-	iommu_add_device(&dev->dev);
1354
+	iommu_add_device(pci->table_group, &dev->dev);
12091355 }
12101356 
1211
-static int dma_set_mask_pSeriesLP(struct device *dev, u64 dma_mask)
1357
+static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask)
12121358 {
1213
-	bool ddw_enabled = false;
1214
-	struct device_node *pdn, *dn;
1215
-	struct pci_dev *pdev;
1359
+	struct device_node *dn = pci_device_to_OF_node(pdev), *pdn;
12161360 	const __be32 *dma_window = NULL;
1217
-	u64 dma_offset;
1218
-
1219
-	if (!dev->dma_mask)
1220
-		return -EIO;
1221
-
1222
-	if (!dev_is_pci(dev))
1223
-		goto check_mask;
1224
-
1225
-	pdev = to_pci_dev(dev);
12261361 
12271362 	/* only attempt to use a new window if 64-bit DMA is requested */
1228
-	if (!disable_ddw && dma_mask == DMA_BIT_MASK(64)) {
1229
-		dn = pci_device_to_OF_node(pdev);
1230
-		dev_dbg(dev, "node is %pOF\n", dn);
1363
+	if (dma_mask < DMA_BIT_MASK(64))
1364
+		return false;
12311365 
1232
-		/*
1233
-		 * the device tree might contain the dma-window properties
1234
-		 * per-device and not necessarily for the bus. So we need to
1235
-		 * search upwards in the tree until we either hit a dma-window
1236
-		 * property, OR find a parent with a table already allocated.
1237
-		 */
1238
-		for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->table_group;
1239
-				pdn = pdn->parent) {
1240
-			dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1241
-			if (dma_window)
1242
-				break;
1243
-		}
1244
-		if (pdn && PCI_DN(pdn)) {
1245
-			dma_offset = enable_ddw(pdev, pdn);
1246
-			if (dma_offset != 0) {
1247
-				dev_info(dev, "Using 64-bit direct DMA at offset %llx\n", dma_offset);
1248
-				set_dma_offset(dev, dma_offset);
1249
-				set_dma_ops(dev, &dma_nommu_ops);
1250
-				ddw_enabled = true;
1251
-			}
1252
-		}
1366
+	dev_dbg(&pdev->dev, "node is %pOF\n", dn);
1367
+
1368
+	/*
1369
+	 * the device tree might contain the dma-window properties
1370
+	 * per-device and not necessarily for the bus. So we need to
1371
+	 * search upwards in the tree until we either hit a dma-window
1372
+	 * property, OR find a parent with a table already allocated.
1373
+	 */
1374
+	for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->table_group;
1375
+			pdn = pdn->parent) {
1376
+		dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1377
+		if (dma_window)
1378
+			break;
12531379 	}
12541380 
1255
-	/* fall back on iommu ops */
1256
-	if (!ddw_enabled && get_dma_ops(dev) != &dma_iommu_ops) {
1257
-		dev_info(dev, "Restoring 32-bit DMA via iommu\n");
1258
-		set_dma_ops(dev, &dma_iommu_ops);
1381
+	if (pdn && PCI_DN(pdn)) {
1382
+		pdev->dev.archdata.dma_offset = enable_ddw(pdev, pdn);
1383
+		if (pdev->dev.archdata.dma_offset)
1384
+			return true;
12591385 	}
12601386 
1261
-check_mask:
1262
-	if (!dma_supported(dev, dma_mask))
1263
-		return -EIO;
1264
-
1265
-	*dev->dma_mask = dma_mask;
1266
-	return 0;
1267
-}
1268
-
1269
-static u64 dma_get_required_mask_pSeriesLP(struct device *dev)
1270
-{
1271
-	if (!dev->dma_mask)
1272
-		return 0;
1273
-
1274
-	if (!disable_ddw && dev_is_pci(dev)) {
1275
-		struct pci_dev *pdev = to_pci_dev(dev);
1276
-		struct device_node *dn;
1277
-
1278
-		dn = pci_device_to_OF_node(pdev);
1279
-
1280
-		/* search upwards for ibm,dma-window */
1281
-		for (; dn && PCI_DN(dn) && !PCI_DN(dn)->table_group;
1282
-				dn = dn->parent)
1283
-			if (of_get_property(dn, "ibm,dma-window", NULL))
1284
-				break;
1285
-		/* if there is a ibm,ddw-applicable property require 64 bits */
1286
-		if (dn && PCI_DN(dn) &&
1287
-				of_get_property(dn, "ibm,ddw-applicable", NULL))
1288
-			return DMA_BIT_MASK(64);
1289
-	}
1290
-
1291
-	return dma_iommu_ops.get_required_mask(dev);
1387
+	return false;
12921388 }
12931389 
12941390 static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
....@@ -1383,8 +1479,9 @@
13831479 	if (firmware_has_feature(FW_FEATURE_LPAR)) {
13841480 		pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
13851481 		pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1386
-		ppc_md.dma_set_mask = dma_set_mask_pSeriesLP;
1387
-		ppc_md.dma_get_required_mask = dma_get_required_mask_pSeriesLP;
1482
+		if (!disable_ddw)
1483
+			pseries_pci_controller_ops.iommu_bypass_supported =
1484
+				iommu_bypass_supported_pSeriesLP;
13881485 	} else {
13891486 		pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries;
13901487 		pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries;
....@@ -1401,13 +1498,38 @@
14011498 {
14021499 	if (strcmp(str, "off") == 0 &&
14031500 	    firmware_has_feature(FW_FEATURE_LPAR) &&
1404
-	    firmware_has_feature(FW_FEATURE_MULTITCE)) {
1501
+	    (firmware_has_feature(FW_FEATURE_PUT_TCE_IND) ||
1502
+	     firmware_has_feature(FW_FEATURE_STUFF_TCE))) {
14051503 		printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
1406
-		powerpc_firmware_features &= ~FW_FEATURE_MULTITCE;
1504
+		powerpc_firmware_features &=
1505
+			~(FW_FEATURE_PUT_TCE_IND | FW_FEATURE_STUFF_TCE);
14071506 	}
14081507 	return 1;
14091508 }
14101509 
14111510 __setup("multitce=", disable_multitce);
14121511 
1512
+static int tce_iommu_bus_notifier(struct notifier_block *nb,
1513
+		unsigned long action, void *data)
1514
+{
1515
+	struct device *dev = data;
1516
+
1517
+	switch (action) {
1518
+	case BUS_NOTIFY_DEL_DEVICE:
1519
+		iommu_del_device(dev);
1520
+		return 0;
1521
+	default:
1522
+		return 0;
1523
+	}
1524
+}
1525
+
1526
+static struct notifier_block tce_iommu_bus_nb = {
1527
+	.notifier_call = tce_iommu_bus_notifier,
1528
+};
1529
+
1530
+static int __init tce_iommu_bus_notifier_init(void)
1531
+{
1532
+	bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
1533
+	return 0;
1534
+}
14131535 machine_subsys_initcall_sync(pseries, tce_iommu_bus_notifier_init);