blame | history | patch | commit | commitdiff
hc
2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf 
 kernel/drivers/gpu/drm/msm/msm_iommu.c
....@@ -1,28 +1,216 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (C) 2013 Red Hat
34  * Author: Rob Clark <robdclark@gmail.com>
4
- *
5
- * This program is free software; you can redistribute it and/or modify it
6
- * under the terms of the GNU General Public License version 2 as published by
7
- * the Free Software Foundation.
8
- *
9
- * This program is distributed in the hope that it will be useful, but WITHOUT
10
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12
- * more details.
13
- *
14
- * You should have received a copy of the GNU General Public License along with
15
- * this program.  If not, see <http://www.gnu.org/licenses/>.
165  */
176 
7
+#include <linux/adreno-smmu-priv.h>
8
+#include <linux/io-pgtable.h>
189 #include "msm_drv.h"
1910 #include "msm_mmu.h"
2011 
2112 struct msm_iommu {
2213 	struct msm_mmu base;
2314 	struct iommu_domain *domain;
15
+	atomic_t pagetables;
2416 };
17
+
2518 #define to_msm_iommu(x) container_of(x, struct msm_iommu, base)
19
+
20
+struct msm_iommu_pagetable {
21
+	struct msm_mmu base;
22
+	struct msm_mmu *parent;
23
+	struct io_pgtable_ops *pgtbl_ops;
24
+	phys_addr_t ttbr;
25
+	u32 asid;
26
+};
27
+static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu)
28
+{
29
+	return container_of(mmu, struct msm_iommu_pagetable, base);
30
+}
31
+
32
+static int msm_iommu_pagetable_unmap(struct msm_mmu *mmu, u64 iova,
33
+		size_t size)
34
+{
35
+	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
36
+	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
37
+	size_t unmapped = 0;
38
+
39
+	/* Unmap the block one page at a time */
40
+	while (size) {
41
+		unmapped += ops->unmap(ops, iova, 4096, NULL);
42
+		iova += 4096;
43
+		size -= 4096;
44
+	}
45
+
46
+	iommu_flush_iotlb_all(to_msm_iommu(pagetable->parent)->domain);
47
+
48
+	return (unmapped == size) ? 0 : -EINVAL;
49
+}
50
+
51
+static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova,
52
+		struct sg_table *sgt, size_t len, int prot)
53
+{
54
+	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
55
+	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
56
+	struct scatterlist *sg;
57
+	size_t mapped = 0;
58
+	u64 addr = iova;
59
+	unsigned int i;
60
+
61
+	for_each_sgtable_sg(sgt, sg, i) {
62
+		size_t size = sg->length;
63
+		phys_addr_t phys = sg_phys(sg);
64
+
65
+		/* Map the block one page at a time */
66
+		while (size) {
67
+			if (ops->map(ops, addr, phys, 4096, prot, GFP_KERNEL)) {
68
+				msm_iommu_pagetable_unmap(mmu, iova, mapped);
69
+				return -EINVAL;
70
+			}
71
+
72
+			phys += 4096;
73
+			addr += 4096;
74
+			size -= 4096;
75
+			mapped += 4096;
76
+		}
77
+	}
78
+
79
+	return 0;
80
+}
81
+
82
+static void msm_iommu_pagetable_destroy(struct msm_mmu *mmu)
83
+{
84
+	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
85
+	struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);
86
+	struct adreno_smmu_priv *adreno_smmu =
87
+		dev_get_drvdata(pagetable->parent->dev);
88
+
89
+	/*
90
+	 * If this is the last attached pagetable for the parent,
91
+	 * disable TTBR0 in the arm-smmu driver
92
+	 */
93
+	if (atomic_dec_return(&iommu->pagetables) == 0)
94
+		adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, NULL);
95
+
96
+	free_io_pgtable_ops(pagetable->pgtbl_ops);
97
+	kfree(pagetable);
98
+}
99
+
100
+int msm_iommu_pagetable_params(struct msm_mmu *mmu,
101
+		phys_addr_t *ttbr, int *asid)
102
+{
103
+	struct msm_iommu_pagetable *pagetable;
104
+
105
+	if (mmu->type != MSM_MMU_IOMMU_PAGETABLE)
106
+		return -EINVAL;
107
+
108
+	pagetable = to_pagetable(mmu);
109
+
110
+	if (ttbr)
111
+		*ttbr = pagetable->ttbr;
112
+
113
+	if (asid)
114
+		*asid = pagetable->asid;
115
+
116
+	return 0;
117
+}
118
+
119
+static const struct msm_mmu_funcs pagetable_funcs = {
120
+		.map = msm_iommu_pagetable_map,
121
+		.unmap = msm_iommu_pagetable_unmap,
122
+		.destroy = msm_iommu_pagetable_destroy,
123
+};
124
+
125
+static void msm_iommu_tlb_flush_all(void *cookie)
126
+{
127
+}
128
+
129
+static void msm_iommu_tlb_flush_walk(unsigned long iova, size_t size,
130
+		size_t granule, void *cookie)
131
+{
132
+}
133
+
134
+static void msm_iommu_tlb_add_page(struct iommu_iotlb_gather *gather,
135
+		unsigned long iova, size_t granule, void *cookie)
136
+{
137
+}
138
+
139
+static const struct iommu_flush_ops null_tlb_ops = {
140
+	.tlb_flush_all = msm_iommu_tlb_flush_all,
141
+	.tlb_flush_walk = msm_iommu_tlb_flush_walk,
142
+	.tlb_add_page = msm_iommu_tlb_add_page,
143
+};
144
+
145
+struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent)
146
+{
147
+	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(parent->dev);
148
+	struct msm_iommu *iommu = to_msm_iommu(parent);
149
+	struct msm_iommu_pagetable *pagetable;
150
+	const struct io_pgtable_cfg *ttbr1_cfg = NULL;
151
+	struct io_pgtable_cfg ttbr0_cfg;
152
+	int ret;
153
+
154
+	/* Get the pagetable configuration from the domain */
155
+	if (adreno_smmu->cookie)
156
+		ttbr1_cfg = adreno_smmu->get_ttbr1_cfg(adreno_smmu->cookie);
157
+
158
+	/*
159
+	 * If you hit this WARN_ONCE() you are probably missing an entry in
160
+	 * qcom_smmu_impl_of_match[] in arm-smmu-qcom.c
161
+	 */
162
+	if (WARN_ONCE(!ttbr1_cfg, "No per-process page tables"))
163
+		return ERR_PTR(-ENODEV);
164
+
165
+	pagetable = kzalloc(sizeof(*pagetable), GFP_KERNEL);
166
+	if (!pagetable)
167
+		return ERR_PTR(-ENOMEM);
168
+
169
+	msm_mmu_init(&pagetable->base, parent->dev, &pagetable_funcs,
170
+		MSM_MMU_IOMMU_PAGETABLE);
171
+
172
+	/* Clone the TTBR1 cfg as starting point for TTBR0 cfg: */
173
+	ttbr0_cfg = *ttbr1_cfg;
174
+
175
+	/* The incoming cfg will have the TTBR1 quirk enabled */
176
+	ttbr0_cfg.quirks &= ~IO_PGTABLE_QUIRK_ARM_TTBR1;
177
+	ttbr0_cfg.tlb = &null_tlb_ops;
178
+
179
+	pagetable->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1,
180
+		&ttbr0_cfg, iommu->domain);
181
+
182
+	if (!pagetable->pgtbl_ops) {
183
+		kfree(pagetable);
184
+		return ERR_PTR(-ENOMEM);
185
+	}
186
+
187
+	/*
188
+	 * If this is the first pagetable that we've allocated, send it back to
189
+	 * the arm-smmu driver as a trigger to set up TTBR0
190
+	 */
191
+	if (atomic_inc_return(&iommu->pagetables) == 1) {
192
+		ret = adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, &ttbr0_cfg);
193
+		if (ret) {
194
+			free_io_pgtable_ops(pagetable->pgtbl_ops);
195
+			kfree(pagetable);
196
+			return ERR_PTR(ret);
197
+		}
198
+	}
199
+
200
+	/* Needed later for TLB flush */
201
+	pagetable->parent = parent;
202
+	pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr;
203
+
204
+	/*
205
+	 * TODO we would like each set of page tables to have a unique ASID
206
+	 * to optimize TLB invalidation.  But iommu_flush_iotlb_all() will
207
+	 * end up flushing the ASID used for TTBR1 pagetables, which is not
208
+	 * what we want.  So for now just use the same ASID as TTBR1.
209
+	 */
210
+	pagetable->asid = 0;
211
+
212
+	return &pagetable->base;
213
+}
26214 
27215 static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
28216 		unsigned long iova, int flags, void *arg)
....@@ -30,55 +218,41 @@
30218 	struct msm_iommu *iommu = arg;
31219 	if (iommu->base.handler)
32220 		return iommu->base.handler(iommu->base.arg, iova, flags);
33
-	pr_warn_ratelimited("*** fault: iova=%08lx, flags=%d\n", iova, flags);
221
+	pr_warn_ratelimited("*** fault: iova=%16lx, flags=%d\n", iova, flags);
34222 	return 0;
35223 }
36224 
37
-static int msm_iommu_attach(struct msm_mmu *mmu, const char * const *names,
38
-			    int cnt)
39
-{
40
-	struct msm_iommu *iommu = to_msm_iommu(mmu);
41
-	int ret;
42
-
43
-	pm_runtime_get_sync(mmu->dev);
44
-	ret = iommu_attach_device(iommu->domain, mmu->dev);
45
-	pm_runtime_put_sync(mmu->dev);
46
-
47
-	return ret;
48
-}
49
-
50
-static void msm_iommu_detach(struct msm_mmu *mmu, const char * const *names,
51
-			     int cnt)
225
+static void msm_iommu_detach(struct msm_mmu *mmu)
52226 {
53227 	struct msm_iommu *iommu = to_msm_iommu(mmu);
54228 
55
-	pm_runtime_get_sync(mmu->dev);
56229 	iommu_detach_device(iommu->domain, mmu->dev);
57
-	pm_runtime_put_sync(mmu->dev);
58230 }
59231 
60232 static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
61
-		struct sg_table *sgt, unsigned len, int prot)
233
+		struct sg_table *sgt, size_t len, int prot)
62234 {
63235 	struct msm_iommu *iommu = to_msm_iommu(mmu);
64236 	size_t ret;
65237 
66
-//	pm_runtime_get_sync(mmu->dev);
67
-	ret = iommu_map_sg(iommu->domain, iova, sgt->sgl, sgt->nents, prot);
68
-//	pm_runtime_put_sync(mmu->dev);
238
+	/* The arm-smmu driver expects the addresses to be sign extended */
239
+	if (iova & BIT_ULL(48))
240
+		iova |= GENMASK_ULL(63, 49);
241
+
242
+	ret = iommu_map_sgtable(iommu->domain, iova, sgt, prot);
69243 	WARN_ON(!ret);
70244 
71245 	return (ret == len) ? 0 : -EINVAL;
72246 }
73247 
74
-static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova,
75
-		struct sg_table *sgt, unsigned len)
248
+static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
76249 {
77250 	struct msm_iommu *iommu = to_msm_iommu(mmu);
78251 
79
-	pm_runtime_get_sync(mmu->dev);
252
+	if (iova & BIT_ULL(48))
253
+		iova |= GENMASK_ULL(63, 49);
254
+
80255 	iommu_unmap(iommu->domain, iova, len);
81
-	pm_runtime_put_sync(mmu->dev);
82256 
83257 	return 0;
84258 }
....@@ -91,7 +265,6 @@
91265 }
92266 
93267 static const struct msm_mmu_funcs funcs = {
94
-		.attach = msm_iommu_attach,
95268 		.detach = msm_iommu_detach,
96269 		.map = msm_iommu_map,
97270 		.unmap = msm_iommu_unmap,
....@@ -101,14 +274,26 @@
101274 struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain)
102275 {
103276 	struct msm_iommu *iommu;
277
+	int ret;
278
+
279
+	if (!domain)
280
+		return ERR_PTR(-ENODEV);
104281 
105282 	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
106283 	if (!iommu)
107284 		return ERR_PTR(-ENOMEM);
108285 
109286 	iommu->domain = domain;
110
-	msm_mmu_init(&iommu->base, dev, &funcs);
287
+	msm_mmu_init(&iommu->base, dev, &funcs, MSM_MMU_IOMMU);
111288 	iommu_set_fault_handler(domain, msm_fault_handler, iommu);
112289 
290
+	atomic_set(&iommu->pagetables, 0);
291
+
292
+	ret = iommu_attach_device(iommu->domain, dev);
293
+	if (ret) {
294
+		kfree(iommu);
295
+		return ERR_PTR(ret);
296
+	}
297
+
113298 	return &iommu->base;
114299 }