add new system file

hc

2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c

 kernel/arch/s390/pci/pci_mmio.c
..
..
@@ -11,6 +11,113 @@
11
11
 #include <linux/mm.h>
12
12
 #include <linux/errno.h>
13
13
 #include <linux/pci.h>
14
+#include <asm/pci_io.h>
15
+#include <asm/pci_debug.h>
16
+
17
+static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
18
+{
19
+	struct {
20
+		u64 offset;
21
+		u8 cc;
22
+		u8 status;
23
+	} data = {offset, cc, status};
24
+
25
+	zpci_err_hex(&data, sizeof(data));
26
+}
27
+
28
+static inline int __pcistb_mio_inuser(
29
+		void __iomem *ioaddr, const void __user *src,
30
+		u64 len, u8 *status)
31
+{
32
+	int cc = -ENXIO;
33
+
34
+	asm volatile (
35
+		"       sacf 256\n"
36
+		"0:     .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
37
+		"1:     ipm     %[cc]\n"
38
+		"       srl     %[cc],28\n"
39
+		"2:     sacf 768\n"
40
+		EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
41
+		: [cc] "+d" (cc), [len] "+d" (len)
42
+		: [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
43
+		: "cc", "memory");
44
+	*status = len >> 24 & 0xff;
45
+	return cc;
46
+}
47
+
48
+static inline int __pcistg_mio_inuser(
49
+		void __iomem *ioaddr, const void __user *src,
50
+		u64 ulen, u8 *status)
51
+{
52
+	register u64 addr asm("2") = (u64 __force) ioaddr;
53
+	register u64 len asm("3") = ulen;
54
+	int cc = -ENXIO;
55
+	u64 val = 0;
56
+	u64 cnt = ulen;
57
+	u8 tmp;
58
+
59
+	/*
60
+	 * copy 0 < @len <= 8 bytes from @src into the right most bytes of
61
+	 * a register, then store it to PCI at @ioaddr while in secondary
62
+	 * address space. pcistg then uses the user mappings.
63
+	 */
64
+	asm volatile (
65
+		"       sacf    256\n"
66
+		"0:     llgc    %[tmp],0(%[src])\n"
67
+		"4:	sllg	%[val],%[val],8\n"
68
+		"       aghi    %[src],1\n"
69
+		"       ogr     %[val],%[tmp]\n"
70
+		"       brctg   %[cnt],0b\n"
71
+		"1:     .insn   rre,0xb9d40000,%[val],%[ioaddr]\n"
72
+		"2:     ipm     %[cc]\n"
73
+		"       srl     %[cc],28\n"
74
+		"3:     sacf    768\n"
75
+		EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
76
+		:
77
+		[src] "+a" (src), [cnt] "+d" (cnt),
78
+		[val] "+d" (val), [tmp] "=d" (tmp),
79
+		[len] "+d" (len), [cc] "+d" (cc),
80
+		[ioaddr] "+a" (addr)
81
+		:: "cc", "memory");
82
+	*status = len >> 24 & 0xff;
83
+
84
+	/* did we read everything from user memory? */
85
+	if (!cc && cnt != 0)
86
+		cc = -EFAULT;
87
+
88
+	return cc;
89
+}
90
+
91
+static inline int __memcpy_toio_inuser(void __iomem *dst,
92
+				   const void __user *src, size_t n)
93
+{
94
+	int size, rc = 0;
95
+	u8 status = 0;
96
+	mm_segment_t old_fs;
97
+
98
+	if (!src)
99
+		return -EINVAL;
100
+
101
+	old_fs = enable_sacf_uaccess();
102
+	while (n > 0) {
103
+		size = zpci_get_max_write_size((u64 __force) dst,
104
+					       (u64 __force) src, n,
105
+					       ZPCI_MAX_WRITE_SIZE);
106
+		if (size > 8) /* main path */
107
+			rc = __pcistb_mio_inuser(dst, src, size, &status);
108
+		else
109
+			rc = __pcistg_mio_inuser(dst, src, size, &status);
110
+		if (rc)
111
+			break;
112
+		src += size;
113
+		dst += size;
114
+		n -= size;
115
+	}
116
+	disable_sacf_uaccess(old_fs);
117
+	if (rc)
118
+		zpci_err_mmio(rc, status, (__force u64) dst);
119
+	return rc;
120
+}
14
121
 
15
122
 static long get_pfn(unsigned long user_addr, unsigned long access,
16
123
 		    unsigned long *pfn)
..
..
@@ -18,17 +125,17 @@
18
125
 	struct vm_area_struct *vma;
19
126
 	long ret;
20
127
 
21
-	down_read(&current->mm->mmap_sem);
128
+	mmap_read_lock(current->mm);
22
129
 	ret = -EINVAL;
23
130
 	vma = find_vma(current->mm, user_addr);
24
-	if (!vma)
131
+	if (!vma || user_addr < vma->vm_start)
25
132
 		goto out;
26
133
 	ret = -EACCES;
27
134
 	if (!(vma->vm_flags & access))
28
135
 		goto out;
29
136
 	ret = follow_pfn(vma, user_addr, pfn);
30
137
 out:
31
-	up_read(&current->mm->mmap_sem);
138
+	mmap_read_unlock(current->mm);
32
139
 	return ret;
33
140
 }
34
141
 
..
..
@@ -46,6 +153,22 @@
46
153
 
47
154
 	if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
48
155
 		return -EINVAL;
156
+
157
+	/*
158
+	 * We only support write access to MIO capable devices if we are on
159
+	 * a MIO enabled system. Otherwise we would have to check for every
160
+	 * address if it is a special ZPCI_ADDR and would have to do
161
+	 * a get_pfn() which we don't need for MIO capable devices.  Currently
162
+	 * ISM devices are the only devices without MIO support and there is no
163
+	 * known need for accessing these from userspace.
164
+	 */
165
+	if (static_branch_likely(&have_mio)) {
166
+		ret = __memcpy_toio_inuser((void  __iomem *) mmio_addr,
167
+					user_buffer,
168
+					length);
169
+		return ret;
170
+	}
171
+
49
172
 	if (length > 64) {
50
173
 		buf = kmalloc(length, GFP_KERNEL);
51
174
 		if (!buf)
..
..
@@ -56,7 +179,8 @@
56
179
 	ret = get_pfn(mmio_addr, VM_WRITE, &pfn);
57
180
 	if (ret)
58
181
 		goto out;
59
-	io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK));
182
+	io_addr = (void __iomem *)((pfn << PAGE_SHIFT) |
183
+			(mmio_addr & ~PAGE_MASK));
60
184
 
61
185
 	ret = -EFAULT;
62
186
 	if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
..
..
@@ -70,6 +194,78 @@
70
194
 	if (buf != local_buf)
71
195
 		kfree(buf);
72
196
 	return ret;
197
+}
198
+
199
+static inline int __pcilg_mio_inuser(
200
+		void __user *dst, const void __iomem *ioaddr,
201
+		u64 ulen, u8 *status)
202
+{
203
+	register u64 addr asm("2") = (u64 __force) ioaddr;
204
+	register u64 len asm("3") = ulen;
205
+	u64 cnt = ulen;
206
+	int shift = ulen * 8;
207
+	int cc = -ENXIO;
208
+	u64 val, tmp;
209
+
210
+	/*
211
+	 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
212
+	 * user space) into a register using pcilg then store these bytes at
213
+	 * user address @dst
214
+	 */
215
+	asm volatile (
216
+		"       sacf    256\n"
217
+		"0:     .insn   rre,0xb9d60000,%[val],%[ioaddr]\n"
218
+		"1:     ipm     %[cc]\n"
219
+		"       srl     %[cc],28\n"
220
+		"       ltr     %[cc],%[cc]\n"
221
+		"       jne     4f\n"
222
+		"2:     ahi     %[shift],-8\n"
223
+		"       srlg    %[tmp],%[val],0(%[shift])\n"
224
+		"3:     stc     %[tmp],0(%[dst])\n"
225
+		"5:	aghi	%[dst],1\n"
226
+		"       brctg   %[cnt],2b\n"
227
+		"4:     sacf    768\n"
228
+		EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b)
229
+		:
230
+		[cc] "+d" (cc), [val] "=d" (val), [len] "+d" (len),
231
+		[dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
232
+		[shift] "+d" (shift)
233
+		:
234
+		[ioaddr] "a" (addr)
235
+		: "cc", "memory");
236
+
237
+	/* did we write everything to the user space buffer? */
238
+	if (!cc && cnt != 0)
239
+		cc = -EFAULT;
240
+
241
+	*status = len >> 24 & 0xff;
242
+	return cc;
243
+}
244
+
245
+static inline int __memcpy_fromio_inuser(void __user *dst,
246
+				     const void __iomem *src,
247
+				     unsigned long n)
248
+{
249
+	int size, rc = 0;
250
+	u8 status;
251
+	mm_segment_t old_fs;
252
+
253
+	old_fs = enable_sacf_uaccess();
254
+	while (n > 0) {
255
+		size = zpci_get_max_write_size((u64 __force) src,
256
+					       (u64 __force) dst, n,
257
+					       ZPCI_MAX_READ_SIZE);
258
+		rc = __pcilg_mio_inuser(dst, src, size, &status);
259
+		if (rc)
260
+			break;
261
+		src += size;
262
+		dst += size;
263
+		n -= size;
264
+	}
265
+	disable_sacf_uaccess(old_fs);
266
+	if (rc)
267
+		zpci_err_mmio(rc, status, (__force u64) dst);
268
+	return rc;
73
269
 }
74
270
 
75
271
 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
..
..
@@ -86,12 +282,29 @@
86
282
 
87
283
 	if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
88
284
 		return -EINVAL;
285
+
286
+	/*
287
+	 * We only support read access to MIO capable devices if we are on
288
+	 * a MIO enabled system. Otherwise we would have to check for every
289
+	 * address if it is a special ZPCI_ADDR and would have to do
290
+	 * a get_pfn() which we don't need for MIO capable devices.  Currently
291
+	 * ISM devices are the only devices without MIO support and there is no
292
+	 * known need for accessing these from userspace.
293
+	 */
294
+	if (static_branch_likely(&have_mio)) {
295
+		ret = __memcpy_fromio_inuser(
296
+				user_buffer, (const void __iomem *)mmio_addr,
297
+				length);
298
+		return ret;
299
+	}
300
+
89
301
 	if (length > 64) {
90
302
 		buf = kmalloc(length, GFP_KERNEL);
91
303
 		if (!buf)
92
304
 			return -ENOMEM;
93
-	} else
305
+	} else {
94
306
 		buf = local_buf;
307
+	}
95
308
 
96
309
 	ret = get_pfn(mmio_addr, VM_READ, &pfn);
97
310
 	if (ret)