change system file

hc

2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771

 kernel/mm/maccess.c
..
..
@@ -1,94 +1,130 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
- * Access kernel memory without faulting.
3
+ * Access kernel or user memory without faulting.
3
4
  */
4
5
 #include <linux/export.h>
5
6
 #include <linux/mm.h>
6
7
 #include <linux/uaccess.h>
7
8
 
8
-static __always_inline long
9
-probe_read_common(void *dst, const void __user *src, size_t size)
9
+bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
10
+		size_t size)
10
11
 {
11
-	long ret;
12
-
13
-	pagefault_disable();
14
-	ret = __copy_from_user_inatomic(dst, src, size);
15
-	pagefault_enable();
16
-
17
-	return ret ? -EFAULT : 0;
12
+	return true;
18
13
 }
19
14
 
20
-static __always_inline long
21
-probe_write_common(void __user *dst, const void *src, size_t size)
15
+#ifdef HAVE_GET_KERNEL_NOFAULT
16
+
17
+#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label)	\
18
+	while (len >= sizeof(type)) {					\
19
+		__get_kernel_nofault(dst, src, type, err_label);		\
20
+		dst += sizeof(type);					\
21
+		src += sizeof(type);					\
22
+		len -= sizeof(type);					\
23
+	}
24
+
25
+long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
22
26
 {
23
-	long ret;
27
+	if (!copy_from_kernel_nofault_allowed(src, size))
28
+		return -ERANGE;
24
29
 
25
30
 	pagefault_disable();
26
-	ret = __copy_to_user_inatomic(dst, src, size);
31
+	copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
32
+	copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
33
+	copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
34
+	copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
27
35
 	pagefault_enable();
36
+	return 0;
37
+Efault:
38
+	pagefault_enable();
39
+	return -EFAULT;
40
+}
41
+EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
28
42
 
29
-	return ret ? -EFAULT : 0;
43
+#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label)	\
44
+	while (len >= sizeof(type)) {					\
45
+		__put_kernel_nofault(dst, src, type, err_label);		\
46
+		dst += sizeof(type);					\
47
+		src += sizeof(type);					\
48
+		len -= sizeof(type);					\
49
+	}
50
+
51
+long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
52
+{
53
+	pagefault_disable();
54
+	copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
55
+	copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
56
+	copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
57
+	copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
58
+	pagefault_enable();
59
+	return 0;
60
+Efault:
61
+	pagefault_enable();
62
+	return -EFAULT;
30
63
 }
31
64
 
65
+long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
66
+{
67
+	const void *src = unsafe_addr;
68
+
69
+	if (unlikely(count <= 0))
70
+		return 0;
71
+	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
72
+		return -ERANGE;
73
+
74
+	pagefault_disable();
75
+	do {
76
+		__get_kernel_nofault(dst, src, u8, Efault);
77
+		dst++;
78
+		src++;
79
+	} while (dst[-1] && src - unsafe_addr < count);
80
+	pagefault_enable();
81
+
82
+	dst[-1] = '\0';
83
+	return src - unsafe_addr;
84
+Efault:
85
+	pagefault_enable();
86
+	dst[0] = '\0';
87
+	return -EFAULT;
88
+}
89
+#else /* HAVE_GET_KERNEL_NOFAULT */
32
90
 /**
33
- * probe_kernel_read(): safely attempt to read from a kernel-space location
91
+ * copy_from_kernel_nofault(): safely attempt to read from kernel-space
34
92
  * @dst: pointer to the buffer that shall take the data
35
93
  * @src: address to read from
36
94
  * @size: size of the data chunk
37
95
  *
38
- * Safely read from address @src to the buffer at @dst.  If a kernel fault
39
- * happens, handle that and return -EFAULT.
96
+ * Safely read from kernel address @src to the buffer at @dst.  If a kernel
97
+ * fault happens, handle that and return -EFAULT.  If @src is not a valid kernel
98
+ * address, return -ERANGE.
40
99
  *
41
100
  * We ensure that the copy_from_user is executed in atomic context so that
42
- * do_page_fault() doesn't attempt to take mmap_sem.  This makes
43
- * probe_kernel_read() suitable for use within regions where the caller
44
- * already holds mmap_sem, or other locks which nest inside mmap_sem.
101
+ * do_page_fault() doesn't attempt to take mmap_lock.  This makes
102
+ * copy_from_kernel_nofault() suitable for use within regions where the caller
103
+ * already holds mmap_lock, or other locks which nest inside mmap_lock.
45
104
  */
46
-
47
-long __weak probe_kernel_read(void *dst, const void *src, size_t size)
48
-    __attribute__((alias("__probe_kernel_read")));
49
-
50
-long __probe_kernel_read(void *dst, const void *src, size_t size)
105
+long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
51
106
 {
52
107
 	long ret;
53
108
 	mm_segment_t old_fs = get_fs();
54
109
 
110
+	if (!copy_from_kernel_nofault_allowed(src, size))
111
+		return -ERANGE;
112
+
55
113
 	set_fs(KERNEL_DS);
56
-	ret = probe_read_common(dst, (__force const void __user *)src, size);
114
+	pagefault_disable();
115
+	ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
116
+			size);
117
+	pagefault_enable();
57
118
 	set_fs(old_fs);
58
119
 
59
-	return ret;
120
+	if (ret)
121
+		return -EFAULT;
122
+	return 0;
60
123
 }
61
-EXPORT_SYMBOL_GPL(probe_kernel_read);
124
+EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
62
125
 
63
126
 /**
64
- * probe_user_read(): safely attempt to read from a user-space location
65
- * @dst: pointer to the buffer that shall take the data
66
- * @src: address to read from. This must be a user address.
67
- * @size: size of the data chunk
68
- *
69
- * Safely read from user address @src to the buffer at @dst. If a kernel fault
70
- * happens, handle that and return -EFAULT.
71
- */
72
-
73
-long __weak probe_user_read(void *dst, const void __user *src, size_t size)
74
-    __attribute__((alias("__probe_user_read")));
75
-
76
-long __probe_user_read(void *dst, const void __user *src, size_t size)
77
-{
78
-	long ret = -EFAULT;
79
-	mm_segment_t old_fs = get_fs();
80
-
81
-	set_fs(USER_DS);
82
-	if (access_ok(VERIFY_READ, src, size))
83
-		ret = probe_read_common(dst, src, size);
84
-	set_fs(old_fs);
85
-
86
-	return ret;
87
-}
88
-EXPORT_SYMBOL_GPL(probe_user_read);
89
-
90
-/**
91
- * probe_kernel_write(): safely attempt to write to a location
127
+ * copy_to_kernel_nofault(): safely attempt to write to a location
92
128
  * @dst: address to write to
93
129
  * @src: pointer to the data that shall be written
94
130
  * @size: size of the data chunk
..
..
@@ -96,52 +132,25 @@
96
132
  * Safely write to address @dst from the buffer at @src.  If a kernel fault
97
133
  * happens, handle that and return -EFAULT.
98
134
  */
99
-
100
-long __weak probe_kernel_write(void *dst, const void *src, size_t size)
101
-    __attribute__((alias("__probe_kernel_write")));
102
-
103
-long __probe_kernel_write(void *dst, const void *src, size_t size)
135
+long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
104
136
 {
105
137
 	long ret;
106
138
 	mm_segment_t old_fs = get_fs();
107
139
 
108
140
 	set_fs(KERNEL_DS);
109
-	ret = probe_write_common((__force void __user *)dst, src, size);
141
+	pagefault_disable();
142
+	ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
143
+	pagefault_enable();
110
144
 	set_fs(old_fs);
111
145
 
112
-	return ret;
146
+	if (ret)
147
+		return -EFAULT;
148
+	return 0;
113
149
 }
114
-EXPORT_SYMBOL_GPL(probe_kernel_write);
115
150
 
116
151
 /**
117
- * probe_user_write(): safely attempt to write to a user-space location
118
- * @dst: address to write to
119
- * @src: pointer to the data that shall be written
120
- * @size: size of the data chunk
121
- *
122
- * Safely write to address @dst from the buffer at @src.  If a kernel fault
123
- * happens, handle that and return -EFAULT.
124
- */
125
-
126
-long __weak probe_user_write(void __user *dst, const void *src, size_t size)
127
-    __attribute__((alias("__probe_user_write")));
128
-
129
-long __probe_user_write(void __user *dst, const void *src, size_t size)
130
-{
131
-	long ret = -EFAULT;
132
-	mm_segment_t old_fs = get_fs();
133
-
134
-	set_fs(USER_DS);
135
-	if (access_ok(VERIFY_WRITE, dst, size))
136
-		ret = probe_write_common(dst, src, size);
137
-	set_fs(old_fs);
138
-
139
-	return ret;
140
-}
141
-EXPORT_SYMBOL_GPL(probe_user_write);
142
-
143
-/**
144
- * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address.
152
+ * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
153
+ *				 address.
145
154
  * @dst:   Destination address, in kernel space.  This buffer must be at
146
155
  *         least @count bytes long.
147
156
  * @unsafe_addr: Unsafe address.
..
..
@@ -151,13 +160,14 @@
151
160
  *
152
161
  * On success, returns the length of the string INCLUDING the trailing NUL.
153
162
  *
154
- * If access fails, returns -EFAULT (some data may have been copied
155
- * and the trailing NUL added).
163
+ * If access fails, returns -EFAULT (some data may have been copied and the
164
+ * trailing NUL added).  If @unsafe_addr is not a valid kernel address, return
165
+ * -ERANGE.
156
166
  *
157
167
  * If @count is smaller than the length of the string, copies @count-1 bytes,
158
168
  * sets the last byte of @dst buffer to NUL and returns @count.
159
169
  */
160
-long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count)
170
+long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
161
171
 {
162
172
 	mm_segment_t old_fs = get_fs();
163
173
 	const void *src = unsafe_addr;
..
..
@@ -165,6 +175,8 @@
165
175
 
166
176
 	if (unlikely(count <= 0))
167
177
 		return 0;
178
+	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
179
+		return -ERANGE;
168
180
 
169
181
 	set_fs(KERNEL_DS);
170
182
 	pagefault_disable();
..
..
@@ -179,9 +191,64 @@
179
191
 
180
192
 	return ret ? -EFAULT : src - unsafe_addr;
181
193
 }
194
+#endif /* HAVE_GET_KERNEL_NOFAULT */
182
195
 
183
196
 /**
184
- * strncpy_from_unsafe_user: - Copy a NUL terminated string from unsafe user
197
+ * copy_from_user_nofault(): safely attempt to read from a user-space location
198
+ * @dst: pointer to the buffer that shall take the data
199
+ * @src: address to read from. This must be a user address.
200
+ * @size: size of the data chunk
201
+ *
202
+ * Safely read from user address @src to the buffer at @dst. If a kernel fault
203
+ * happens, handle that and return -EFAULT.
204
+ */
205
+long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
206
+{
207
+	long ret = -EFAULT;
208
+	mm_segment_t old_fs = force_uaccess_begin();
209
+
210
+	if (access_ok(src, size)) {
211
+		pagefault_disable();
212
+		ret = __copy_from_user_inatomic(dst, src, size);
213
+		pagefault_enable();
214
+	}
215
+	force_uaccess_end(old_fs);
216
+
217
+	if (ret)
218
+		return -EFAULT;
219
+	return 0;
220
+}
221
+EXPORT_SYMBOL_GPL(copy_from_user_nofault);
222
+
223
+/**
224
+ * copy_to_user_nofault(): safely attempt to write to a user-space location
225
+ * @dst: address to write to
226
+ * @src: pointer to the data that shall be written
227
+ * @size: size of the data chunk
228
+ *
229
+ * Safely write to address @dst from the buffer at @src.  If a kernel fault
230
+ * happens, handle that and return -EFAULT.
231
+ */
232
+long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
233
+{
234
+	long ret = -EFAULT;
235
+	mm_segment_t old_fs = force_uaccess_begin();
236
+
237
+	if (access_ok(dst, size)) {
238
+		pagefault_disable();
239
+		ret = __copy_to_user_inatomic(dst, src, size);
240
+		pagefault_enable();
241
+	}
242
+	force_uaccess_end(old_fs);
243
+
244
+	if (ret)
245
+		return -EFAULT;
246
+	return 0;
247
+}
248
+EXPORT_SYMBOL_GPL(copy_to_user_nofault);
249
+
250
+/**
251
+ * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
185
252
  *				address.
186
253
  * @dst:   Destination address, in kernel space.  This buffer must be at
187
254
  *         least @count bytes long.
..
..
@@ -198,20 +265,20 @@
198
265
  * If @count is smaller than the length of the string, copies @count-1 bytes,
199
266
  * sets the last byte of @dst buffer to NUL and returns @count.
200
267
  */
201
-long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr,
268
+long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
202
269
 			      long count)
203
270
 {
204
-	mm_segment_t old_fs = get_fs();
271
+	mm_segment_t old_fs;
205
272
 	long ret;
206
273
 
207
274
 	if (unlikely(count <= 0))
208
275
 		return 0;
209
276
 
210
-	set_fs(USER_DS);
277
+	old_fs = force_uaccess_begin();
211
278
 	pagefault_disable();
212
279
 	ret = strncpy_from_user(dst, unsafe_addr, count);
213
280
 	pagefault_enable();
214
-	set_fs(old_fs);
281
+	force_uaccess_end(old_fs);
215
282
 
216
283
 	if (ret >= count) {
217
284
 		ret = count;
..
..
@@ -224,7 +291,7 @@
224
291
 }
225
292
 
226
293
 /**
227
- * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL.
294
+ * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
228
295
  * @unsafe_addr: The string to measure.
229
296
  * @count: Maximum count (including NUL)
230
297
  *
..
..
@@ -239,16 +306,16 @@
239
306
  * Unlike strnlen_user, this can be used from IRQ handler etc. because
240
307
  * it disables pagefaults.
241
308
  */
242
-long strnlen_unsafe_user(const void __user *unsafe_addr, long count)
309
+long strnlen_user_nofault(const void __user *unsafe_addr, long count)
243
310
 {
244
-	mm_segment_t old_fs = get_fs();
311
+	mm_segment_t old_fs;
245
312
 	int ret;
246
313
 
247
-	set_fs(USER_DS);
314
+	old_fs = force_uaccess_begin();
248
315
 	pagefault_disable();
249
316
 	ret = strnlen_user(unsafe_addr, count);
250
317
 	pagefault_enable();
251
-	set_fs(old_fs);
318
+	force_uaccess_end(old_fs);
252
319
 
253
320
 	return ret;
254
321
 }