add new system file

hc

2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c

 kernel/mm/userfaultfd.c
..
..
@@ -1,10 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  *  mm/userfaultfd.c
3
4
  *
4
5
  *  Copyright (C) 2015  Red Hat, Inc.
5
- *
6
- *  This work is licensed under the terms of the GNU GPL, version 2. See
7
- *  the COPYING file in the top-level directory.
8
6
  */
9
7
 
10
8
 #include <linux/mm.h>
..
..
@@ -20,21 +18,119 @@
20
18
 #include <asm/tlbflush.h>
21
19
 #include "internal.h"
22
20
 
21
+static __always_inline
22
+struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm,
23
+				    unsigned long dst_start,
24
+				    unsigned long len)
25
+{
26
+	/*
27
+	 * Make sure that the dst range is both valid and fully within a
28
+	 * single existing vma.
29
+	 */
30
+	struct vm_area_struct *dst_vma;
31
+
32
+	dst_vma = find_vma(dst_mm, dst_start);
33
+	if (!dst_vma)
34
+		return NULL;
35
+
36
+	if (dst_start < dst_vma->vm_start ||
37
+	    dst_start + len > dst_vma->vm_end)
38
+		return NULL;
39
+
40
+	/*
41
+	 * Check the vma is registered in uffd, this is required to
42
+	 * enforce the VM_MAYWRITE check done at uffd registration
43
+	 * time.
44
+	 */
45
+	if (!dst_vma->vm_userfaultfd_ctx.ctx)
46
+		return NULL;
47
+
48
+	return dst_vma;
49
+}
50
+
51
+/*
52
+ * Install PTEs, to map dst_addr (within dst_vma) to page.
53
+ *
54
+ * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem
55
+ * and anon, and for both shared and private VMAs.
56
+ */
57
+int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
58
+			     struct vm_area_struct *dst_vma,
59
+			     unsigned long dst_addr, struct page *page,
60
+			     bool newly_allocated, bool wp_copy)
61
+{
62
+	int ret;
63
+	pte_t _dst_pte, *dst_pte;
64
+	bool writable = dst_vma->vm_flags & VM_WRITE;
65
+	bool vm_shared = dst_vma->vm_flags & VM_SHARED;
66
+	bool page_in_cache = page_mapping(page);
67
+	spinlock_t *ptl;
68
+	struct inode *inode;
69
+	pgoff_t offset, max_off;
70
+
71
+	_dst_pte = mk_pte(page, dst_vma->vm_page_prot);
72
+	if (page_in_cache && !vm_shared)
73
+		writable = false;
74
+	if (writable || !page_in_cache)
75
+		_dst_pte = pte_mkdirty(_dst_pte);
76
+	if (writable) {
77
+		if (wp_copy)
78
+			_dst_pte = pte_mkuffd_wp(_dst_pte);
79
+		else
80
+			_dst_pte = pte_mkwrite(_dst_pte);
81
+	}
82
+
83
+	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
84
+
85
+	if (vma_is_shmem(dst_vma)) {
86
+		/* serialize against truncate with the page table lock */
87
+		inode = dst_vma->vm_file->f_inode;
88
+		offset = linear_page_index(dst_vma, dst_addr);
89
+		max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
90
+		ret = -EFAULT;
91
+		if (unlikely(offset >= max_off))
92
+			goto out_unlock;
93
+	}
94
+
95
+	ret = -EEXIST;
96
+	if (!pte_none(*dst_pte))
97
+		goto out_unlock;
98
+
99
+	if (page_in_cache)
100
+		page_add_file_rmap(page, false);
101
+	else
102
+		page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
103
+
104
+	/*
105
+	 * Must happen after rmap, as mm_counter() checks mapping (via
106
+	 * PageAnon()), which is set by __page_set_anon_rmap().
107
+	 */
108
+	inc_mm_counter(dst_mm, mm_counter(page));
109
+
110
+	if (newly_allocated)
111
+		lru_cache_add_inactive_or_unevictable(page, dst_vma);
112
+
113
+	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
114
+
115
+	/* No need to invalidate - it was non-present before */
116
+	update_mmu_cache(dst_vma, dst_addr, dst_pte);
117
+	ret = 0;
118
+out_unlock:
119
+	pte_unmap_unlock(dst_pte, ptl);
120
+	return ret;
121
+}
122
+
23
123
 static int mcopy_atomic_pte(struct mm_struct *dst_mm,
24
124
 			    pmd_t *dst_pmd,
25
125
 			    struct vm_area_struct *dst_vma,
26
126
 			    unsigned long dst_addr,
27
127
 			    unsigned long src_addr,
28
-			    struct page **pagep)
128
+			    struct page **pagep,
129
+			    bool wp_copy)
29
130
 {
30
-	struct mem_cgroup *memcg;
31
-	pte_t _dst_pte, *dst_pte;
32
-	spinlock_t *ptl;
33
131
 	void *page_kaddr;
34
132
 	int ret;
35
133
 	struct page *page;
36
-	pgoff_t offset, max_off;
37
-	struct inode *inode;
38
134
 
39
135
 	if (!*pagep) {
40
136
 		ret = -ENOMEM;
..
..
@@ -48,13 +144,15 @@
48
144
 				     PAGE_SIZE);
49
145
 		kunmap_atomic(page_kaddr);
50
146
 
51
-		/* fallback to copy_from_user outside mmap_sem */
147
+		/* fallback to copy_from_user outside mmap_lock */
52
148
 		if (unlikely(ret)) {
53
149
 			ret = -ENOENT;
54
150
 			*pagep = page;
55
151
 			/* don't free the page */
56
152
 			goto out;
57
153
 		}
154
+
155
+		flush_dcache_page(page);
58
156
 	} else {
59
157
 		page = *pagep;
60
158
 		*pagep = NULL;
..
..
@@ -62,50 +160,21 @@
62
160
 
63
161
 	/*
64
162
 	 * The memory barrier inside __SetPageUptodate makes sure that
65
-	 * preceeding stores to the page contents become visible before
163
+	 * preceding stores to the page contents become visible before
66
164
 	 * the set_pte_at() write.
67
165
 	 */
68
166
 	__SetPageUptodate(page);
69
167
 
70
168
 	ret = -ENOMEM;
71
-	if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg, false))
169
+	if (mem_cgroup_charge(page, dst_mm, GFP_KERNEL))
72
170
 		goto out_release;
73
171
 
74
-	_dst_pte = mk_pte(page, dst_vma->vm_page_prot);
75
-	if (dst_vma->vm_flags & VM_WRITE)
76
-		_dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
77
-
78
-	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
79
-	if (dst_vma->vm_file) {
80
-		/* the shmem MAP_PRIVATE case requires checking the i_size */
81
-		inode = dst_vma->vm_file->f_inode;
82
-		offset = linear_page_index(dst_vma, dst_addr);
83
-		max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
84
-		ret = -EFAULT;
85
-		if (unlikely(offset >= max_off))
86
-			goto out_release_uncharge_unlock;
87
-	}
88
-	ret = -EEXIST;
89
-	if (!pte_none(*dst_pte))
90
-		goto out_release_uncharge_unlock;
91
-
92
-	inc_mm_counter(dst_mm, MM_ANONPAGES);
93
-	page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
94
-	mem_cgroup_commit_charge(page, memcg, false, false);
95
-	lru_cache_add_active_or_unevictable(page, dst_vma);
96
-
97
-	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
98
-
99
-	/* No need to invalidate - it was non-present before */
100
-	update_mmu_cache(dst_vma, dst_addr, dst_pte);
101
-
102
-	pte_unmap_unlock(dst_pte, ptl);
103
-	ret = 0;
172
+	ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
173
+				       page, true, wp_copy);
174
+	if (ret)
175
+		goto out_release;
104
176
 out:
105
177
 	return ret;
106
-out_release_uncharge_unlock:
107
-	pte_unmap_unlock(dst_pte, ptl);
108
-	mem_cgroup_cancel_charge(page, memcg, false);
109
178
 out_release:
110
179
 	put_page(page);
111
180
 	goto out;
..
..
@@ -146,6 +215,41 @@
146
215
 	return ret;
147
216
 }
148
217
 
218
+/* Handles UFFDIO_CONTINUE for all shmem VMAs (shared or private). */
219
+static int mcontinue_atomic_pte(struct mm_struct *dst_mm,
220
+				pmd_t *dst_pmd,
221
+				struct vm_area_struct *dst_vma,
222
+				unsigned long dst_addr,
223
+				bool wp_copy)
224
+{
225
+	struct inode *inode = file_inode(dst_vma->vm_file);
226
+	pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
227
+	struct page *page;
228
+	int ret;
229
+
230
+	ret = shmem_getpage(inode, pgoff, &page, SGP_READ);
231
+	if (ret)
232
+		goto out;
233
+	if (!page) {
234
+		ret = -EFAULT;
235
+		goto out;
236
+	}
237
+
238
+	ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
239
+				       page, false, wp_copy);
240
+	if (ret)
241
+		goto out_release;
242
+
243
+	unlock_page(page);
244
+	ret = 0;
245
+out:
246
+	return ret;
247
+out_release:
248
+	unlock_page(page);
249
+	put_page(page);
250
+	goto out;
251
+}
252
+
149
253
 static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
150
254
 {
151
255
 	pgd_t *pgd;
..
..
@@ -170,14 +274,14 @@
170
274
 #ifdef CONFIG_HUGETLB_PAGE
171
275
 /*
172
276
  * __mcopy_atomic processing for HUGETLB vmas.  Note that this routine is
173
- * called with mmap_sem held, it will release mmap_sem before returning.
277
+ * called with mmap_lock held, it will release mmap_lock before returning.
174
278
  */
175
279
 static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
176
280
 					      struct vm_area_struct *dst_vma,
177
281
 					      unsigned long dst_start,
178
282
 					      unsigned long src_start,
179
283
 					      unsigned long len,
180
-					      bool zeropage)
284
+					      enum mcopy_atomic_mode mode)
181
285
 {
182
286
 	int vm_alloc_shared = dst_vma->vm_flags & VM_SHARED;
183
287
 	int vm_shared = dst_vma->vm_flags & VM_SHARED;
..
..
@@ -186,7 +290,6 @@
186
290
 	unsigned long src_addr, dst_addr;
187
291
 	long copied;
188
292
 	struct page *page;
189
-	struct hstate *h;
190
293
 	unsigned long vma_hpagesize;
191
294
 	pgoff_t idx;
192
295
 	u32 hash;
..
..
@@ -198,8 +301,8 @@
198
301
 	 * by THP.  Since we can not reliably insert a zero page, this
199
302
 	 * feature is not supported.
200
303
 	 */
201
-	if (zeropage) {
202
-		up_read(&dst_mm->mmap_sem);
304
+	if (mode == MCOPY_ATOMIC_ZEROPAGE) {
305
+		mmap_read_unlock(dst_mm);
203
306
 		return -EINVAL;
204
307
 	}
205
308
 
..
..
@@ -218,24 +321,13 @@
218
321
 
219
322
 retry:
220
323
 	/*
221
-	 * On routine entry dst_vma is set.  If we had to drop mmap_sem and
324
+	 * On routine entry dst_vma is set.  If we had to drop mmap_lock and
222
325
 	 * retry, dst_vma will be set to NULL and we must lookup again.
223
326
 	 */
224
327
 	if (!dst_vma) {
225
328
 		err = -ENOENT;
226
-		dst_vma = find_vma(dst_mm, dst_start);
329
+		dst_vma = find_dst_vma(dst_mm, dst_start, len);
227
330
 		if (!dst_vma || !is_vm_hugetlb_page(dst_vma))
228
-			goto out_unlock;
229
-		/*
230
-		 * Check the vma is registered in uffd, this is
231
-		 * required to enforce the VM_MAYWRITE check done at
232
-		 * uffd registration time.
233
-		 */
234
-		if (!dst_vma->vm_userfaultfd_ctx.ctx)
235
-			goto out_unlock;
236
-
237
-		if (dst_start < dst_vma->vm_start ||
238
-		    dst_start + len > dst_vma->vm_end)
239
331
 			goto out_unlock;
240
332
 
241
333
 		err = -EINVAL;
..
..
@@ -244,10 +336,6 @@
244
336
 
245
337
 		vm_shared = dst_vma->vm_flags & VM_SHARED;
246
338
 	}
247
-
248
-	if (WARN_ON(dst_addr & (vma_hpagesize - 1) ||
249
-		    (len - copied) & (vma_hpagesize - 1)))
250
-		goto out_unlock;
251
339
 
252
340
 	/*
253
341
 	 * If not shared, ensure the dst_vma has a anon_vma.
..
..
@@ -258,56 +346,59 @@
258
346
 			goto out_unlock;
259
347
 	}
260
348
 
261
-	h = hstate_vma(dst_vma);
262
-
263
349
 	while (src_addr < src_start + len) {
264
-		pte_t dst_pteval;
265
-
266
350
 		BUG_ON(dst_addr >= dst_start + len);
267
-		VM_BUG_ON(dst_addr & ~huge_page_mask(h));
268
351
 
269
352
 		/*
270
-		 * Serialize via hugetlb_fault_mutex
353
+		 * Serialize via i_mmap_rwsem and hugetlb_fault_mutex.
354
+		 * i_mmap_rwsem ensures the dst_pte remains valid even
355
+		 * in the case of shared pmds.  fault mutex prevents
356
+		 * races with other faulting threads.
271
357
 		 */
272
-		idx = linear_page_index(dst_vma, dst_addr);
273
358
 		mapping = dst_vma->vm_file->f_mapping;
274
-		hash = hugetlb_fault_mutex_hash(h, mapping, idx);
359
+		i_mmap_lock_read(mapping);
360
+		idx = linear_page_index(dst_vma, dst_addr);
361
+		hash = hugetlb_fault_mutex_hash(mapping, idx);
275
362
 		mutex_lock(&hugetlb_fault_mutex_table[hash]);
276
363
 
277
364
 		err = -ENOMEM;
278
-		dst_pte = huge_pte_alloc(dst_mm, dst_addr, huge_page_size(h));
365
+		dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize);
279
366
 		if (!dst_pte) {
280
367
 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
368
+			i_mmap_unlock_read(mapping);
281
369
 			goto out_unlock;
282
370
 		}
283
371
 
284
-		err = -EEXIST;
285
-		dst_pteval = huge_ptep_get(dst_pte);
286
-		if (!huge_pte_none(dst_pteval)) {
372
+		if (mode != MCOPY_ATOMIC_CONTINUE &&
373
+		    !huge_pte_none(huge_ptep_get(dst_pte))) {
374
+			err = -EEXIST;
287
375
 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
376
+			i_mmap_unlock_read(mapping);
288
377
 			goto out_unlock;
289
378
 		}
290
379
 
291
380
 		err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma,
292
-						dst_addr, src_addr, &page);
381
+					       dst_addr, src_addr, mode, &page);
293
382
 
294
383
 		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
384
+		i_mmap_unlock_read(mapping);
295
385
 		vm_alloc_shared = vm_shared;
296
386
 
297
387
 		cond_resched();
298
388
 
299
389
 		if (unlikely(err == -ENOENT)) {
300
-			up_read(&dst_mm->mmap_sem);
390
+			mmap_read_unlock(dst_mm);
301
391
 			BUG_ON(!page);
302
392
 
303
393
 			err = copy_huge_page_from_user(page,
304
394
 						(const void __user *)src_addr,
305
-						pages_per_huge_page(h), true);
395
+						vma_hpagesize / PAGE_SIZE,
396
+						true);
306
397
 			if (unlikely(err)) {
307
398
 				err = -EFAULT;
308
399
 				goto out;
309
400
 			}
310
-			down_read(&dst_mm->mmap_sem);
401
+			mmap_read_lock(dst_mm);
311
402
 
312
403
 			dst_vma = NULL;
313
404
 			goto retry;
..
..
@@ -327,7 +418,7 @@
327
418
 	}
328
419
 
329
420
 out_unlock:
330
-	up_read(&dst_mm->mmap_sem);
421
+	mmap_read_unlock(dst_mm);
331
422
 out:
332
423
 	if (page) {
333
424
 		/*
..
..
@@ -338,7 +429,7 @@
338
429
 		 * private and shared mappings.  See the routine
339
430
 		 * restore_reserve_on_error for details.  Unfortunately, we
340
431
 		 * can not call restore_reserve_on_error now as it would
341
-		 * require holding mmap_sem.
432
+		 * require holding mmap_lock.
342
433
 		 *
343
434
 		 * If a reservation for the page existed in the reservation
344
435
 		 * map of a private mapping, the map was modified to indicate
..
..
@@ -389,7 +480,7 @@
389
480
 				      unsigned long dst_start,
390
481
 				      unsigned long src_start,
391
482
 				      unsigned long len,
392
-				      bool zeropage);
483
+				      enum mcopy_atomic_mode mode);
393
484
 #endif /* CONFIG_HUGETLB_PAGE */
394
485
 
395
486
 static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm,
..
..
@@ -398,9 +489,15 @@
398
489
 						unsigned long dst_addr,
399
490
 						unsigned long src_addr,
400
491
 						struct page **page,
401
-						bool zeropage)
492
+						enum mcopy_atomic_mode mode,
493
+						bool wp_copy)
402
494
 {
403
495
 	ssize_t err;
496
+
497
+	if (mode == MCOPY_ATOMIC_CONTINUE) {
498
+		return mcontinue_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
499
+					    wp_copy);
500
+	}
404
501
 
405
502
 	/*
406
503
 	 * The normal page fault path for a shmem will invoke the
..
..
@@ -413,20 +510,19 @@
413
510
 	 * and not in the radix tree.
414
511
 	 */
415
512
 	if (!(dst_vma->vm_flags & VM_SHARED)) {
416
-		if (!zeropage)
513
+		if (mode == MCOPY_ATOMIC_NORMAL)
417
514
 			err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma,
418
-					       dst_addr, src_addr, page);
515
+					       dst_addr, src_addr, page,
516
+					       wp_copy);
419
517
 		else
420
518
 			err = mfill_zeropage_pte(dst_mm, dst_pmd,
421
519
 						 dst_vma, dst_addr);
422
520
 	} else {
423
-		if (!zeropage)
424
-			err = shmem_mcopy_atomic_pte(dst_mm, dst_pmd,
425
-						     dst_vma, dst_addr,
426
-						     src_addr, page);
427
-		else
428
-			err = shmem_mfill_zeropage_pte(dst_mm, dst_pmd,
429
-						       dst_vma, dst_addr);
521
+		VM_WARN_ON_ONCE(wp_copy);
522
+		err = shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
523
+					     dst_addr, src_addr,
524
+					     mode != MCOPY_ATOMIC_NORMAL,
525
+					     page);
430
526
 	}
431
527
 
432
528
 	return err;
..
..
@@ -436,8 +532,9 @@
436
532
 					      unsigned long dst_start,
437
533
 					      unsigned long src_start,
438
534
 					      unsigned long len,
439
-					      bool zeropage,
440
-					      bool *mmap_changing)
535
+					      enum mcopy_atomic_mode mcopy_mode,
536
+					      bool *mmap_changing,
537
+					      __u64 mode)
441
538
 {
442
539
 	struct vm_area_struct *dst_vma;
443
540
 	ssize_t err;
..
..
@@ -445,6 +542,7 @@
445
542
 	unsigned long src_addr, dst_addr;
446
543
 	long copied;
447
544
 	struct page *page;
545
+	bool wp_copy;
448
546
 
449
547
 	/*
450
548
 	 * Sanitize the command parameters:
..
..
@@ -461,7 +559,7 @@
461
559
 	copied = 0;
462
560
 	page = NULL;
463
561
 retry:
464
-	down_read(&dst_mm->mmap_sem);
562
+	mmap_read_lock(dst_mm);
465
563
 
466
564
 	/*
467
565
 	 * If memory mappings are changing because of non-cooperative
..
..
@@ -477,19 +575,8 @@
477
575
 	 * both valid and fully within a single existing vma.
478
576
 	 */
479
577
 	err = -ENOENT;
480
-	dst_vma = find_vma(dst_mm, dst_start);
578
+	dst_vma = find_dst_vma(dst_mm, dst_start, len);
481
579
 	if (!dst_vma)
482
-		goto out_unlock;
483
-	/*
484
-	 * Check the vma is registered in uffd, this is required to
485
-	 * enforce the VM_MAYWRITE check done at uffd registration
486
-	 * time.
487
-	 */
488
-	if (!dst_vma->vm_userfaultfd_ctx.ctx)
489
-		goto out_unlock;
490
-
491
-	if (dst_start < dst_vma->vm_start ||
492
-	    dst_start + len > dst_vma->vm_end)
493
580
 		goto out_unlock;
494
581
 
495
582
 	err = -EINVAL;
..
..
@@ -502,13 +589,23 @@
502
589
 		goto out_unlock;
503
590
 
504
591
 	/*
592
+	 * validate 'mode' now that we know the dst_vma: don't allow
593
+	 * a wrprotect copy if the userfaultfd didn't register as WP.
594
+	 */
595
+	wp_copy = mode & UFFDIO_COPY_MODE_WP;
596
+	if (wp_copy && !(dst_vma->vm_flags & VM_UFFD_WP))
597
+		goto out_unlock;
598
+
599
+	/*
505
600
 	 * If this is a HUGETLB vma, pass off to appropriate routine
506
601
 	 */
507
602
 	if (is_vm_hugetlb_page(dst_vma))
508
603
 		return  __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start,
509
-						src_start, len, zeropage);
604
+						src_start, len, mcopy_mode);
510
605
 
511
606
 	if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma))
607
+		goto out_unlock;
608
+	if (!vma_is_shmem(dst_vma) && mcopy_mode == MCOPY_ATOMIC_CONTINUE)
512
609
 		goto out_unlock;
513
610
 
514
611
 	/*
..
..
@@ -542,7 +639,7 @@
542
639
 			break;
543
640
 		}
544
641
 		if (unlikely(pmd_none(dst_pmdval)) &&
545
-		    unlikely(__pte_alloc(dst_mm, dst_pmd, dst_addr))) {
642
+		    unlikely(__pte_alloc(dst_mm, dst_pmd))) {
546
643
 			err = -ENOMEM;
547
644
 			break;
548
645
 		}
..
..
@@ -556,13 +653,13 @@
556
653
 		BUG_ON(pmd_trans_huge(*dst_pmd));
557
654
 
558
655
 		err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
559
-				       src_addr, &page, zeropage);
656
+				       src_addr, &page, mcopy_mode, wp_copy);
560
657
 		cond_resched();
561
658
 
562
659
 		if (unlikely(err == -ENOENT)) {
563
660
 			void *page_kaddr;
564
661
 
565
-			up_read(&dst_mm->mmap_sem);
662
+			mmap_read_unlock(dst_mm);
566
663
 			BUG_ON(!page);
567
664
 
568
665
 			page_kaddr = kmap(page);
..
..
@@ -574,6 +671,7 @@
574
671
 				err = -EFAULT;
575
672
 				goto out;
576
673
 			}
674
+			flush_dcache_page(page);
577
675
 			goto retry;
578
676
 		} else
579
677
 			BUG_ON(page);
..
..
@@ -591,7 +689,7 @@
591
689
 	}
592
690
 
593
691
 out_unlock:
594
-	up_read(&dst_mm->mmap_sem);
692
+	mmap_read_unlock(dst_mm);
595
693
 out:
596
694
 	if (page)
597
695
 		put_page(page);
..
..
@@ -603,14 +701,76 @@
603
701
 
604
702
 ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
605
703
 		     unsigned long src_start, unsigned long len,
606
-		     bool *mmap_changing)
704
+		     bool *mmap_changing, __u64 mode)
607
705
 {
608
-	return __mcopy_atomic(dst_mm, dst_start, src_start, len, false,
609
-			      mmap_changing);
706
+	return __mcopy_atomic(dst_mm, dst_start, src_start, len,
707
+			      MCOPY_ATOMIC_NORMAL, mmap_changing, mode);
610
708
 }
611
709
 
612
710
 ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start,
613
711
 		       unsigned long len, bool *mmap_changing)
614
712
 {
615
-	return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing);
713
+	return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE,
714
+			      mmap_changing, 0);
715
+}
716
+
717
+ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start,
718
+		       unsigned long len, bool *mmap_changing)
719
+{
720
+	return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_CONTINUE,
721
+			      mmap_changing, 0);
722
+}
723
+
724
+int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start,
725
+			unsigned long len, bool enable_wp, bool *mmap_changing)
726
+{
727
+	struct vm_area_struct *dst_vma;
728
+	pgprot_t newprot;
729
+	int err;
730
+
731
+	/*
732
+	 * Sanitize the command parameters:
733
+	 */
734
+	BUG_ON(start & ~PAGE_MASK);
735
+	BUG_ON(len & ~PAGE_MASK);
736
+
737
+	/* Does the address range wrap, or is the span zero-sized? */
738
+	BUG_ON(start + len <= start);
739
+
740
+	mmap_read_lock(dst_mm);
741
+
742
+	/*
743
+	 * If memory mappings are changing because of non-cooperative
744
+	 * operation (e.g. mremap) running in parallel, bail out and
745
+	 * request the user to retry later
746
+	 */
747
+	err = -EAGAIN;
748
+	if (mmap_changing && READ_ONCE(*mmap_changing))
749
+		goto out_unlock;
750
+
751
+	err = -ENOENT;
752
+	dst_vma = find_dst_vma(dst_mm, start, len);
753
+	/*
754
+	 * Make sure the vma is not shared, that the dst range is
755
+	 * both valid and fully within a single existing vma.
756
+	 */
757
+	if (!dst_vma || (dst_vma->vm_flags & VM_SHARED))
758
+		goto out_unlock;
759
+	if (!userfaultfd_wp(dst_vma))
760
+		goto out_unlock;
761
+	if (!vma_is_anonymous(dst_vma))
762
+		goto out_unlock;
763
+
764
+	if (enable_wp)
765
+		newprot = vm_get_page_prot(dst_vma->vm_flags & ~(VM_WRITE));
766
+	else
767
+		newprot = vm_get_page_prot(dst_vma->vm_flags);
768
+
769
+	change_protection(dst_vma, start, start + len, newprot,
770
+			  enable_wp ? MM_CP_UFFD_WP : MM_CP_UFFD_WP_RESOLVE);
771
+
772
+	err = 0;
773
+out_unlock:
774
+	mmap_read_unlock(dst_mm);
775
+	return err;
616
776
 }