forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 093a6c67005148ae32a5c9e4553491b9f5c2457b 
 kernel/include/linux/mmu_notifier.h
....@@ -2,44 +2,70 @@
22 #ifndef _LINUX_MMU_NOTIFIER_H
33 #define _LINUX_MMU_NOTIFIER_H
44 
5
-#include <linux/types.h>
65 #include <linux/list.h>
76 #include <linux/spinlock.h>
87 #include <linux/mm_types.h>
8
+#include <linux/mmap_lock.h>
9
+#include <linux/percpu-rwsem.h>
10
+#include <linux/slab.h>
911 #include <linux/srcu.h>
12
+#include <linux/interval_tree.h>
1013 #include <linux/android_kabi.h>
1114 
15
+struct mmu_notifier_subscriptions;
1216 struct mmu_notifier;
13
-struct mmu_notifier_ops;
17
+struct mmu_notifier_range;
18
+struct mmu_interval_notifier;
1419 
15
-/* mmu_notifier_ops flags */
16
-#define MMU_INVALIDATE_DOES_NOT_BLOCK	(0x01)
17
-
18
-#ifdef CONFIG_MMU_NOTIFIER
19
-
20
-/*
21
- * The mmu notifier_mm structure is allocated and installed in
22
- * mm->mmu_notifier_mm inside the mm_take_all_locks() protected
23
- * critical section and it's released only when mm_count reaches zero
24
- * in mmdrop().
25
- */
26
-struct mmu_notifier_mm {
27
-	/* all mmu notifiers registerd in this mm are queued in this list */
28
-	struct hlist_head list;
29
-	/* to serialize the list modifications and hlist_unhashed */
30
-	spinlock_t lock;
20
+struct mmu_notifier_subscriptions_hdr {
21
+	bool valid;
22
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
23
+	struct percpu_rw_semaphore_atomic *mmu_notifier_lock;
24
+#endif
3125 };
3226 
33
-struct mmu_notifier_ops {
34
-	/*
35
-	 * Flags to specify behavior of callbacks for this MMU notifier.
36
-	 * Used to determine which context an operation may be called.
37
-	 *
38
-	 * MMU_INVALIDATE_DOES_NOT_BLOCK: invalidate_range_* callbacks do not
39
-	 *	block
40
-	 */
41
-	int flags;
27
+/**
28
+ * enum mmu_notifier_event - reason for the mmu notifier callback
29
+ * @MMU_NOTIFY_UNMAP: either munmap() that unmap the range or a mremap() that
30
+ * move the range
31
+ *
32
+ * @MMU_NOTIFY_CLEAR: clear page table entry (many reasons for this like
33
+ * madvise() or replacing a page by another one, ...).
34
+ *
35
+ * @MMU_NOTIFY_PROTECTION_VMA: update is due to protection change for the range
36
+ * ie using the vma access permission (vm_page_prot) to update the whole range
37
+ * is enough no need to inspect changes to the CPU page table (mprotect()
38
+ * syscall)
39
+ *
40
+ * @MMU_NOTIFY_PROTECTION_PAGE: update is due to change in read/write flag for
41
+ * pages in the range so to mirror those changes the user must inspect the CPU
42
+ * page table (from the end callback).
43
+ *
44
+ * @MMU_NOTIFY_SOFT_DIRTY: soft dirty accounting (still same page and same
45
+ * access flags). User should soft dirty the page in the end callback to make
46
+ * sure that anyone relying on soft dirtyness catch pages that might be written
47
+ * through non CPU mappings.
48
+ *
49
+ * @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal
50
+ * that the mm refcount is zero and the range is no longer accessible.
51
+ *
52
+ * @MMU_NOTIFY_MIGRATE: used during migrate_vma_collect() invalidate to signal
53
+ * a device driver to possibly ignore the invalidation if the
54
+ * migrate_pgmap_owner field matches the driver's device private pgmap owner.
55
+ */
56
+enum mmu_notifier_event {
57
+	MMU_NOTIFY_UNMAP = 0,
58
+	MMU_NOTIFY_CLEAR,
59
+	MMU_NOTIFY_PROTECTION_VMA,
60
+	MMU_NOTIFY_PROTECTION_PAGE,
61
+	MMU_NOTIFY_SOFT_DIRTY,
62
+	MMU_NOTIFY_RELEASE,
63
+	MMU_NOTIFY_MIGRATE,
64
+};
4265 
66
+#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
67
+
68
+struct mmu_notifier_ops {
4369 	/*
4470 	 * Called either by mmu_notifier_unregister or when the mm is
4571 	 * being destroyed by exit_mmap, always before all pages are
....@@ -63,7 +89,7 @@
6389 	 * through the gart alias address, so leading to memory
6490 	 * corruption.
6591 	 */
66
-	void (*release)(struct mmu_notifier *mn,
92
+	void (*release)(struct mmu_notifier *subscription,
6793 			struct mm_struct *mm);
6894 
6995 	/*
....@@ -75,7 +101,7 @@
75101 	 * Start-end is necessary in case the secondary MMU is mapping the page
76102 	 * at a smaller granularity than the primary MMU.
77103 	 */
78
-	int (*clear_flush_young)(struct mmu_notifier *mn,
104
+	int (*clear_flush_young)(struct mmu_notifier *subscription,
79105 				 struct mm_struct *mm,
80106 				 unsigned long start,
81107 				 unsigned long end);
....@@ -85,7 +111,7 @@
85111 	 * latter, it is supposed to test-and-clear the young/accessed bitflag
86112 	 * in the secondary pte, but it may omit flushing the secondary tlb.
87113 	 */
88
-	int (*clear_young)(struct mmu_notifier *mn,
114
+	int (*clear_young)(struct mmu_notifier *subscription,
89115 			   struct mm_struct *mm,
90116 			   unsigned long start,
91117 			   unsigned long end);
....@@ -96,7 +122,7 @@
96122 	 * frequently used without actually clearing the flag or tearing
97123 	 * down the secondary mapping on the page.
98124 	 */
99
-	int (*test_young)(struct mmu_notifier *mn,
125
+	int (*test_young)(struct mmu_notifier *subscription,
100126 			  struct mm_struct *mm,
101127 			  unsigned long address);
102128 
....@@ -104,14 +130,14 @@
104130 	 * change_pte is called in cases that pte mapping to page is changed:
105131 	 * for example, when ksm remaps pte to point to a new shared page.
106132 	 */
107
-	void (*change_pte)(struct mmu_notifier *mn,
133
+	void (*change_pte)(struct mmu_notifier *subscription,
108134 			   struct mm_struct *mm,
109135 			   unsigned long address,
110136 			   pte_t pte);
111137 
112138 	/*
113139 	 * invalidate_range_start() and invalidate_range_end() must be
114
-	 * paired and are called only when the mmap_sem and/or the
140
+	 * paired and are called only when the mmap_lock and/or the
115141 	 * locks protecting the reverse maps are held. If the subsystem
116142 	 * can't guarantee that no additional references are taken to
117143 	 * the pages in the range, it has to implement the
....@@ -153,17 +179,16 @@
153179 	 * the last refcount is dropped.
154180 	 *
155181 	 * If blockable argument is set to false then the callback cannot
156
-	 * sleep and has to return with -EAGAIN. 0 should be returned
157
-	 * otherwise.
158
-	 *
182
+	 * sleep and has to return with -EAGAIN if sleeping would be required.
183
+	 * 0 should be returned otherwise. Please note that notifiers that can
184
+	 * fail invalidate_range_start are not allowed to implement
185
+	 * invalidate_range_end, as there is no mechanism for informing the
186
+	 * notifier that its start failed.
159187 	 */
160
-	int (*invalidate_range_start)(struct mmu_notifier *mn,
161
-				       struct mm_struct *mm,
162
-				       unsigned long start, unsigned long end,
163
-				       bool blockable);
164
-	void (*invalidate_range_end)(struct mmu_notifier *mn,
165
-				     struct mm_struct *mm,
166
-				     unsigned long start, unsigned long end);
188
+	int (*invalidate_range_start)(struct mmu_notifier *subscription,
189
+				      const struct mmu_notifier_range *range);
190
+	void (*invalidate_range_end)(struct mmu_notifier *subscription,
191
+				     const struct mmu_notifier_range *range);
167192 
168193 	/*
169194 	 * invalidate_range() is either called between
....@@ -182,13 +207,24 @@
182207 	 * Note that this function might be called with just a sub-range
183208 	 * of what was passed to invalidate_range_start()/end(), if
184209 	 * called between those functions.
185
-	 *
186
-	 * If this callback cannot block, and invalidate_range_{start,end}
187
-	 * cannot block, mmu_notifier_ops.flags should have
188
-	 * MMU_INVALIDATE_DOES_NOT_BLOCK set.
189210 	 */
190
-	void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm,
191
-				 unsigned long start, unsigned long end);
211
+	void (*invalidate_range)(struct mmu_notifier *subscription,
212
+				 struct mm_struct *mm,
213
+				 unsigned long start,
214
+				 unsigned long end);
215
+
216
+	/*
217
+	 * These callbacks are used with the get/put interface to manage the
218
+	 * lifetime of the mmu_notifier memory. alloc_notifier() returns a new
219
+	 * notifier for use with the mm.
220
+	 *
221
+	 * free_notifier() is only called after the mmu_notifier has been
222
+	 * fully put, calls to any ops callback are prevented and no ops
223
+	 * callbacks are currently running. It is called from a SRCU callback
224
+	 * and cannot sleep.
225
+	 */
226
+	struct mmu_notifier *(*alloc_notifier)(struct mm_struct *mm);
227
+	void (*free_notifier)(struct mmu_notifier *subscription);
192228 
193229 	ANDROID_KABI_RESERVE(1);
194230 	ANDROID_KABI_RESERVE(2);
....@@ -197,38 +233,190 @@
197233 };
198234 
199235 /*
200
- * The notifier chains are protected by mmap_sem and/or the reverse map
236
+ * The notifier chains are protected by mmap_lock and/or the reverse map
201237  * semaphores. Notifier chains are only changed when all reverse maps and
202
- * the mmap_sem locks are taken.
238
+ * the mmap_lock locks are taken.
203239  *
204240  * Therefore notifier chains can only be traversed when either
205241  *
206
- * 1. mmap_sem is held.
242
+ * 1. mmap_lock is held.
207243  * 2. One of the reverse map locks is held (i_mmap_rwsem or anon_vma->rwsem).
208244  * 3. No other concurrent thread can access the list (release)
209245  */
210246 struct mmu_notifier {
211247 	struct hlist_node hlist;
212248 	const struct mmu_notifier_ops *ops;
249
+	struct mm_struct *mm;
250
+	struct rcu_head rcu;
251
+	unsigned int users;
213252 
214253 	ANDROID_KABI_RESERVE(1);
215254 	ANDROID_KABI_RESERVE(2);
216255 };
217256 
218
-static inline int mm_has_notifiers(struct mm_struct *mm)
257
+/**
258
+ * struct mmu_interval_notifier_ops
259
+ * @invalidate: Upon return the caller must stop using any SPTEs within this
260
+ *              range. This function can sleep. Return false only if sleeping
261
+ *              was required but mmu_notifier_range_blockable(range) is false.
262
+ */
263
+struct mmu_interval_notifier_ops {
264
+	bool (*invalidate)(struct mmu_interval_notifier *interval_sub,
265
+			   const struct mmu_notifier_range *range,
266
+			   unsigned long cur_seq);
267
+};
268
+
269
+struct mmu_interval_notifier {
270
+	struct interval_tree_node interval_tree;
271
+	const struct mmu_interval_notifier_ops *ops;
272
+	struct mm_struct *mm;
273
+	struct hlist_node deferred_item;
274
+	unsigned long invalidate_seq;
275
+};
276
+
277
+#ifdef CONFIG_MMU_NOTIFIER
278
+
279
+#ifdef CONFIG_LOCKDEP
280
+extern struct lockdep_map __mmu_notifier_invalidate_range_start_map;
281
+#endif
282
+
283
+struct mmu_notifier_range {
284
+	struct vm_area_struct *vma;
285
+	struct mm_struct *mm;
286
+	unsigned long start;
287
+	unsigned long end;
288
+	unsigned flags;
289
+	enum mmu_notifier_event event;
290
+	void *migrate_pgmap_owner;
291
+};
292
+
293
+static inline
294
+struct mmu_notifier_subscriptions_hdr *get_notifier_subscriptions_hdr(
295
+							struct mm_struct *mm)
219296 {
220
-	return unlikely(mm->mmu_notifier_mm);
297
+	/*
298
+	 * container_of() can't be used here because mmu_notifier_subscriptions
299
+	 * struct should be kept invisible to mm_struct, otherwise it
300
+	 * introduces KMI CRC breakage. Therefore the callers don't know what
301
+	 * members struct mmu_notifier_subscriptions contains and can't call
302
+	 * container_of(), which requires a member name.
303
+	 *
304
+	 * WARNING: For this typecasting to work, mmu_notifier_subscriptions_hdr
305
+	 * should be the first member of struct mmu_notifier_subscriptions.
306
+	 */
307
+	return (struct mmu_notifier_subscriptions_hdr *)mm->notifier_subscriptions;
221308 }
222309 
223
-extern int mmu_notifier_register(struct mmu_notifier *mn,
310
+static inline int mm_has_notifiers(struct mm_struct *mm)
311
+{
312
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
313
+	return unlikely(get_notifier_subscriptions_hdr(mm)->valid);
314
+#else
315
+	return unlikely(mm->notifier_subscriptions);
316
+#endif
317
+}
318
+
319
+struct mmu_notifier *mmu_notifier_get_locked(const struct mmu_notifier_ops *ops,
320
+					     struct mm_struct *mm);
321
+static inline struct mmu_notifier *
322
+mmu_notifier_get(const struct mmu_notifier_ops *ops, struct mm_struct *mm)
323
+{
324
+	struct mmu_notifier *ret;
325
+
326
+	mmap_write_lock(mm);
327
+	ret = mmu_notifier_get_locked(ops, mm);
328
+	mmap_write_unlock(mm);
329
+	return ret;
330
+}
331
+void mmu_notifier_put(struct mmu_notifier *subscription);
332
+void mmu_notifier_synchronize(void);
333
+
334
+extern int mmu_notifier_register(struct mmu_notifier *subscription,
224335 				 struct mm_struct *mm);
225
-extern int __mmu_notifier_register(struct mmu_notifier *mn,
336
+extern int __mmu_notifier_register(struct mmu_notifier *subscription,
226337 				   struct mm_struct *mm);
227
-extern void mmu_notifier_unregister(struct mmu_notifier *mn,
338
+extern void mmu_notifier_unregister(struct mmu_notifier *subscription,
228339 				    struct mm_struct *mm);
229
-extern void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
230
-					       struct mm_struct *mm);
231
-extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
340
+
341
+unsigned long
342
+mmu_interval_read_begin(struct mmu_interval_notifier *interval_sub);
343
+int mmu_interval_notifier_insert(struct mmu_interval_notifier *interval_sub,
344
+				 struct mm_struct *mm, unsigned long start,
345
+				 unsigned long length,
346
+				 const struct mmu_interval_notifier_ops *ops);
347
+int mmu_interval_notifier_insert_locked(
348
+	struct mmu_interval_notifier *interval_sub, struct mm_struct *mm,
349
+	unsigned long start, unsigned long length,
350
+	const struct mmu_interval_notifier_ops *ops);
351
+void mmu_interval_notifier_remove(struct mmu_interval_notifier *interval_sub);
352
+
353
+/**
354
+ * mmu_interval_set_seq - Save the invalidation sequence
355
+ * @interval_sub - The subscription passed to invalidate
356
+ * @cur_seq - The cur_seq passed to the invalidate() callback
357
+ *
358
+ * This must be called unconditionally from the invalidate callback of a
359
+ * struct mmu_interval_notifier_ops under the same lock that is used to call
360
+ * mmu_interval_read_retry(). It updates the sequence number for later use by
361
+ * mmu_interval_read_retry(). The provided cur_seq will always be odd.
362
+ *
363
+ * If the caller does not call mmu_interval_read_begin() or
364
+ * mmu_interval_read_retry() then this call is not required.
365
+ */
366
+static inline void
367
+mmu_interval_set_seq(struct mmu_interval_notifier *interval_sub,
368
+		     unsigned long cur_seq)
369
+{
370
+	WRITE_ONCE(interval_sub->invalidate_seq, cur_seq);
371
+}
372
+
373
+/**
374
+ * mmu_interval_read_retry - End a read side critical section against a VA range
375
+ * interval_sub: The subscription
376
+ * seq: The return of the paired mmu_interval_read_begin()
377
+ *
378
+ * This MUST be called under a user provided lock that is also held
379
+ * unconditionally by op->invalidate() when it calls mmu_interval_set_seq().
380
+ *
381
+ * Each call should be paired with a single mmu_interval_read_begin() and
382
+ * should be used to conclude the read side.
383
+ *
384
+ * Returns true if an invalidation collided with this critical section, and
385
+ * the caller should retry.
386
+ */
387
+static inline bool
388
+mmu_interval_read_retry(struct mmu_interval_notifier *interval_sub,
389
+			unsigned long seq)
390
+{
391
+	return interval_sub->invalidate_seq != seq;
392
+}
393
+
394
+/**
395
+ * mmu_interval_check_retry - Test if a collision has occurred
396
+ * interval_sub: The subscription
397
+ * seq: The return of the matching mmu_interval_read_begin()
398
+ *
399
+ * This can be used in the critical section between mmu_interval_read_begin()
400
+ * and mmu_interval_read_retry().  A return of true indicates an invalidation
401
+ * has collided with this critical region and a future
402
+ * mmu_interval_read_retry() will return true.
403
+ *
404
+ * False is not reliable and only suggests a collision may not have
405
+ * occured. It can be called many times and does not have to hold the user
406
+ * provided lock.
407
+ *
408
+ * This call can be used as part of loops and other expensive operations to
409
+ * expedite a retry.
410
+ */
411
+static inline bool
412
+mmu_interval_check_retry(struct mmu_interval_notifier *interval_sub,
413
+			 unsigned long seq)
414
+{
415
+	/* Pairs with the WRITE_ONCE in mmu_interval_set_seq() */
416
+	return READ_ONCE(interval_sub->invalidate_seq) != seq;
417
+}
418
+
419
+extern void __mmu_notifier_subscriptions_destroy(struct mm_struct *mm);
232420 extern void __mmu_notifier_release(struct mm_struct *mm);
233421 extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
234422 					  unsigned long start,
....@@ -240,15 +428,19 @@
240428 				     unsigned long address);
241429 extern void __mmu_notifier_change_pte(struct mm_struct *mm,
242430 				      unsigned long address, pte_t pte);
243
-extern int __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
244
-				  unsigned long start, unsigned long end,
245
-				  bool blockable);
246
-extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
247
-				  unsigned long start, unsigned long end,
431
+extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r);
432
+extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r,
248433 				  bool only_end);
249434 extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
250435 				  unsigned long start, unsigned long end);
251
-extern bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm);
436
+extern bool
437
+mmu_notifier_range_update_to_read_only(const struct mmu_notifier_range *range);
438
+
439
+static inline bool
440
+mmu_notifier_range_blockable(const struct mmu_notifier_range *range)
441
+{
442
+	return (range->flags & MMU_NOTIFIER_RANGE_BLOCKABLE);
443
+}
252444 
253445 static inline void mmu_notifier_release(struct mm_struct *mm)
254446 {
....@@ -289,33 +481,48 @@
289481 		__mmu_notifier_change_pte(mm, address, pte);
290482 }
291483 
292
-static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
293
-				  unsigned long start, unsigned long end)
484
+static inline void
485
+mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
294486 {
295
-	if (mm_has_notifiers(mm))
296
-		__mmu_notifier_invalidate_range_start(mm, start, end, true);
487
+	might_sleep();
488
+
489
+	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
490
+	if (mm_has_notifiers(range->mm)) {
491
+		range->flags |= MMU_NOTIFIER_RANGE_BLOCKABLE;
492
+		__mmu_notifier_invalidate_range_start(range);
493
+	}
494
+	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
297495 }
298496 
299
-static inline int mmu_notifier_invalidate_range_start_nonblock(struct mm_struct *mm,
300
-				  unsigned long start, unsigned long end)
497
+static inline int
498
+mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
301499 {
302
-	if (mm_has_notifiers(mm))
303
-		return __mmu_notifier_invalidate_range_start(mm, start, end, false);
304
-	return 0;
500
+	int ret = 0;
501
+
502
+	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
503
+	if (mm_has_notifiers(range->mm)) {
504
+		range->flags &= ~MMU_NOTIFIER_RANGE_BLOCKABLE;
505
+		ret = __mmu_notifier_invalidate_range_start(range);
506
+	}
507
+	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
508
+	return ret;
305509 }
306510 
307
-static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
308
-				  unsigned long start, unsigned long end)
511
+static inline void
512
+mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
309513 {
310
-	if (mm_has_notifiers(mm))
311
-		__mmu_notifier_invalidate_range_end(mm, start, end, false);
514
+	if (mmu_notifier_range_blockable(range))
515
+		might_sleep();
516
+
517
+	if (mm_has_notifiers(range->mm))
518
+		__mmu_notifier_invalidate_range_end(range, false);
312519 }
313520 
314
-static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
315
-				  unsigned long start, unsigned long end)
521
+static inline void
522
+mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
316523 {
317
-	if (mm_has_notifiers(mm))
318
-		__mmu_notifier_invalidate_range_end(mm, start, end, true);
524
+	if (mm_has_notifiers(range->mm))
525
+		__mmu_notifier_invalidate_range_end(range, true);
319526 }
320527 
321528 static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
....@@ -325,15 +532,72 @@
325532 		__mmu_notifier_invalidate_range(mm, start, end);
326533 }
327534 
328
-static inline void mmu_notifier_mm_init(struct mm_struct *mm)
535
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
536
+
537
+extern bool mmu_notifier_subscriptions_init(struct mm_struct *mm);
538
+extern void mmu_notifier_subscriptions_destroy(struct mm_struct *mm);
539
+
540
+static inline bool mmu_notifier_trylock(struct mm_struct *mm)
329541 {
330
-	mm->mmu_notifier_mm = NULL;
542
+	return percpu_down_read_trylock(
543
+		&get_notifier_subscriptions_hdr(mm)->mmu_notifier_lock->rw_sem);
331544 }
332545 
333
-static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
546
+static inline void mmu_notifier_unlock(struct mm_struct *mm)
547
+{
548
+	percpu_up_read(
549
+		&get_notifier_subscriptions_hdr(mm)->mmu_notifier_lock->rw_sem);
550
+}
551
+
552
+#else /* CONFIG_SPECULATIVE_PAGE_FAULT */
553
+
554
+static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)
555
+{
556
+	mm->notifier_subscriptions = NULL;
557
+	return true;
558
+}
559
+
560
+static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
334561 {
335562 	if (mm_has_notifiers(mm))
336
-		__mmu_notifier_mm_destroy(mm);
563
+		__mmu_notifier_subscriptions_destroy(mm);
564
+}
565
+
566
+static inline bool mmu_notifier_trylock(struct mm_struct *mm)
567
+{
568
+	return true;
569
+}
570
+
571
+static inline void mmu_notifier_unlock(struct mm_struct *mm)
572
+{
573
+}
574
+
575
+#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
576
+
577
+static inline void mmu_notifier_range_init(struct mmu_notifier_range *range,
578
+					   enum mmu_notifier_event event,
579
+					   unsigned flags,
580
+					   struct vm_area_struct *vma,
581
+					   struct mm_struct *mm,
582
+					   unsigned long start,
583
+					   unsigned long end)
584
+{
585
+	range->vma = vma;
586
+	range->event = event;
587
+	range->mm = mm;
588
+	range->start = start;
589
+	range->end = end;
590
+	range->flags = flags;
591
+}
592
+
593
+static inline void mmu_notifier_range_init_migrate(
594
+			struct mmu_notifier_range *range, unsigned int flags,
595
+			struct vm_area_struct *vma, struct mm_struct *mm,
596
+			unsigned long start, unsigned long end, void *pgmap)
597
+{
598
+	mmu_notifier_range_init(range, MMU_NOTIFY_MIGRATE, flags, vma, mm,
599
+				start, end);
600
+	range->migrate_pgmap_owner = pgmap;
337601 }
338602 
339603 #define ptep_clear_flush_young_notify(__vma, __address, __ptep)		\
....@@ -443,11 +707,32 @@
443707 	set_pte_at(___mm, ___address, __ptep, ___pte);			\
444708 })
445709 
446
-extern void mmu_notifier_call_srcu(struct rcu_head *rcu,
447
-				   void (*func)(struct rcu_head *rcu));
448
-extern void mmu_notifier_synchronize(void);
449
-
450710 #else /* CONFIG_MMU_NOTIFIER */
711
+
712
+struct mmu_notifier_range {
713
+	unsigned long start;
714
+	unsigned long end;
715
+};
716
+
717
+static inline void _mmu_notifier_range_init(struct mmu_notifier_range *range,
718
+					    unsigned long start,
719
+					    unsigned long end)
720
+{
721
+	range->start = start;
722
+	range->end = end;
723
+}
724
+
725
+#define mmu_notifier_range_init(range,event,flags,vma,mm,start,end)  \
726
+	_mmu_notifier_range_init(range, start, end)
727
+#define mmu_notifier_range_init_migrate(range, flags, vma, mm, start, end, \
728
+					pgmap) \
729
+	_mmu_notifier_range_init(range, start, end)
730
+
731
+static inline bool
732
+mmu_notifier_range_blockable(const struct mmu_notifier_range *range)
733
+{
734
+	return true;
735
+}
451736 
452737 static inline int mm_has_notifiers(struct mm_struct *mm)
453738 {
....@@ -476,24 +761,24 @@
476761 {
477762 }
478763 
479
-static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
480
-				  unsigned long start, unsigned long end)
764
+static inline void
765
+mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
481766 {
482767 }
483768 
484
-static inline int mmu_notifier_invalidate_range_start_nonblock(struct mm_struct *mm,
485
-				  unsigned long start, unsigned long end)
769
+static inline int
770
+mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
486771 {
487772 	return 0;
488773 }
489774 
490
-static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
491
-				  unsigned long start, unsigned long end)
775
+static inline
776
+void mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
492777 {
493778 }
494779 
495
-static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
496
-				  unsigned long start, unsigned long end)
780
+static inline void
781
+mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
497782 {
498783 }
499784 
....@@ -502,18 +787,25 @@
502787 {
503788 }
504789 
505
-static inline bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm)
790
+static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)
506791 {
507
-	return false;
792
+	return true;
508793 }
509794 
510
-static inline void mmu_notifier_mm_init(struct mm_struct *mm)
795
+static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
511796 {
512797 }
513798 
514
-static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
799
+static inline bool mmu_notifier_trylock(struct mm_struct *mm)
800
+{
801
+	return true;
802
+}
803
+
804
+static inline void mmu_notifier_unlock(struct mm_struct *mm)
515805 {
516806 }
807
+
808
+#define mmu_notifier_range_update_to_read_only(r) false
517809 
518810 #define ptep_clear_flush_young_notify ptep_clear_flush_young
519811 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
....@@ -524,6 +816,10 @@
524816 #define pudp_huge_clear_flush_notify pudp_huge_clear_flush
525817 #define set_pte_at_notify set_pte_at
526818 
819
+static inline void mmu_notifier_synchronize(void)
820
+{
821
+}
822
+
527823 #endif /* CONFIG_MMU_NOTIFIER */
528824 
529825 #endif /* _LINUX_MMU_NOTIFIER_H */