disable pwm7

hc

2024-05-10 23fa18eaa71266feff7ba8d83022d9e1cc83c65a

 kernel/include/linux/mmu_notifier.h
..
..
@@ -2,44 +2,70 @@
2
2
 #ifndef _LINUX_MMU_NOTIFIER_H
3
3
 #define _LINUX_MMU_NOTIFIER_H
4
4
 
5
-#include <linux/types.h>
6
5
 #include <linux/list.h>
7
6
 #include <linux/spinlock.h>
8
7
 #include <linux/mm_types.h>
8
+#include <linux/mmap_lock.h>
9
+#include <linux/percpu-rwsem.h>
10
+#include <linux/slab.h>
9
11
 #include <linux/srcu.h>
12
+#include <linux/interval_tree.h>
10
13
 #include <linux/android_kabi.h>
11
14
 
15
+struct mmu_notifier_subscriptions;
12
16
 struct mmu_notifier;
13
-struct mmu_notifier_ops;
17
+struct mmu_notifier_range;
18
+struct mmu_interval_notifier;
14
19
 
15
-/* mmu_notifier_ops flags */
16
-#define MMU_INVALIDATE_DOES_NOT_BLOCK	(0x01)
17
-
18
-#ifdef CONFIG_MMU_NOTIFIER
19
-
20
-/*
21
- * The mmu notifier_mm structure is allocated and installed in
22
- * mm->mmu_notifier_mm inside the mm_take_all_locks() protected
23
- * critical section and it's released only when mm_count reaches zero
24
- * in mmdrop().
25
- */
26
-struct mmu_notifier_mm {
27
-	/* all mmu notifiers registerd in this mm are queued in this list */
28
-	struct hlist_head list;
29
-	/* to serialize the list modifications and hlist_unhashed */
30
-	spinlock_t lock;
20
+struct mmu_notifier_subscriptions_hdr {
21
+	bool valid;
22
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
23
+	struct percpu_rw_semaphore_atomic *mmu_notifier_lock;
24
+#endif
31
25
 };
32
26
 
33
-struct mmu_notifier_ops {
34
-	/*
35
-	 * Flags to specify behavior of callbacks for this MMU notifier.
36
-	 * Used to determine which context an operation may be called.
37
-	 *
38
-	 * MMU_INVALIDATE_DOES_NOT_BLOCK: invalidate_range_* callbacks do not
39
-	 *	block
40
-	 */
41
-	int flags;
27
+/**
28
+ * enum mmu_notifier_event - reason for the mmu notifier callback
29
+ * @MMU_NOTIFY_UNMAP: either munmap() that unmap the range or a mremap() that
30
+ * move the range
31
+ *
32
+ * @MMU_NOTIFY_CLEAR: clear page table entry (many reasons for this like
33
+ * madvise() or replacing a page by another one, ...).
34
+ *
35
+ * @MMU_NOTIFY_PROTECTION_VMA: update is due to protection change for the range
36
+ * ie using the vma access permission (vm_page_prot) to update the whole range
37
+ * is enough no need to inspect changes to the CPU page table (mprotect()
38
+ * syscall)
39
+ *
40
+ * @MMU_NOTIFY_PROTECTION_PAGE: update is due to change in read/write flag for
41
+ * pages in the range so to mirror those changes the user must inspect the CPU
42
+ * page table (from the end callback).
43
+ *
44
+ * @MMU_NOTIFY_SOFT_DIRTY: soft dirty accounting (still same page and same
45
+ * access flags). User should soft dirty the page in the end callback to make
46
+ * sure that anyone relying on soft dirtyness catch pages that might be written
47
+ * through non CPU mappings.
48
+ *
49
+ * @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal
50
+ * that the mm refcount is zero and the range is no longer accessible.
51
+ *
52
+ * @MMU_NOTIFY_MIGRATE: used during migrate_vma_collect() invalidate to signal
53
+ * a device driver to possibly ignore the invalidation if the
54
+ * migrate_pgmap_owner field matches the driver's device private pgmap owner.
55
+ */
56
+enum mmu_notifier_event {
57
+	MMU_NOTIFY_UNMAP = 0,
58
+	MMU_NOTIFY_CLEAR,
59
+	MMU_NOTIFY_PROTECTION_VMA,
60
+	MMU_NOTIFY_PROTECTION_PAGE,
61
+	MMU_NOTIFY_SOFT_DIRTY,
62
+	MMU_NOTIFY_RELEASE,
63
+	MMU_NOTIFY_MIGRATE,
64
+};
42
65
 
66
+#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
67
+
68
+struct mmu_notifier_ops {
43
69
 	/*
44
70
 	 * Called either by mmu_notifier_unregister or when the mm is
45
71
 	 * being destroyed by exit_mmap, always before all pages are
..
..
@@ -63,7 +89,7 @@
63
89
 	 * through the gart alias address, so leading to memory
64
90
 	 * corruption.
65
91
 	 */
66
-	void (*release)(struct mmu_notifier *mn,
92
+	void (*release)(struct mmu_notifier *subscription,
67
93
 			struct mm_struct *mm);
68
94
 
69
95
 	/*
..
..
@@ -75,7 +101,7 @@
75
101
 	 * Start-end is necessary in case the secondary MMU is mapping the page
76
102
 	 * at a smaller granularity than the primary MMU.
77
103
 	 */
78
-	int (*clear_flush_young)(struct mmu_notifier *mn,
104
+	int (*clear_flush_young)(struct mmu_notifier *subscription,
79
105
 				 struct mm_struct *mm,
80
106
 				 unsigned long start,
81
107
 				 unsigned long end);
..
..
@@ -85,7 +111,7 @@
85
111
 	 * latter, it is supposed to test-and-clear the young/accessed bitflag
86
112
 	 * in the secondary pte, but it may omit flushing the secondary tlb.
87
113
 	 */
88
-	int (*clear_young)(struct mmu_notifier *mn,
114
+	int (*clear_young)(struct mmu_notifier *subscription,
89
115
 			   struct mm_struct *mm,
90
116
 			   unsigned long start,
91
117
 			   unsigned long end);
..
..
@@ -96,7 +122,7 @@
96
122
 	 * frequently used without actually clearing the flag or tearing
97
123
 	 * down the secondary mapping on the page.
98
124
 	 */
99
-	int (*test_young)(struct mmu_notifier *mn,
125
+	int (*test_young)(struct mmu_notifier *subscription,
100
126
 			  struct mm_struct *mm,
101
127
 			  unsigned long address);
102
128
 
..
..
@@ -104,14 +130,14 @@
104
130
 	 * change_pte is called in cases that pte mapping to page is changed:
105
131
 	 * for example, when ksm remaps pte to point to a new shared page.
106
132
 	 */
107
-	void (*change_pte)(struct mmu_notifier *mn,
133
+	void (*change_pte)(struct mmu_notifier *subscription,
108
134
 			   struct mm_struct *mm,
109
135
 			   unsigned long address,
110
136
 			   pte_t pte);
111
137
 
112
138
 	/*
113
139
 	 * invalidate_range_start() and invalidate_range_end() must be
114
-	 * paired and are called only when the mmap_sem and/or the
140
+	 * paired and are called only when the mmap_lock and/or the
115
141
 	 * locks protecting the reverse maps are held. If the subsystem
116
142
 	 * can't guarantee that no additional references are taken to
117
143
 	 * the pages in the range, it has to implement the
..
..
@@ -153,17 +179,16 @@
153
179
 	 * the last refcount is dropped.
154
180
 	 *
155
181
 	 * If blockable argument is set to false then the callback cannot
156
-	 * sleep and has to return with -EAGAIN. 0 should be returned
157
-	 * otherwise.
158
-	 *
182
+	 * sleep and has to return with -EAGAIN if sleeping would be required.
183
+	 * 0 should be returned otherwise. Please note that notifiers that can
184
+	 * fail invalidate_range_start are not allowed to implement
185
+	 * invalidate_range_end, as there is no mechanism for informing the
186
+	 * notifier that its start failed.
159
187
 	 */
160
-	int (*invalidate_range_start)(struct mmu_notifier *mn,
161
-				       struct mm_struct *mm,
162
-				       unsigned long start, unsigned long end,
163
-				       bool blockable);
164
-	void (*invalidate_range_end)(struct mmu_notifier *mn,
165
-				     struct mm_struct *mm,
166
-				     unsigned long start, unsigned long end);
188
+	int (*invalidate_range_start)(struct mmu_notifier *subscription,
189
+				      const struct mmu_notifier_range *range);
190
+	void (*invalidate_range_end)(struct mmu_notifier *subscription,
191
+				     const struct mmu_notifier_range *range);
167
192
 
168
193
 	/*
169
194
 	 * invalidate_range() is either called between
..
..
@@ -182,13 +207,24 @@
182
207
 	 * Note that this function might be called with just a sub-range
183
208
 	 * of what was passed to invalidate_range_start()/end(), if
184
209
 	 * called between those functions.
185
-	 *
186
-	 * If this callback cannot block, and invalidate_range_{start,end}
187
-	 * cannot block, mmu_notifier_ops.flags should have
188
-	 * MMU_INVALIDATE_DOES_NOT_BLOCK set.
189
210
 	 */
190
-	void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm,
191
-				 unsigned long start, unsigned long end);
211
+	void (*invalidate_range)(struct mmu_notifier *subscription,
212
+				 struct mm_struct *mm,
213
+				 unsigned long start,
214
+				 unsigned long end);
215
+
216
+	/*
217
+	 * These callbacks are used with the get/put interface to manage the
218
+	 * lifetime of the mmu_notifier memory. alloc_notifier() returns a new
219
+	 * notifier for use with the mm.
220
+	 *
221
+	 * free_notifier() is only called after the mmu_notifier has been
222
+	 * fully put, calls to any ops callback are prevented and no ops
223
+	 * callbacks are currently running. It is called from a SRCU callback
224
+	 * and cannot sleep.
225
+	 */
226
+	struct mmu_notifier *(*alloc_notifier)(struct mm_struct *mm);
227
+	void (*free_notifier)(struct mmu_notifier *subscription);
192
228
 
193
229
 	ANDROID_KABI_RESERVE(1);
194
230
 	ANDROID_KABI_RESERVE(2);
..
..
@@ -197,38 +233,190 @@
197
233
 };
198
234
 
199
235
 /*
200
- * The notifier chains are protected by mmap_sem and/or the reverse map
236
+ * The notifier chains are protected by mmap_lock and/or the reverse map
201
237
  * semaphores. Notifier chains are only changed when all reverse maps and
202
- * the mmap_sem locks are taken.
238
+ * the mmap_lock locks are taken.
203
239
  *
204
240
  * Therefore notifier chains can only be traversed when either
205
241
  *
206
- * 1. mmap_sem is held.
242
+ * 1. mmap_lock is held.
207
243
  * 2. One of the reverse map locks is held (i_mmap_rwsem or anon_vma->rwsem).
208
244
  * 3. No other concurrent thread can access the list (release)
209
245
  */
210
246
 struct mmu_notifier {
211
247
 	struct hlist_node hlist;
212
248
 	const struct mmu_notifier_ops *ops;
249
+	struct mm_struct *mm;
250
+	struct rcu_head rcu;
251
+	unsigned int users;
213
252
 
214
253
 	ANDROID_KABI_RESERVE(1);
215
254
 	ANDROID_KABI_RESERVE(2);
216
255
 };
217
256
 
218
-static inline int mm_has_notifiers(struct mm_struct *mm)
257
+/**
258
+ * struct mmu_interval_notifier_ops
259
+ * @invalidate: Upon return the caller must stop using any SPTEs within this
260
+ *              range. This function can sleep. Return false only if sleeping
261
+ *              was required but mmu_notifier_range_blockable(range) is false.
262
+ */
263
+struct mmu_interval_notifier_ops {
264
+	bool (*invalidate)(struct mmu_interval_notifier *interval_sub,
265
+			   const struct mmu_notifier_range *range,
266
+			   unsigned long cur_seq);
267
+};
268
+
269
+struct mmu_interval_notifier {
270
+	struct interval_tree_node interval_tree;
271
+	const struct mmu_interval_notifier_ops *ops;
272
+	struct mm_struct *mm;
273
+	struct hlist_node deferred_item;
274
+	unsigned long invalidate_seq;
275
+};
276
+
277
+#ifdef CONFIG_MMU_NOTIFIER
278
+
279
+#ifdef CONFIG_LOCKDEP
280
+extern struct lockdep_map __mmu_notifier_invalidate_range_start_map;
281
+#endif
282
+
283
+struct mmu_notifier_range {
284
+	struct vm_area_struct *vma;
285
+	struct mm_struct *mm;
286
+	unsigned long start;
287
+	unsigned long end;
288
+	unsigned flags;
289
+	enum mmu_notifier_event event;
290
+	void *migrate_pgmap_owner;
291
+};
292
+
293
+static inline
294
+struct mmu_notifier_subscriptions_hdr *get_notifier_subscriptions_hdr(
295
+							struct mm_struct *mm)
219
296
 {
220
-	return unlikely(mm->mmu_notifier_mm);
297
+	/*
298
+	 * container_of() can't be used here because mmu_notifier_subscriptions
299
+	 * struct should be kept invisible to mm_struct, otherwise it
300
+	 * introduces KMI CRC breakage. Therefore the callers don't know what
301
+	 * members struct mmu_notifier_subscriptions contains and can't call
302
+	 * container_of(), which requires a member name.
303
+	 *
304
+	 * WARNING: For this typecasting to work, mmu_notifier_subscriptions_hdr
305
+	 * should be the first member of struct mmu_notifier_subscriptions.
306
+	 */
307
+	return (struct mmu_notifier_subscriptions_hdr *)mm->notifier_subscriptions;
221
308
 }
222
309
 
223
-extern int mmu_notifier_register(struct mmu_notifier *mn,
310
+static inline int mm_has_notifiers(struct mm_struct *mm)
311
+{
312
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
313
+	return unlikely(get_notifier_subscriptions_hdr(mm)->valid);
314
+#else
315
+	return unlikely(mm->notifier_subscriptions);
316
+#endif
317
+}
318
+
319
+struct mmu_notifier *mmu_notifier_get_locked(const struct mmu_notifier_ops *ops,
320
+					     struct mm_struct *mm);
321
+static inline struct mmu_notifier *
322
+mmu_notifier_get(const struct mmu_notifier_ops *ops, struct mm_struct *mm)
323
+{
324
+	struct mmu_notifier *ret;
325
+
326
+	mmap_write_lock(mm);
327
+	ret = mmu_notifier_get_locked(ops, mm);
328
+	mmap_write_unlock(mm);
329
+	return ret;
330
+}
331
+void mmu_notifier_put(struct mmu_notifier *subscription);
332
+void mmu_notifier_synchronize(void);
333
+
334
+extern int mmu_notifier_register(struct mmu_notifier *subscription,
224
335
 				 struct mm_struct *mm);
225
-extern int __mmu_notifier_register(struct mmu_notifier *mn,
336
+extern int __mmu_notifier_register(struct mmu_notifier *subscription,
226
337
 				   struct mm_struct *mm);
227
-extern void mmu_notifier_unregister(struct mmu_notifier *mn,
338
+extern void mmu_notifier_unregister(struct mmu_notifier *subscription,
228
339
 				    struct mm_struct *mm);
229
-extern void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
230
-					       struct mm_struct *mm);
231
-extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
340
+
341
+unsigned long
342
+mmu_interval_read_begin(struct mmu_interval_notifier *interval_sub);
343
+int mmu_interval_notifier_insert(struct mmu_interval_notifier *interval_sub,
344
+				 struct mm_struct *mm, unsigned long start,
345
+				 unsigned long length,
346
+				 const struct mmu_interval_notifier_ops *ops);
347
+int mmu_interval_notifier_insert_locked(
348
+	struct mmu_interval_notifier *interval_sub, struct mm_struct *mm,
349
+	unsigned long start, unsigned long length,
350
+	const struct mmu_interval_notifier_ops *ops);
351
+void mmu_interval_notifier_remove(struct mmu_interval_notifier *interval_sub);
352
+
353
+/**
354
+ * mmu_interval_set_seq - Save the invalidation sequence
355
+ * @interval_sub - The subscription passed to invalidate
356
+ * @cur_seq - The cur_seq passed to the invalidate() callback
357
+ *
358
+ * This must be called unconditionally from the invalidate callback of a
359
+ * struct mmu_interval_notifier_ops under the same lock that is used to call
360
+ * mmu_interval_read_retry(). It updates the sequence number for later use by
361
+ * mmu_interval_read_retry(). The provided cur_seq will always be odd.
362
+ *
363
+ * If the caller does not call mmu_interval_read_begin() or
364
+ * mmu_interval_read_retry() then this call is not required.
365
+ */
366
+static inline void
367
+mmu_interval_set_seq(struct mmu_interval_notifier *interval_sub,
368
+		     unsigned long cur_seq)
369
+{
370
+	WRITE_ONCE(interval_sub->invalidate_seq, cur_seq);
371
+}
372
+
373
+/**
374
+ * mmu_interval_read_retry - End a read side critical section against a VA range
375
+ * interval_sub: The subscription
376
+ * seq: The return of the paired mmu_interval_read_begin()
377
+ *
378
+ * This MUST be called under a user provided lock that is also held
379
+ * unconditionally by op->invalidate() when it calls mmu_interval_set_seq().
380
+ *
381
+ * Each call should be paired with a single mmu_interval_read_begin() and
382
+ * should be used to conclude the read side.
383
+ *
384
+ * Returns true if an invalidation collided with this critical section, and
385
+ * the caller should retry.
386
+ */
387
+static inline bool
388
+mmu_interval_read_retry(struct mmu_interval_notifier *interval_sub,
389
+			unsigned long seq)
390
+{
391
+	return interval_sub->invalidate_seq != seq;
392
+}
393
+
394
+/**
395
+ * mmu_interval_check_retry - Test if a collision has occurred
396
+ * interval_sub: The subscription
397
+ * seq: The return of the matching mmu_interval_read_begin()
398
+ *
399
+ * This can be used in the critical section between mmu_interval_read_begin()
400
+ * and mmu_interval_read_retry().  A return of true indicates an invalidation
401
+ * has collided with this critical region and a future
402
+ * mmu_interval_read_retry() will return true.
403
+ *
404
+ * False is not reliable and only suggests a collision may not have
405
+ * occured. It can be called many times and does not have to hold the user
406
+ * provided lock.
407
+ *
408
+ * This call can be used as part of loops and other expensive operations to
409
+ * expedite a retry.
410
+ */
411
+static inline bool
412
+mmu_interval_check_retry(struct mmu_interval_notifier *interval_sub,
413
+			 unsigned long seq)
414
+{
415
+	/* Pairs with the WRITE_ONCE in mmu_interval_set_seq() */
416
+	return READ_ONCE(interval_sub->invalidate_seq) != seq;
417
+}
418
+
419
+extern void __mmu_notifier_subscriptions_destroy(struct mm_struct *mm);
232
420
 extern void __mmu_notifier_release(struct mm_struct *mm);
233
421
 extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
234
422
 					  unsigned long start,
..
..
@@ -240,15 +428,19 @@
240
428
 				     unsigned long address);
241
429
 extern void __mmu_notifier_change_pte(struct mm_struct *mm,
242
430
 				      unsigned long address, pte_t pte);
243
-extern int __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
244
-				  unsigned long start, unsigned long end,
245
-				  bool blockable);
246
-extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
247
-				  unsigned long start, unsigned long end,
431
+extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r);
432
+extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r,
248
433
 				  bool only_end);
249
434
 extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
250
435
 				  unsigned long start, unsigned long end);
251
-extern bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm);
436
+extern bool
437
+mmu_notifier_range_update_to_read_only(const struct mmu_notifier_range *range);
438
+
439
+static inline bool
440
+mmu_notifier_range_blockable(const struct mmu_notifier_range *range)
441
+{
442
+	return (range->flags & MMU_NOTIFIER_RANGE_BLOCKABLE);
443
+}
252
444
 
253
445
 static inline void mmu_notifier_release(struct mm_struct *mm)
254
446
 {
..
..
@@ -289,33 +481,48 @@
289
481
 		__mmu_notifier_change_pte(mm, address, pte);
290
482
 }
291
483
 
292
-static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
293
-				  unsigned long start, unsigned long end)
484
+static inline void
485
+mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
294
486
 {
295
-	if (mm_has_notifiers(mm))
296
-		__mmu_notifier_invalidate_range_start(mm, start, end, true);
487
+	might_sleep();
488
+
489
+	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
490
+	if (mm_has_notifiers(range->mm)) {
491
+		range->flags |= MMU_NOTIFIER_RANGE_BLOCKABLE;
492
+		__mmu_notifier_invalidate_range_start(range);
493
+	}
494
+	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
297
495
 }
298
496
 
299
-static inline int mmu_notifier_invalidate_range_start_nonblock(struct mm_struct *mm,
300
-				  unsigned long start, unsigned long end)
497
+static inline int
498
+mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
301
499
 {
302
-	if (mm_has_notifiers(mm))
303
-		return __mmu_notifier_invalidate_range_start(mm, start, end, false);
304
-	return 0;
500
+	int ret = 0;
501
+
502
+	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
503
+	if (mm_has_notifiers(range->mm)) {
504
+		range->flags &= ~MMU_NOTIFIER_RANGE_BLOCKABLE;
505
+		ret = __mmu_notifier_invalidate_range_start(range);
506
+	}
507
+	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
508
+	return ret;
305
509
 }
306
510
 
307
-static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
308
-				  unsigned long start, unsigned long end)
511
+static inline void
512
+mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
309
513
 {
310
-	if (mm_has_notifiers(mm))
311
-		__mmu_notifier_invalidate_range_end(mm, start, end, false);
514
+	if (mmu_notifier_range_blockable(range))
515
+		might_sleep();
516
+
517
+	if (mm_has_notifiers(range->mm))
518
+		__mmu_notifier_invalidate_range_end(range, false);
312
519
 }
313
520
 
314
-static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
315
-				  unsigned long start, unsigned long end)
521
+static inline void
522
+mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
316
523
 {
317
-	if (mm_has_notifiers(mm))
318
-		__mmu_notifier_invalidate_range_end(mm, start, end, true);
524
+	if (mm_has_notifiers(range->mm))
525
+		__mmu_notifier_invalidate_range_end(range, true);
319
526
 }
320
527
 
321
528
 static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
..
..
@@ -325,15 +532,72 @@
325
532
 		__mmu_notifier_invalidate_range(mm, start, end);
326
533
 }
327
534
 
328
-static inline void mmu_notifier_mm_init(struct mm_struct *mm)
535
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
536
+
537
+extern bool mmu_notifier_subscriptions_init(struct mm_struct *mm);
538
+extern void mmu_notifier_subscriptions_destroy(struct mm_struct *mm);
539
+
540
+static inline bool mmu_notifier_trylock(struct mm_struct *mm)
329
541
 {
330
-	mm->mmu_notifier_mm = NULL;
542
+	return percpu_down_read_trylock(
543
+		&get_notifier_subscriptions_hdr(mm)->mmu_notifier_lock->rw_sem);
331
544
 }
332
545
 
333
-static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
546
+static inline void mmu_notifier_unlock(struct mm_struct *mm)
547
+{
548
+	percpu_up_read(
549
+		&get_notifier_subscriptions_hdr(mm)->mmu_notifier_lock->rw_sem);
550
+}
551
+
552
+#else /* CONFIG_SPECULATIVE_PAGE_FAULT */
553
+
554
+static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)
555
+{
556
+	mm->notifier_subscriptions = NULL;
557
+	return true;
558
+}
559
+
560
+static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
334
561
 {
335
562
 	if (mm_has_notifiers(mm))
336
-		__mmu_notifier_mm_destroy(mm);
563
+		__mmu_notifier_subscriptions_destroy(mm);
564
+}
565
+
566
+static inline bool mmu_notifier_trylock(struct mm_struct *mm)
567
+{
568
+	return true;
569
+}
570
+
571
+static inline void mmu_notifier_unlock(struct mm_struct *mm)
572
+{
573
+}
574
+
575
+#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
576
+
577
+static inline void mmu_notifier_range_init(struct mmu_notifier_range *range,
578
+					   enum mmu_notifier_event event,
579
+					   unsigned flags,
580
+					   struct vm_area_struct *vma,
581
+					   struct mm_struct *mm,
582
+					   unsigned long start,
583
+					   unsigned long end)
584
+{
585
+	range->vma = vma;
586
+	range->event = event;
587
+	range->mm = mm;
588
+	range->start = start;
589
+	range->end = end;
590
+	range->flags = flags;
591
+}
592
+
593
+static inline void mmu_notifier_range_init_migrate(
594
+			struct mmu_notifier_range *range, unsigned int flags,
595
+			struct vm_area_struct *vma, struct mm_struct *mm,
596
+			unsigned long start, unsigned long end, void *pgmap)
597
+{
598
+	mmu_notifier_range_init(range, MMU_NOTIFY_MIGRATE, flags, vma, mm,
599
+				start, end);
600
+	range->migrate_pgmap_owner = pgmap;
337
601
 }
338
602
 
339
603
 #define ptep_clear_flush_young_notify(__vma, __address, __ptep)		\
..
..
@@ -443,11 +707,32 @@
443
707
 	set_pte_at(___mm, ___address, __ptep, ___pte);			\
444
708
 })
445
709
 
446
-extern void mmu_notifier_call_srcu(struct rcu_head *rcu,
447
-				   void (*func)(struct rcu_head *rcu));
448
-extern void mmu_notifier_synchronize(void);
449
-
450
710
 #else /* CONFIG_MMU_NOTIFIER */
711
+
712
+struct mmu_notifier_range {
713
+	unsigned long start;
714
+	unsigned long end;
715
+};
716
+
717
+static inline void _mmu_notifier_range_init(struct mmu_notifier_range *range,
718
+					    unsigned long start,
719
+					    unsigned long end)
720
+{
721
+	range->start = start;
722
+	range->end = end;
723
+}
724
+
725
+#define mmu_notifier_range_init(range,event,flags,vma,mm,start,end)  \
726
+	_mmu_notifier_range_init(range, start, end)
727
+#define mmu_notifier_range_init_migrate(range, flags, vma, mm, start, end, \
728
+					pgmap) \
729
+	_mmu_notifier_range_init(range, start, end)
730
+
731
+static inline bool
732
+mmu_notifier_range_blockable(const struct mmu_notifier_range *range)
733
+{
734
+	return true;
735
+}
451
736
 
452
737
 static inline int mm_has_notifiers(struct mm_struct *mm)
453
738
 {
..
..
@@ -476,24 +761,24 @@
476
761
 {
477
762
 }
478
763
 
479
-static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
480
-				  unsigned long start, unsigned long end)
764
+static inline void
765
+mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
481
766
 {
482
767
 }
483
768
 
484
-static inline int mmu_notifier_invalidate_range_start_nonblock(struct mm_struct *mm,
485
-				  unsigned long start, unsigned long end)
769
+static inline int
770
+mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
486
771
 {
487
772
 	return 0;
488
773
 }
489
774
 
490
-static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
491
-				  unsigned long start, unsigned long end)
775
+static inline
776
+void mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
492
777
 {
493
778
 }
494
779
 
495
-static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
496
-				  unsigned long start, unsigned long end)
780
+static inline void
781
+mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
497
782
 {
498
783
 }
499
784
 
..
..
@@ -502,18 +787,25 @@
502
787
 {
503
788
 }
504
789
 
505
-static inline bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm)
790
+static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)
506
791
 {
507
-	return false;
792
+	return true;
508
793
 }
509
794
 
510
-static inline void mmu_notifier_mm_init(struct mm_struct *mm)
795
+static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
511
796
 {
512
797
 }
513
798
 
514
-static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
799
+static inline bool mmu_notifier_trylock(struct mm_struct *mm)
800
+{
801
+	return true;
802
+}
803
+
804
+static inline void mmu_notifier_unlock(struct mm_struct *mm)
515
805
 {
516
806
 }
807
+
808
+#define mmu_notifier_range_update_to_read_only(r) false
517
809
 
518
810
 #define ptep_clear_flush_young_notify ptep_clear_flush_young
519
811
 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
..
..
@@ -524,6 +816,10 @@
524
816
 #define pudp_huge_clear_flush_notify pudp_huge_clear_flush
525
817
 #define set_pte_at_notify set_pte_at
526
818
 
819
+static inline void mmu_notifier_synchronize(void)
820
+{
821
+}
822
+
527
823
 #endif /* CONFIG_MMU_NOTIFIER */
528
824
 
529
825
 #endif /* _LINUX_MMU_NOTIFIER_H */