hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c
kernel/include/linux/sched/mm.h
....@@ -23,7 +23,7 @@
2323 * will still exist later on and mmget_not_zero() has to be used before
2424 * accessing it.
2525 *
26
- * This is a preferred way to to pin @mm for a longer/unbounded amount
26
+ * This is a preferred way to pin @mm for a longer/unbounded amount
2727 * of time.
2828 *
2929 * Use mmdrop() to release the reference acquired by mmgrab().
....@@ -49,32 +49,16 @@
4949 __mmdrop(mm);
5050 }
5151
52
-void mmdrop(struct mm_struct *mm);
53
-
54
-/*
55
- * This has to be called after a get_task_mm()/mmget_not_zero()
56
- * followed by taking the mmap_sem for writing before modifying the
57
- * vmas or anything the coredump pretends not to change from under it.
58
- *
59
- * It also has to be called when mmgrab() is used in the context of
60
- * the process, but then the mm_count refcount is transferred outside
61
- * the context of the process to run down_write() on that pinned mm.
62
- *
63
- * NOTE: find_extend_vma() called from GUP context is the only place
64
- * that can modify the "mm" (notably the vm_start/end) under mmap_sem
65
- * for reading and outside the context of the process, so it is also
66
- * the only case that holds the mmap_sem for reading that must call
67
- * this function. Generally if the mmap_sem is hold for reading
68
- * there's no need of this check after get_task_mm()/mmget_not_zero().
69
- *
70
- * This function can be obsoleted and the check can be removed, after
71
- * the coredump code will hold the mmap_sem for writing before
72
- * invoking the ->core_dump methods.
73
- */
74
-static inline bool mmget_still_valid(struct mm_struct *mm)
52
+#ifdef CONFIG_PREEMPT_RT
53
+extern void __mmdrop_delayed(struct rcu_head *rhp);
54
+static inline void mmdrop_delayed(struct mm_struct *mm)
7555 {
76
- return likely(!mm->core_state);
56
+ if (atomic_dec_and_test(&mm->mm_count))
57
+ call_rcu(&mm->delayed_drop, __mmdrop_delayed);
7758 }
59
+#else
60
+# define mmdrop_delayed(mm) mmdrop(mm)
61
+#endif
7862
7963 /**
8064 * mmget() - Pin the address space associated with a &struct mm_struct.
....@@ -133,6 +117,14 @@
133117 #endif /* CONFIG_MEMCG */
134118
135119 #ifdef CONFIG_MMU
120
+#ifndef arch_get_mmap_end
121
+#define arch_get_mmap_end(addr) (TASK_SIZE)
122
+#endif
123
+
124
+#ifndef arch_get_mmap_base
125
+#define arch_get_mmap_base(addr, base) (base)
126
+#endif
127
+
136128 extern void arch_pick_mmap_layout(struct mm_struct *mm,
137129 struct rlimit *rlim_stack);
138130 extern unsigned long
....@@ -181,14 +173,18 @@
181173 */
182174 static inline gfp_t current_gfp_context(gfp_t flags)
183175 {
184
- /*
185
- * NOIO implies both NOIO and NOFS and it is a weaker context
186
- * so always make sure it makes precendence
187
- */
188
- if (unlikely(current->flags & PF_MEMALLOC_NOIO))
189
- flags &= ~(__GFP_IO | __GFP_FS);
190
- else if (unlikely(current->flags & PF_MEMALLOC_NOFS))
191
- flags &= ~__GFP_FS;
176
+ unsigned int pflags = READ_ONCE(current->flags);
177
+
178
+ if (unlikely(pflags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS))) {
179
+ /*
180
+ * NOIO implies both NOIO and NOFS and it is a weaker context
181
+ * so always make sure it makes precedence
182
+ */
183
+ if (pflags & PF_MEMALLOC_NOIO)
184
+ flags &= ~(__GFP_IO | __GFP_FS);
185
+ else if (pflags & PF_MEMALLOC_NOFS)
186
+ flags &= ~__GFP_FS;
187
+ }
192188 return flags;
193189 }
194190
....@@ -227,7 +223,7 @@
227223 * @flags: Flags to restore.
228224 *
229225 * Ends the implicit GFP_NOIO scope started by memalloc_noio_save function.
230
- * Always make sure that that the given flags is the return value from the
226
+ * Always make sure that the given flags is the return value from the
231227 * pairing memalloc_noio_save call.
232228 */
233229 static inline void memalloc_noio_restore(unsigned int flags)
....@@ -258,7 +254,7 @@
258254 * @flags: Flags to restore.
259255 *
260256 * Ends the implicit GFP_NOFS scope started by memalloc_nofs_save function.
261
- * Always make sure that that the given flags is the return value from the
257
+ * Always make sure that the given flags is the return value from the
262258 * pairing memalloc_nofs_save call.
263259 */
264260 static inline void memalloc_nofs_restore(unsigned int flags)
....@@ -278,40 +274,63 @@
278274 current->flags = (current->flags & ~PF_MEMALLOC) | flags;
279275 }
280276
277
+#ifdef CONFIG_CMA
278
+static inline unsigned int memalloc_nocma_save(void)
279
+{
280
+ unsigned int flags = current->flags & PF_MEMALLOC_NOCMA;
281
+
282
+ current->flags |= PF_MEMALLOC_NOCMA;
283
+ return flags;
284
+}
285
+
286
+static inline void memalloc_nocma_restore(unsigned int flags)
287
+{
288
+ current->flags = (current->flags & ~PF_MEMALLOC_NOCMA) | flags;
289
+}
290
+#else
291
+static inline unsigned int memalloc_nocma_save(void)
292
+{
293
+ return 0;
294
+}
295
+
296
+static inline void memalloc_nocma_restore(unsigned int flags)
297
+{
298
+}
299
+#endif
300
+
281301 #ifdef CONFIG_MEMCG
302
+DECLARE_PER_CPU(struct mem_cgroup *, int_active_memcg);
282303 /**
283
- * memalloc_use_memcg - Starts the remote memcg charging scope.
304
+ * set_active_memcg - Starts the remote memcg charging scope.
284305 * @memcg: memcg to charge.
285306 *
286307 * This function marks the beginning of the remote memcg charging scope. All the
287308 * __GFP_ACCOUNT allocations till the end of the scope will be charged to the
288309 * given memcg.
289310 *
290
- * NOTE: This function is not nesting safe.
311
+ * NOTE: This function can nest. Users must save the return value and
312
+ * reset the previous value after their own charging scope is over.
291313 */
292
-static inline void memalloc_use_memcg(struct mem_cgroup *memcg)
314
+static inline struct mem_cgroup *
315
+set_active_memcg(struct mem_cgroup *memcg)
293316 {
294
- WARN_ON_ONCE(current->active_memcg);
295
- current->active_memcg = memcg;
296
-}
317
+ struct mem_cgroup *old;
297318
298
-/**
299
- * memalloc_unuse_memcg - Ends the remote memcg charging scope.
300
- *
301
- * This function marks the end of the remote memcg charging scope started by
302
- * memalloc_use_memcg().
303
- */
304
-static inline void memalloc_unuse_memcg(void)
305
-{
306
- current->active_memcg = NULL;
319
+ if (in_interrupt()) {
320
+ old = this_cpu_read(int_active_memcg);
321
+ this_cpu_write(int_active_memcg, memcg);
322
+ } else {
323
+ old = current->active_memcg;
324
+ current->active_memcg = memcg;
325
+ }
326
+
327
+ return old;
307328 }
308329 #else
309
-static inline void memalloc_use_memcg(struct mem_cgroup *memcg)
330
+static inline struct mem_cgroup *
331
+set_active_memcg(struct mem_cgroup *memcg)
310332 {
311
-}
312
-
313
-static inline void memalloc_unuse_memcg(void)
314
-{
333
+ return NULL;
315334 }
316335 #endif
317336
....@@ -323,10 +342,13 @@
323342 MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3),
324343 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = (1U << 4),
325344 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = (1U << 5),
345
+ MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY = (1U << 6),
346
+ MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ = (1U << 7),
326347 };
327348
328349 enum {
329350 MEMBARRIER_FLAG_SYNC_CORE = (1U << 0),
351
+ MEMBARRIER_FLAG_RSEQ = (1U << 1),
330352 };
331353
332354 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
....@@ -343,10 +365,8 @@
343365 sync_core_before_usermode();
344366 }
345367
346
-static inline void membarrier_execve(struct task_struct *t)
347
-{
348
- atomic_set(&t->mm->membarrier_state, 0);
349
-}
368
+extern void membarrier_exec_mmap(struct mm_struct *mm);
369
+
350370 #else
351371 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
352372 static inline void membarrier_arch_switch_mm(struct mm_struct *prev,
....@@ -355,7 +375,7 @@
355375 {
356376 }
357377 #endif
358
-static inline void membarrier_execve(struct task_struct *t)
378
+static inline void membarrier_exec_mmap(struct mm_struct *mm)
359379 {
360380 }
361381 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm)