forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771 
 kernel/mm/slab.h
....@@ -46,6 +46,7 @@
4646 #include <linux/kmemleak.h>
4747 #include <linux/random.h>
4848 #include <linux/sched/mm.h>
49
+#include <linux/android_vendor.h>
4950 
5051 /*
5152  * State of the slab allocator.
....@@ -76,7 +77,7 @@
7677 
7778 /* A table of kmalloc cache names and sizes */
7879 extern const struct kmalloc_info_struct {
79
-	const char *name;
80
+	const char *name[NR_KMALLOC_TYPES];
8081 	unsigned int size;
8182 } kmalloc_info[];
8283 
....@@ -89,6 +90,28 @@
8990 struct kmem_cache *kmalloc_slab(size_t, gfp_t);
9091 #endif
9192 
93
+gfp_t kmalloc_fix_flags(gfp_t flags);
94
+
95
+#ifdef CONFIG_SLUB
96
+/*
97
+ * Tracking user of a slab.
98
+ */
99
+#define TRACK_ADDRS_COUNT 16
100
+struct track {
101
+	unsigned long addr;	/* Called from address */
102
+#ifdef CONFIG_STACKTRACE
103
+	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
104
+#endif
105
+	int cpu;		/* Was running on cpu */
106
+	int pid;		/* Pid context */
107
+	unsigned long when;	/* When did the operation occur */
108
+#ifdef CONFIG_STACKTRACE
109
+	ANDROID_OEM_DATA(1);
110
+#endif
111
+};
112
+
113
+enum track_item { TRACK_ALLOC, TRACK_FREE };
114
+#endif
92115 
93116 /* Functions provided by the slab allocators */
94117 int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags);
....@@ -109,8 +132,7 @@
109132 		   slab_flags_t flags, void (*ctor)(void *));
110133 
111134 slab_flags_t kmem_cache_flags(unsigned int object_size,
112
-	slab_flags_t flags, const char *name,
113
-	void (*ctor)(void *));
135
+	slab_flags_t flags, const char *name);
114136 #else
115137 static inline struct kmem_cache *
116138 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
....@@ -118,8 +140,7 @@
118140 { return NULL; }
119141 
120142 static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
121
-	slab_flags_t flags, const char *name,
122
-	void (*ctor)(void *))
143
+	slab_flags_t flags, const char *name)
123144 {
124145 	return flags;
125146 }
....@@ -171,7 +192,6 @@
171192 int __kmem_cache_shutdown(struct kmem_cache *);
172193 void __kmem_cache_release(struct kmem_cache *);
173194 int __kmem_cache_shrink(struct kmem_cache *);
174
-void __kmemcg_cache_deactivate(struct kmem_cache *s);
175195 void slab_kmem_cache_release(struct kmem_cache *);
176196 
177197 struct seq_file;
....@@ -204,181 +224,294 @@
204224 void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
205225 int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
206226 
227
+static inline int cache_vmstat_idx(struct kmem_cache *s)
228
+{
229
+	return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
230
+		NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
231
+}
232
+
233
+#ifdef CONFIG_SLUB_DEBUG
234
+#ifdef CONFIG_SLUB_DEBUG_ON
235
+DECLARE_STATIC_KEY_TRUE(slub_debug_enabled);
236
+#else
237
+DECLARE_STATIC_KEY_FALSE(slub_debug_enabled);
238
+#endif
239
+extern void print_tracking(struct kmem_cache *s, void *object);
240
+extern unsigned long get_each_object_track(struct kmem_cache *s,
241
+		struct page *page, enum track_item alloc,
242
+		int (*fn)(const struct kmem_cache *, const void *,
243
+		const struct track *, void *), void *private);
244
+extern slab_flags_t slub_debug;
245
+static inline bool __slub_debug_enabled(void)
246
+{
247
+	return static_branch_unlikely(&slub_debug_enabled);
248
+}
249
+#else
250
+static inline void print_tracking(struct kmem_cache *s, void *object)
251
+{
252
+}
253
+static inline bool __slub_debug_enabled(void)
254
+{
255
+	return false;
256
+}
257
+#ifdef CONFIG_SLUB
258
+static inline unsigned long get_each_object_track(struct kmem_cache *s,
259
+		struct page *page, enum track_item alloc,
260
+		int (*fn)(const struct kmem_cache *, const void *,
261
+		const struct track *, void *), void *private)
262
+{
263
+	return 0;
264
+}
265
+#endif
266
+#endif
267
+
268
+/*
269
+ * Returns true if any of the specified slub_debug flags is enabled for the
270
+ * cache. Use only for flags parsed by setup_slub_debug() as it also enables
271
+ * the static key.
272
+ */
273
+static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)
274
+{
275
+	if (IS_ENABLED(CONFIG_SLUB_DEBUG))
276
+		VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS));
277
+	if (__slub_debug_enabled())
278
+		return s->flags & flags;
279
+	return false;
280
+}
281
+
207282 #ifdef CONFIG_MEMCG_KMEM
208
-
209
-/* List of all root caches. */
210
-extern struct list_head		slab_root_caches;
211
-#define root_caches_node	memcg_params.__root_caches_node
212
-
213
-/*
214
- * Iterate over all memcg caches of the given root cache. The caller must hold
215
- * slab_mutex.
216
- */
217
-#define for_each_memcg_cache(iter, root) \
218
-	list_for_each_entry(iter, &(root)->memcg_params.children, \
219
-			    memcg_params.children_node)
220
-
221
-static inline bool is_root_cache(struct kmem_cache *s)
283
+static inline struct obj_cgroup **page_obj_cgroups(struct page *page)
222284 {
223
-	return !s->memcg_params.root_cache;
285
+	/*
286
+	 * page->mem_cgroup and page->obj_cgroups are sharing the same
287
+	 * space. To distinguish between them in case we don't know for sure
288
+	 * that the page is a slab page (e.g. page_cgroup_ino()), let's
289
+	 * always set the lowest bit of obj_cgroups.
290
+	 */
291
+	return (struct obj_cgroup **)
292
+		((unsigned long)page->obj_cgroups & ~0x1UL);
224293 }
225294 
226
-static inline bool slab_equal_or_root(struct kmem_cache *s,
227
-				      struct kmem_cache *p)
295
+static inline bool page_has_obj_cgroups(struct page *page)
228296 {
229
-	return p == s || p == s->memcg_params.root_cache;
297
+	return ((unsigned long)page->obj_cgroups & 0x1UL);
298
+}
299
+
300
+int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
301
+				 gfp_t gfp);
302
+
303
+static inline void memcg_free_page_obj_cgroups(struct page *page)
304
+{
305
+	kfree(page_obj_cgroups(page));
306
+	page->obj_cgroups = NULL;
307
+}
308
+
309
+static inline size_t obj_full_size(struct kmem_cache *s)
310
+{
311
+	/*
312
+	 * For each accounted object there is an extra space which is used
313
+	 * to store obj_cgroup membership. Charge it too.
314
+	 */
315
+	return s->size + sizeof(struct obj_cgroup *);
230316 }
231317 
232318 /*
233
- * We use suffixes to the name in memcg because we can't have caches
234
- * created in the system with the same name. But when we print them
235
- * locally, better refer to them with the base name
319
+ * Returns false if the allocation should fail.
236320  */
237
-static inline const char *cache_name(struct kmem_cache *s)
321
+static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
322
+					     struct obj_cgroup **objcgp,
323
+					     size_t objects, gfp_t flags)
238324 {
239
-	if (!is_root_cache(s))
240
-		s = s->memcg_params.root_cache;
241
-	return s->name;
325
+	struct obj_cgroup *objcg;
326
+
327
+	if (!memcg_kmem_enabled())
328
+		return true;
329
+
330
+	if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))
331
+		return true;
332
+
333
+	objcg = get_obj_cgroup_from_current();
334
+	if (!objcg)
335
+		return true;
336
+
337
+	if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) {
338
+		obj_cgroup_put(objcg);
339
+		return false;
340
+	}
341
+
342
+	*objcgp = objcg;
343
+	return true;
242344 }
243345 
244
-/*
245
- * Note, we protect with RCU only the memcg_caches array, not per-memcg caches.
246
- * That said the caller must assure the memcg's cache won't go away by either
247
- * taking a css reference to the owner cgroup, or holding the slab_mutex.
248
- */
249
-static inline struct kmem_cache *
250
-cache_from_memcg_idx(struct kmem_cache *s, int idx)
346
+static inline void mod_objcg_state(struct obj_cgroup *objcg,
347
+				   struct pglist_data *pgdat,
348
+				   int idx, int nr)
251349 {
252
-	struct kmem_cache *cachep;
253
-	struct memcg_cache_array *arr;
350
+	struct mem_cgroup *memcg;
351
+	struct lruvec *lruvec;
254352 
255353 	rcu_read_lock();
256
-	arr = rcu_dereference(s->memcg_params.memcg_caches);
257
-
258
-	/*
259
-	 * Make sure we will access the up-to-date value. The code updating
260
-	 * memcg_caches issues a write barrier to match this (see
261
-	 * memcg_create_kmem_cache()).
262
-	 */
263
-	cachep = READ_ONCE(arr->entries[idx]);
354
+	memcg = obj_cgroup_memcg(objcg);
355
+	lruvec = mem_cgroup_lruvec(memcg, pgdat);
356
+	mod_memcg_lruvec_state(lruvec, idx, nr);
264357 	rcu_read_unlock();
265
-
266
-	return cachep;
267358 }
268359 
269
-static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
360
+static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
361
+					      struct obj_cgroup *objcg,
362
+					      gfp_t flags, size_t size,
363
+					      void **p)
270364 {
271
-	if (is_root_cache(s))
272
-		return s;
273
-	return s->memcg_params.root_cache;
365
+	struct page *page;
366
+	unsigned long off;
367
+	size_t i;
368
+
369
+	if (!memcg_kmem_enabled() || !objcg)
370
+		return;
371
+
372
+	for (i = 0; i < size; i++) {
373
+		if (likely(p[i])) {
374
+			page = virt_to_head_page(p[i]);
375
+
376
+			if (!page_has_obj_cgroups(page) &&
377
+			    memcg_alloc_page_obj_cgroups(page, s, flags)) {
378
+				obj_cgroup_uncharge(objcg, obj_full_size(s));
379
+				continue;
380
+			}
381
+
382
+			off = obj_to_index(s, page, p[i]);
383
+			obj_cgroup_get(objcg);
384
+			page_obj_cgroups(page)[off] = objcg;
385
+			mod_objcg_state(objcg, page_pgdat(page),
386
+					cache_vmstat_idx(s), obj_full_size(s));
387
+		} else {
388
+			obj_cgroup_uncharge(objcg, obj_full_size(s));
389
+		}
390
+	}
391
+	obj_cgroup_put(objcg);
274392 }
275393 
276
-static __always_inline int memcg_charge_slab(struct page *page,
277
-					     gfp_t gfp, int order,
278
-					     struct kmem_cache *s)
394
+static inline void memcg_slab_free_hook(struct kmem_cache *s_orig,
395
+					void **p, int objects)
279396 {
280
-	if (!memcg_kmem_enabled())
281
-		return 0;
282
-	if (is_root_cache(s))
283
-		return 0;
284
-	return memcg_kmem_charge_memcg(page, gfp, order, s->memcg_params.memcg);
285
-}
397
+	struct kmem_cache *s;
398
+	struct obj_cgroup *objcg;
399
+	struct page *page;
400
+	unsigned int off;
401
+	int i;
286402 
287
-static __always_inline void memcg_uncharge_slab(struct page *page, int order,
288
-						struct kmem_cache *s)
289
-{
290403 	if (!memcg_kmem_enabled())
291404 		return;
292
-	memcg_kmem_uncharge(page, order);
293
-}
294405 
295
-extern void slab_init_memcg_params(struct kmem_cache *);
296
-extern void memcg_link_cache(struct kmem_cache *s);
297
-extern void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s,
298
-				void (*deact_fn)(struct kmem_cache *));
406
+	for (i = 0; i < objects; i++) {
407
+		if (unlikely(!p[i]))
408
+			continue;
409
+
410
+		page = virt_to_head_page(p[i]);
411
+		if (!page_has_obj_cgroups(page))
412
+			continue;
413
+
414
+		if (!s_orig)
415
+			s = page->slab_cache;
416
+		else
417
+			s = s_orig;
418
+
419
+		off = obj_to_index(s, page, p[i]);
420
+		objcg = page_obj_cgroups(page)[off];
421
+		if (!objcg)
422
+			continue;
423
+
424
+		page_obj_cgroups(page)[off] = NULL;
425
+		obj_cgroup_uncharge(objcg, obj_full_size(s));
426
+		mod_objcg_state(objcg, page_pgdat(page), cache_vmstat_idx(s),
427
+				-obj_full_size(s));
428
+		obj_cgroup_put(objcg);
429
+	}
430
+}
299431 
300432 #else /* CONFIG_MEMCG_KMEM */
301
-
302
-/* If !memcg, all caches are root. */
303
-#define slab_root_caches	slab_caches
304
-#define root_caches_node	list
305
-
306
-#define for_each_memcg_cache(iter, root) \
307
-	for ((void)(iter), (void)(root); 0; )
308
-
309
-static inline bool is_root_cache(struct kmem_cache *s)
433
+static inline bool page_has_obj_cgroups(struct page *page)
310434 {
311
-	return true;
435
+	return false;
312436 }
313437 
314
-static inline bool slab_equal_or_root(struct kmem_cache *s,
315
-				      struct kmem_cache *p)
316
-{
317
-	return true;
318
-}
319
-
320
-static inline const char *cache_name(struct kmem_cache *s)
321
-{
322
-	return s->name;
323
-}
324
-
325
-static inline struct kmem_cache *
326
-cache_from_memcg_idx(struct kmem_cache *s, int idx)
438
+static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
327439 {
328440 	return NULL;
329441 }
330442 
331
-static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
332
-{
333
-	return s;
334
-}
335
-
336
-static inline int memcg_charge_slab(struct page *page, gfp_t gfp, int order,
337
-				    struct kmem_cache *s)
443
+static inline int memcg_alloc_page_obj_cgroups(struct page *page,
444
+					       struct kmem_cache *s, gfp_t gfp)
338445 {
339446 	return 0;
340447 }
341448 
342
-static inline void memcg_uncharge_slab(struct page *page, int order,
343
-				       struct kmem_cache *s)
449
+static inline void memcg_free_page_obj_cgroups(struct page *page)
344450 {
345451 }
346452 
347
-static inline void slab_init_memcg_params(struct kmem_cache *s)
453
+static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
454
+					     struct obj_cgroup **objcgp,
455
+					     size_t objects, gfp_t flags)
456
+{
457
+	return true;
458
+}
459
+
460
+static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
461
+					      struct obj_cgroup *objcg,
462
+					      gfp_t flags, size_t size,
463
+					      void **p)
348464 {
349465 }
350466 
351
-static inline void memcg_link_cache(struct kmem_cache *s)
467
+static inline void memcg_slab_free_hook(struct kmem_cache *s,
468
+					void **p, int objects)
352469 {
353470 }
354
-
355471 #endif /* CONFIG_MEMCG_KMEM */
472
+
473
+static inline struct kmem_cache *virt_to_cache(const void *obj)
474
+{
475
+	struct page *page;
476
+
477
+	page = virt_to_head_page(obj);
478
+	if (WARN_ONCE(!PageSlab(page), "%s: Object is not a Slab page!\n",
479
+					__func__))
480
+		return NULL;
481
+	return page->slab_cache;
482
+}
483
+
484
+static __always_inline void account_slab_page(struct page *page, int order,
485
+					      struct kmem_cache *s)
486
+{
487
+	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
488
+			    PAGE_SIZE << order);
489
+}
490
+
491
+static __always_inline void unaccount_slab_page(struct page *page, int order,
492
+						struct kmem_cache *s)
493
+{
494
+	if (memcg_kmem_enabled())
495
+		memcg_free_page_obj_cgroups(page);
496
+
497
+	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
498
+			    -(PAGE_SIZE << order));
499
+}
356500 
357501 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
358502 {
359503 	struct kmem_cache *cachep;
360
-	struct page *page;
361504 
362
-	/*
363
-	 * When kmemcg is not being used, both assignments should return the
364
-	 * same value. but we don't want to pay the assignment price in that
365
-	 * case. If it is not compiled in, the compiler should be smart enough
366
-	 * to not do even the assignment. In that case, slab_equal_or_root
367
-	 * will also be a constant.
368
-	 */
369
-	if (!memcg_kmem_enabled() &&
370
-	    !unlikely(s->flags & SLAB_CONSISTENCY_CHECKS))
505
+	if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
506
+	    !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
371507 		return s;
372508 
373
-	page = virt_to_head_page(x);
374
-	cachep = page->slab_cache;
375
-	if (slab_equal_or_root(cachep, s))
376
-		return cachep;
377
-
378
-	pr_err("%s: Wrong slab cache. %s but object is from %s\n",
379
-	       __func__, s->name, cachep->name);
380
-	WARN_ON_ONCE(1);
381
-	return s;
509
+	cachep = virt_to_cache(x);
510
+	if (WARN(cachep && cachep != s,
511
+		  "%s: Wrong slab cache. %s but object is from %s\n",
512
+		  __func__, s->name, cachep->name))
513
+		print_tracking(cachep, x);
514
+	return cachep;
382515 }
383516 
384517 static inline size_t slab_ksize(const struct kmem_cache *s)
....@@ -412,7 +545,8 @@
412545 }
413546 
414547 static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
415
-						     gfp_t flags)
548
+						     struct obj_cgroup **objcgp,
549
+						     size_t size, gfp_t flags)
416550 {
417551 	flags &= gfp_allowed_mask;
418552 
....@@ -424,27 +558,36 @@
424558 	if (should_failslab(s, flags))
425559 		return NULL;
426560 
427
-	if (memcg_kmem_enabled() &&
428
-	    ((flags & __GFP_ACCOUNT) || (s->flags & SLAB_ACCOUNT)))
429
-		return memcg_kmem_get_cache(s);
561
+	if (!memcg_slab_pre_alloc_hook(s, objcgp, size, flags))
562
+		return NULL;
430563 
431564 	return s;
432565 }
433566 
434
-static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
435
-					size_t size, void **p)
567
+static inline void slab_post_alloc_hook(struct kmem_cache *s,
568
+					struct obj_cgroup *objcg, gfp_t flags,
569
+					size_t size, void **p, bool init)
436570 {
437571 	size_t i;
438572 
439573 	flags &= gfp_allowed_mask;
574
+
575
+	/*
576
+	 * As memory initialization might be integrated into KASAN,
577
+	 * kasan_slab_alloc and initialization memset must be
578
+	 * kept together to avoid discrepancies in behavior.
579
+	 *
580
+	 * As p[i] might get tagged, memset and kmemleak hook come after KASAN.
581
+	 */
440582 	for (i = 0; i < size; i++) {
441
-		p[i] = kasan_slab_alloc(s, p[i], flags);
583
+		p[i] = kasan_slab_alloc(s, p[i], flags, init);
584
+		if (p[i] && init && !kasan_has_integrated_init())
585
+			memset(p[i], 0, s->object_size);
442586 		kmemleak_alloc_recursive(p[i], s->object_size, 1,
443587 					 s->flags, flags);
444588 	}
445589 
446
-	if (memcg_kmem_enabled())
447
-		memcg_kmem_put_cache(s);
590
+	memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
448591 }
449592 
450593 #ifndef CONFIG_SLOB
....@@ -499,9 +642,6 @@
499642 void *slab_start(struct seq_file *m, loff_t *pos);
500643 void *slab_next(struct seq_file *m, void *p, loff_t *pos);
501644 void slab_stop(struct seq_file *m, void *p);
502
-void *memcg_slab_start(struct seq_file *m, loff_t *pos);
503
-void *memcg_slab_next(struct seq_file *m, void *p, loff_t *pos);
504
-void memcg_slab_stop(struct seq_file *m, void *p);
505645 int memcg_slab_show(struct seq_file *m, void *p);
506646 
507647 #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
....@@ -547,4 +687,10 @@
547687 	return false;
548688 }
549689 
690
+#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
691
+void debugfs_slab_release(struct kmem_cache *);
692
+#else
693
+static inline void debugfs_slab_release(struct kmem_cache *s) { }
694
+#endif
695
+
550696 #endif /* MM_SLAB_H */