forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-10-12 a5969cabbb4660eab42b6ef0412cbbd1200cf14d 
 kernel/lib/debugobjects.c
....@@ -19,21 +19,43 @@
1919 #include <linux/slab.h>
2020 #include <linux/hash.h>
2121 #include <linux/kmemleak.h>
22
+#include <linux/cpu.h>
2223 
2324 #define ODEBUG_HASH_BITS	14
2425 #define ODEBUG_HASH_SIZE	(1 << ODEBUG_HASH_BITS)
2526 
2627 #define ODEBUG_POOL_SIZE	1024
2728 #define ODEBUG_POOL_MIN_LEVEL	256
29
+#define ODEBUG_POOL_PERCPU_SIZE	64
30
+#define ODEBUG_BATCH_SIZE	16
2831 
2932 #define ODEBUG_CHUNK_SHIFT	PAGE_SHIFT
3033 #define ODEBUG_CHUNK_SIZE	(1 << ODEBUG_CHUNK_SHIFT)
3134 #define ODEBUG_CHUNK_MASK	(~(ODEBUG_CHUNK_SIZE - 1))
3235 
36
+/*
37
+ * We limit the freeing of debug objects via workqueue at a maximum
38
+ * frequency of 10Hz and about 1024 objects for each freeing operation.
39
+ * So it is freeing at most 10k debug objects per second.
40
+ */
41
+#define ODEBUG_FREE_WORK_MAX	1024
42
+#define ODEBUG_FREE_WORK_DELAY	DIV_ROUND_UP(HZ, 10)
43
+
3344 struct debug_bucket {
3445 	struct hlist_head	list;
3546 	raw_spinlock_t		lock;
3647 };
48
+
49
+/*
50
+ * Debug object percpu free list
51
+ * Access is protected by disabling irq
52
+ */
53
+struct debug_percpu_free {
54
+	struct hlist_head	free_objs;
55
+	int			obj_free;
56
+};
57
+
58
+static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
3759 
3860 static struct debug_bucket	obj_hash[ODEBUG_HASH_SIZE];
3961 
....@@ -44,13 +66,20 @@
4466 static HLIST_HEAD(obj_pool);
4567 static HLIST_HEAD(obj_to_free);
4668 
69
+/*
70
+ * Because of the presence of percpu free pools, obj_pool_free will
71
+ * under-count those in the percpu free pools. Similarly, obj_pool_used
72
+ * will over-count those in the percpu free pools. Adjustments will be
73
+ * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
74
+ * can be off.
75
+ */
4776 static int			obj_pool_min_free = ODEBUG_POOL_SIZE;
4877 static int			obj_pool_free = ODEBUG_POOL_SIZE;
4978 static int			obj_pool_used;
5079 static int			obj_pool_max_used;
80
+static bool			obj_freeing;
5181 /* The number of objs on the global free list */
5282 static int			obj_nr_tofree;
53
-static struct kmem_cache	*obj_cache;
5483 
5584 static int			debug_objects_maxchain __read_mostly;
5685 static int __maybe_unused	debug_objects_maxchecked __read_mostly;
....@@ -62,7 +91,8 @@
6291 				= ODEBUG_POOL_SIZE;
6392 static int			debug_objects_pool_min_level __read_mostly
6493 				= ODEBUG_POOL_MIN_LEVEL;
65
-static struct debug_obj_descr	*descr_test  __read_mostly;
94
+static const struct debug_obj_descr *descr_test  __read_mostly;
95
+static struct kmem_cache	*obj_cache __read_mostly;
6696 
6797 /*
6898  * Track numbers of kmem_cache_alloc()/free() calls done.
....@@ -71,7 +101,7 @@
71101 static int			debug_objects_freed;
72102 
73103 static void free_obj_work(struct work_struct *work);
74
-static DECLARE_WORK(debug_obj_work, free_obj_work);
104
+static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
75105 
76106 static int __init enable_object_debug(char *str)
77107 {
....@@ -99,29 +129,33 @@
99129 
100130 static void fill_pool(void)
101131 {
102
-	gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
103
-	struct debug_obj *new, *obj;
132
+	gfp_t gfp = __GFP_HIGH | __GFP_NOWARN;
133
+	struct debug_obj *obj;
104134 	unsigned long flags;
105135 
106
-	if (likely(obj_pool_free >= debug_objects_pool_min_level))
136
+	if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
107137 		return;
108138 
109139 	/*
110140 	 * Reuse objs from the global free list; they will be reinitialized
111141 	 * when allocating.
142
+	 *
143
+	 * Both obj_nr_tofree and obj_pool_free are checked locklessly; the
144
+	 * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical
145
+	 * sections.
112146 	 */
113
-	while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
147
+	while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) {
114148 		raw_spin_lock_irqsave(&pool_lock, flags);
115149 		/*
116150 		 * Recheck with the lock held as the worker thread might have
117151 		 * won the race and freed the global free list already.
118152 		 */
119
-		if (obj_nr_tofree) {
153
+		while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
120154 			obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
121155 			hlist_del(&obj->node);
122
-			obj_nr_tofree--;
156
+			WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
123157 			hlist_add_head(&obj->node, &obj_pool);
124
-			obj_pool_free++;
158
+			WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
125159 		}
126160 		raw_spin_unlock_irqrestore(&pool_lock, flags);
127161 	}
....@@ -129,16 +163,24 @@
129163 	if (unlikely(!obj_cache))
130164 		return;
131165 
132
-	while (obj_pool_free < debug_objects_pool_min_level) {
166
+	while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
167
+		struct debug_obj *new[ODEBUG_BATCH_SIZE];
168
+		int cnt;
133169 
134
-		new = kmem_cache_zalloc(obj_cache, gfp);
135
-		if (!new)
170
+		for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
171
+			new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
172
+			if (!new[cnt])
173
+				break;
174
+		}
175
+		if (!cnt)
136176 			return;
137177 
138178 		raw_spin_lock_irqsave(&pool_lock, flags);
139
-		hlist_add_head(&new->node, &obj_pool);
140
-		debug_objects_allocated++;
141
-		obj_pool_free++;
179
+		while (cnt) {
180
+			hlist_add_head(&new[--cnt]->node, &obj_pool);
181
+			debug_objects_allocated++;
182
+			WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
183
+		}
142184 		raw_spin_unlock_irqrestore(&pool_lock, flags);
143185 	}
144186 }
....@@ -163,36 +205,77 @@
163205 }
164206 
165207 /*
166
- * Allocate a new object. If the pool is empty, switch off the debugger.
167
- * Must be called with interrupts disabled.
208
+ * Allocate a new object from the hlist
168209  */
169
-static struct debug_obj *
170
-alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
210
+static struct debug_obj *__alloc_object(struct hlist_head *list)
171211 {
172212 	struct debug_obj *obj = NULL;
173213 
174
-	raw_spin_lock(&pool_lock);
175
-	if (obj_pool.first) {
176
-		obj	    = hlist_entry(obj_pool.first, typeof(*obj), node);
177
-
178
-		obj->object = addr;
179
-		obj->descr  = descr;
180
-		obj->state  = ODEBUG_STATE_NONE;
181
-		obj->astate = 0;
214
+	if (list->first) {
215
+		obj = hlist_entry(list->first, typeof(*obj), node);
182216 		hlist_del(&obj->node);
217
+	}
183218 
184
-		hlist_add_head(&obj->node, &b->list);
219
+	return obj;
220
+}
185221 
222
+static struct debug_obj *
223
+alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
224
+{
225
+	struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
226
+	struct debug_obj *obj;
227
+
228
+	if (likely(obj_cache)) {
229
+		obj = __alloc_object(&percpu_pool->free_objs);
230
+		if (obj) {
231
+			percpu_pool->obj_free--;
232
+			goto init_obj;
233
+		}
234
+	}
235
+
236
+	raw_spin_lock(&pool_lock);
237
+	obj = __alloc_object(&obj_pool);
238
+	if (obj) {
186239 		obj_pool_used++;
240
+		WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
241
+
242
+		/*
243
+		 * Looking ahead, allocate one batch of debug objects and
244
+		 * put them into the percpu free pool.
245
+		 */
246
+		if (likely(obj_cache)) {
247
+			int i;
248
+
249
+			for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
250
+				struct debug_obj *obj2;
251
+
252
+				obj2 = __alloc_object(&obj_pool);
253
+				if (!obj2)
254
+					break;
255
+				hlist_add_head(&obj2->node,
256
+					       &percpu_pool->free_objs);
257
+				percpu_pool->obj_free++;
258
+				obj_pool_used++;
259
+				WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
260
+			}
261
+		}
262
+
187263 		if (obj_pool_used > obj_pool_max_used)
188264 			obj_pool_max_used = obj_pool_used;
189265 
190
-		obj_pool_free--;
191266 		if (obj_pool_free < obj_pool_min_free)
192267 			obj_pool_min_free = obj_pool_free;
193268 	}
194269 	raw_spin_unlock(&pool_lock);
195270 
271
+init_obj:
272
+	if (obj) {
273
+		obj->object = addr;
274
+		obj->descr  = descr;
275
+		obj->state  = ODEBUG_STATE_NONE;
276
+		obj->astate = 0;
277
+		hlist_add_head(&obj->node, &b->list);
278
+	}
196279 	return obj;
197280 }
198281 
....@@ -209,22 +292,31 @@
209292 	unsigned long flags;
210293 	HLIST_HEAD(tofree);
211294 
295
+	WRITE_ONCE(obj_freeing, false);
212296 	if (!raw_spin_trylock_irqsave(&pool_lock, flags))
213297 		return;
298
+
299
+	if (obj_pool_free >= debug_objects_pool_size)
300
+		goto free_objs;
214301 
215302 	/*
216303 	 * The objs on the pool list might be allocated before the work is
217304 	 * run, so recheck if pool list it full or not, if not fill pool
218
-	 * list from the global free list
305
+	 * list from the global free list. As it is likely that a workload
306
+	 * may be gearing up to use more and more objects, don't free any
307
+	 * of them until the next round.
219308 	 */
220309 	while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
221310 		obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
222311 		hlist_del(&obj->node);
223312 		hlist_add_head(&obj->node, &obj_pool);
224
-		obj_pool_free++;
225
-		obj_nr_tofree--;
313
+		WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
314
+		WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
226315 	}
316
+	raw_spin_unlock_irqrestore(&pool_lock, flags);
317
+	return;
227318 
319
+free_objs:
228320 	/*
229321 	 * Pool list is already full and there are still objs on the free
230322 	 * list. Move remaining free objs to a temporary list to free the
....@@ -233,7 +325,7 @@
233325 	if (obj_nr_tofree) {
234326 		hlist_move_list(&obj_to_free, &tofree);
235327 		debug_objects_freed += obj_nr_tofree;
236
-		obj_nr_tofree = 0;
328
+		WRITE_ONCE(obj_nr_tofree, 0);
237329 	}
238330 	raw_spin_unlock_irqrestore(&pool_lock, flags);
239331 
....@@ -243,24 +335,86 @@
243335 	}
244336 }
245337 
246
-static bool __free_object(struct debug_obj *obj)
338
+static void __free_object(struct debug_obj *obj)
247339 {
340
+	struct debug_obj *objs[ODEBUG_BATCH_SIZE];
341
+	struct debug_percpu_free *percpu_pool;
342
+	int lookahead_count = 0;
248343 	unsigned long flags;
249344 	bool work;
250345 
251
-	raw_spin_lock_irqsave(&pool_lock, flags);
252
-	work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
346
+	local_irq_save(flags);
347
+	if (!obj_cache)
348
+		goto free_to_obj_pool;
349
+
350
+	/*
351
+	 * Try to free it into the percpu pool first.
352
+	 */
353
+	percpu_pool = this_cpu_ptr(&percpu_obj_pool);
354
+	if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
355
+		hlist_add_head(&obj->node, &percpu_pool->free_objs);
356
+		percpu_pool->obj_free++;
357
+		local_irq_restore(flags);
358
+		return;
359
+	}
360
+
361
+	/*
362
+	 * As the percpu pool is full, look ahead and pull out a batch
363
+	 * of objects from the percpu pool and free them as well.
364
+	 */
365
+	for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
366
+		objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
367
+		if (!objs[lookahead_count])
368
+			break;
369
+		percpu_pool->obj_free--;
370
+	}
371
+
372
+free_to_obj_pool:
373
+	raw_spin_lock(&pool_lock);
374
+	work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
375
+	       (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
253376 	obj_pool_used--;
254377 
255378 	if (work) {
256
-		obj_nr_tofree++;
379
+		WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
257380 		hlist_add_head(&obj->node, &obj_to_free);
381
+		if (lookahead_count) {
382
+			WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
383
+			obj_pool_used -= lookahead_count;
384
+			while (lookahead_count) {
385
+				hlist_add_head(&objs[--lookahead_count]->node,
386
+					       &obj_to_free);
387
+			}
388
+		}
389
+
390
+		if ((obj_pool_free > debug_objects_pool_size) &&
391
+		    (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
392
+			int i;
393
+
394
+			/*
395
+			 * Free one more batch of objects from obj_pool.
396
+			 */
397
+			for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
398
+				obj = __alloc_object(&obj_pool);
399
+				hlist_add_head(&obj->node, &obj_to_free);
400
+				WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
401
+				WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
402
+			}
403
+		}
258404 	} else {
259
-		obj_pool_free++;
405
+		WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
260406 		hlist_add_head(&obj->node, &obj_pool);
407
+		if (lookahead_count) {
408
+			WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
409
+			obj_pool_used -= lookahead_count;
410
+			while (lookahead_count) {
411
+				hlist_add_head(&objs[--lookahead_count]->node,
412
+					       &obj_pool);
413
+			}
414
+		}
261415 	}
262
-	raw_spin_unlock_irqrestore(&pool_lock, flags);
263
-	return work;
416
+	raw_spin_unlock(&pool_lock);
417
+	local_irq_restore(flags);
264418 }
265419 
266420 /*
....@@ -269,9 +423,38 @@
269423  */
270424 static void free_object(struct debug_obj *obj)
271425 {
272
-	if (__free_object(obj))
273
-		schedule_work(&debug_obj_work);
426
+	__free_object(obj);
427
+	if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
428
+		WRITE_ONCE(obj_freeing, true);
429
+		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
430
+	}
274431 }
432
+
433
+#ifdef CONFIG_HOTPLUG_CPU
434
+static int object_cpu_offline(unsigned int cpu)
435
+{
436
+	struct debug_percpu_free *percpu_pool;
437
+	struct hlist_node *tmp;
438
+	struct debug_obj *obj;
439
+	unsigned long flags;
440
+
441
+	/* Remote access is safe as the CPU is dead already */
442
+	percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu);
443
+	hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) {
444
+		hlist_del(&obj->node);
445
+		kmem_cache_free(obj_cache, obj);
446
+	}
447
+
448
+	raw_spin_lock_irqsave(&pool_lock, flags);
449
+	obj_pool_used -= percpu_pool->obj_free;
450
+	debug_objects_freed += percpu_pool->obj_free;
451
+	raw_spin_unlock_irqrestore(&pool_lock, flags);
452
+
453
+	percpu_pool->obj_free = 0;
454
+
455
+	return 0;
456
+}
457
+#endif
275458 
276459 /*
277460  * We run out of memory. That means we probably have tons of objects
....@@ -315,8 +498,17 @@
315498 
316499 static void debug_print_object(struct debug_obj *obj, char *msg)
317500 {
318
-	struct debug_obj_descr *descr = obj->descr;
501
+	const struct debug_obj_descr *descr = obj->descr;
319502 	static int limit;
503
+
504
+	/*
505
+	 * Don't report if lookup_object_or_alloc() by the current thread
506
+	 * failed because lookup_object_or_alloc()/debug_objects_oom() by a
507
+	 * concurrent thread turned off debug_objects_enabled and cleared
508
+	 * the hash buckets.
509
+	 */
510
+	if (!debug_objects_enabled)
511
+		return;
320512 
321513 	if (limit < 5 && descr != descr_test) {
322514 		void *hint = descr->debug_hint ?
....@@ -368,30 +560,74 @@
368560 	WARN_ON(1);
369561 }
370562 
563
+static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket *b,
564
+						const struct debug_obj_descr *descr,
565
+						bool onstack, bool alloc_ifstatic)
566
+{
567
+	struct debug_obj *obj = lookup_object(addr, b);
568
+	enum debug_obj_state state = ODEBUG_STATE_NONE;
569
+
570
+	if (likely(obj))
571
+		return obj;
572
+
573
+	/*
574
+	 * debug_object_init() unconditionally allocates untracked
575
+	 * objects. It does not matter whether it is a static object or
576
+	 * not.
577
+	 *
578
+	 * debug_object_assert_init() and debug_object_activate() allow
579
+	 * allocation only if the descriptor callback confirms that the
580
+	 * object is static and considered initialized. For non-static
581
+	 * objects the allocation needs to be done from the fixup callback.
582
+	 */
583
+	if (unlikely(alloc_ifstatic)) {
584
+		if (!descr->is_static_object || !descr->is_static_object(addr))
585
+			return ERR_PTR(-ENOENT);
586
+		/* Statically allocated objects are considered initialized */
587
+		state = ODEBUG_STATE_INIT;
588
+	}
589
+
590
+	obj = alloc_object(addr, b, descr);
591
+	if (likely(obj)) {
592
+		obj->state = state;
593
+		debug_object_is_on_stack(addr, onstack);
594
+		return obj;
595
+	}
596
+
597
+	/* Out of memory. Do the cleanup outside of the locked region */
598
+	debug_objects_enabled = 0;
599
+	return NULL;
600
+}
601
+
602
+static void debug_objects_fill_pool(void)
603
+{
604
+	/*
605
+	 * On RT enabled kernels the pool refill must happen in preemptible
606
+	 * context:
607
+	 */
608
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible())
609
+		fill_pool();
610
+}
611
+
371612 static void
372
-__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
613
+__debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack)
373614 {
374615 	enum debug_obj_state state;
375616 	struct debug_bucket *db;
376617 	struct debug_obj *obj;
377618 	unsigned long flags;
378619 
379
-	fill_pool();
620
+	debug_objects_fill_pool();
380621 
381622 	db = get_bucket((unsigned long) addr);
382623 
383624 	raw_spin_lock_irqsave(&db->lock, flags);
384625 
385
-	obj = lookup_object(addr, db);
386
-	if (!obj) {
387
-		obj = alloc_object(addr, db, descr);
388
-		if (!obj) {
389
-			debug_objects_enabled = 0;
390
-			raw_spin_unlock_irqrestore(&db->lock, flags);
391
-			debug_objects_oom();
392
-			return;
393
-		}
394
-		debug_object_is_on_stack(addr, onstack);
626
+	obj = lookup_object_or_alloc(addr, db, descr, onstack, false);
627
+	if (unlikely(!obj)) {
628
+		raw_spin_unlock_irqrestore(&db->lock, flags);
629
+		debug_objects_oom();
630
+		return;
395631 	}
396632 
397633 	switch (obj->state) {
....@@ -402,15 +638,16 @@
402638 		break;
403639 
404640 	case ODEBUG_STATE_ACTIVE:
405
-		debug_print_object(obj, "init");
406641 		state = obj->state;
407642 		raw_spin_unlock_irqrestore(&db->lock, flags);
643
+		debug_print_object(obj, "init");
408644 		debug_object_fixup(descr->fixup_init, addr, state);
409645 		return;
410646 
411647 	case ODEBUG_STATE_DESTROYED:
648
+		raw_spin_unlock_irqrestore(&db->lock, flags);
412649 		debug_print_object(obj, "init");
413
-		break;
650
+		return;
414651 	default:
415652 		break;
416653 	}
....@@ -423,7 +660,7 @@
423660  * @addr:	address of the object
424661  * @descr:	pointer to an object specific debug description structure
425662  */
426
-void debug_object_init(void *addr, struct debug_obj_descr *descr)
663
+void debug_object_init(void *addr, const struct debug_obj_descr *descr)
427664 {
428665 	if (!debug_objects_enabled)
429666 		return;
....@@ -438,7 +675,7 @@
438675  * @addr:	address of the object
439676  * @descr:	pointer to an object specific debug description structure
440677  */
441
-void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
678
+void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr)
442679 {
443680 	if (!debug_objects_enabled)
444681 		return;
....@@ -453,26 +690,28 @@
453690  * @descr:	pointer to an object specific debug description structure
454691  * Returns 0 for success, -EINVAL for check failed.
455692  */
456
-int debug_object_activate(void *addr, struct debug_obj_descr *descr)
693
+int debug_object_activate(void *addr, const struct debug_obj_descr *descr)
457694 {
695
+	struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr };
458696 	enum debug_obj_state state;
459697 	struct debug_bucket *db;
460698 	struct debug_obj *obj;
461699 	unsigned long flags;
462700 	int ret;
463
-	struct debug_obj o = { .object = addr,
464
-			       .state = ODEBUG_STATE_NOTAVAILABLE,
465
-			       .descr = descr };
466701 
467702 	if (!debug_objects_enabled)
468703 		return 0;
704
+
705
+	debug_objects_fill_pool();
469706 
470707 	db = get_bucket((unsigned long) addr);
471708 
472709 	raw_spin_lock_irqsave(&db->lock, flags);
473710 
474
-	obj = lookup_object(addr, db);
475
-	if (obj) {
711
+	obj = lookup_object_or_alloc(addr, db, descr, false, true);
712
+	if (likely(!IS_ERR_OR_NULL(obj))) {
713
+		bool print_object = false;
714
+
476715 		switch (obj->state) {
477716 		case ODEBUG_STATE_INIT:
478717 		case ODEBUG_STATE_INACTIVE:
....@@ -481,14 +720,14 @@
481720 			break;
482721 
483722 		case ODEBUG_STATE_ACTIVE:
484
-			debug_print_object(obj, "activate");
485723 			state = obj->state;
486724 			raw_spin_unlock_irqrestore(&db->lock, flags);
725
+			debug_print_object(obj, "activate");
487726 			ret = debug_object_fixup(descr->fixup_activate, addr, state);
488727 			return ret ? 0 : -EINVAL;
489728 
490729 		case ODEBUG_STATE_DESTROYED:
491
-			debug_print_object(obj, "activate");
730
+			print_object = true;
492731 			ret = -EINVAL;
493732 			break;
494733 		default:
....@@ -496,28 +735,23 @@
496735 			break;
497736 		}
498737 		raw_spin_unlock_irqrestore(&db->lock, flags);
738
+		if (print_object)
739
+			debug_print_object(obj, "activate");
499740 		return ret;
500741 	}
501742 
502743 	raw_spin_unlock_irqrestore(&db->lock, flags);
503
-	/*
504
-	 * We are here when a static object is activated. We
505
-	 * let the type specific code confirm whether this is
506
-	 * true or not. if true, we just make sure that the
507
-	 * static object is tracked in the object tracker. If
508
-	 * not, this must be a bug, so we try to fix it up.
509
-	 */
510
-	if (descr->is_static_object && descr->is_static_object(addr)) {
511
-		/* track this static object */
512
-		debug_object_init(addr, descr);
513
-		debug_object_activate(addr, descr);
514
-	} else {
515
-		debug_print_object(&o, "activate");
516
-		ret = debug_object_fixup(descr->fixup_activate, addr,
517
-					ODEBUG_STATE_NOTAVAILABLE);
518
-		return ret ? 0 : -EINVAL;
744
+
745
+	/* If NULL the allocation has hit OOM */
746
+	if (!obj) {
747
+		debug_objects_oom();
748
+		return 0;
519749 	}
520
-	return 0;
750
+
751
+	/* Object is neither static nor tracked. It's not initialized */
752
+	debug_print_object(&o, "activate");
753
+	ret = debug_object_fixup(descr->fixup_activate, addr, ODEBUG_STATE_NOTAVAILABLE);
754
+	return ret ? 0 : -EINVAL;
521755 }
522756 EXPORT_SYMBOL_GPL(debug_object_activate);
523757 
....@@ -526,11 +760,12 @@
526760  * @addr:	address of the object
527761  * @descr:	pointer to an object specific debug description structure
528762  */
529
-void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
763
+void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr)
530764 {
531765 	struct debug_bucket *db;
532766 	struct debug_obj *obj;
533767 	unsigned long flags;
768
+	bool print_object = false;
534769 
535770 	if (!debug_objects_enabled)
536771 		return;
....@@ -548,24 +783,27 @@
548783 			if (!obj->astate)
549784 				obj->state = ODEBUG_STATE_INACTIVE;
550785 			else
551
-				debug_print_object(obj, "deactivate");
786
+				print_object = true;
552787 			break;
553788 
554789 		case ODEBUG_STATE_DESTROYED:
555
-			debug_print_object(obj, "deactivate");
790
+			print_object = true;
556791 			break;
557792 		default:
558793 			break;
559794 		}
560
-	} else {
795
+	}
796
+
797
+	raw_spin_unlock_irqrestore(&db->lock, flags);
798
+	if (!obj) {
561799 		struct debug_obj o = { .object = addr,
562800 				       .state = ODEBUG_STATE_NOTAVAILABLE,
563801 				       .descr = descr };
564802 
565803 		debug_print_object(&o, "deactivate");
804
+	} else if (print_object) {
805
+		debug_print_object(obj, "deactivate");
566806 	}
567
-
568
-	raw_spin_unlock_irqrestore(&db->lock, flags);
569807 }
570808 EXPORT_SYMBOL_GPL(debug_object_deactivate);
571809 
....@@ -574,12 +812,13 @@
574812  * @addr:	address of the object
575813  * @descr:	pointer to an object specific debug description structure
576814  */
577
-void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
815
+void debug_object_destroy(void *addr, const struct debug_obj_descr *descr)
578816 {
579817 	enum debug_obj_state state;
580818 	struct debug_bucket *db;
581819 	struct debug_obj *obj;
582820 	unsigned long flags;
821
+	bool print_object = false;
583822 
584823 	if (!debug_objects_enabled)
585824 		return;
....@@ -599,20 +838,22 @@
599838 		obj->state = ODEBUG_STATE_DESTROYED;
600839 		break;
601840 	case ODEBUG_STATE_ACTIVE:
602
-		debug_print_object(obj, "destroy");
603841 		state = obj->state;
604842 		raw_spin_unlock_irqrestore(&db->lock, flags);
843
+		debug_print_object(obj, "destroy");
605844 		debug_object_fixup(descr->fixup_destroy, addr, state);
606845 		return;
607846 
608847 	case ODEBUG_STATE_DESTROYED:
609
-		debug_print_object(obj, "destroy");
848
+		print_object = true;
610849 		break;
611850 	default:
612851 		break;
613852 	}
614853 out_unlock:
615854 	raw_spin_unlock_irqrestore(&db->lock, flags);
855
+	if (print_object)
856
+		debug_print_object(obj, "destroy");
616857 }
617858 EXPORT_SYMBOL_GPL(debug_object_destroy);
618859 
....@@ -621,7 +862,7 @@
621862  * @addr:	address of the object
622863  * @descr:	pointer to an object specific debug description structure
623864  */
624
-void debug_object_free(void *addr, struct debug_obj_descr *descr)
865
+void debug_object_free(void *addr, const struct debug_obj_descr *descr)
625866 {
626867 	enum debug_obj_state state;
627868 	struct debug_bucket *db;
....@@ -641,9 +882,9 @@
641882 
642883 	switch (obj->state) {
643884 	case ODEBUG_STATE_ACTIVE:
644
-		debug_print_object(obj, "free");
645885 		state = obj->state;
646886 		raw_spin_unlock_irqrestore(&db->lock, flags);
887
+		debug_print_object(obj, "free");
647888 		debug_object_fixup(descr->fixup_free, addr, state);
648889 		return;
649890 	default:
....@@ -662,8 +903,9 @@
662903  * @addr:	address of the object
663904  * @descr:	pointer to an object specific debug description structure
664905  */
665
-void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
906
+void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr)
666907 {
908
+	struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr };
667909 	struct debug_bucket *db;
668910 	struct debug_obj *obj;
669911 	unsigned long flags;
....@@ -671,34 +913,25 @@
671913 	if (!debug_objects_enabled)
672914 		return;
673915 
916
+	debug_objects_fill_pool();
917
+
674918 	db = get_bucket((unsigned long) addr);
675919 
676920 	raw_spin_lock_irqsave(&db->lock, flags);
921
+	obj = lookup_object_or_alloc(addr, db, descr, false, true);
922
+	raw_spin_unlock_irqrestore(&db->lock, flags);
923
+	if (likely(!IS_ERR_OR_NULL(obj)))
924
+		return;
677925 
678
-	obj = lookup_object(addr, db);
926
+	/* If NULL the allocation has hit OOM */
679927 	if (!obj) {
680
-		struct debug_obj o = { .object = addr,
681
-				       .state = ODEBUG_STATE_NOTAVAILABLE,
682
-				       .descr = descr };
683
-
684
-		raw_spin_unlock_irqrestore(&db->lock, flags);
685
-		/*
686
-		 * Maybe the object is static, and we let the type specific
687
-		 * code confirm. Track this static object if true, else invoke
688
-		 * fixup.
689
-		 */
690
-		if (descr->is_static_object && descr->is_static_object(addr)) {
691
-			/* Track this static object */
692
-			debug_object_init(addr, descr);
693
-		} else {
694
-			debug_print_object(&o, "assert_init");
695
-			debug_object_fixup(descr->fixup_assert_init, addr,
696
-					   ODEBUG_STATE_NOTAVAILABLE);
697
-		}
928
+		debug_objects_oom();
698929 		return;
699930 	}
700931 
701
-	raw_spin_unlock_irqrestore(&db->lock, flags);
932
+	/* Object is neither tracked nor static. It's not initialized. */
933
+	debug_print_object(&o, "assert_init");
934
+	debug_object_fixup(descr->fixup_assert_init, addr, ODEBUG_STATE_NOTAVAILABLE);
702935 }
703936 EXPORT_SYMBOL_GPL(debug_object_assert_init);
704937 
....@@ -710,12 +943,13 @@
710943  * @next:	state to move to if expected state is found
711944  */
712945 void
713
-debug_object_active_state(void *addr, struct debug_obj_descr *descr,
946
+debug_object_active_state(void *addr, const struct debug_obj_descr *descr,
714947 			  unsigned int expect, unsigned int next)
715948 {
716949 	struct debug_bucket *db;
717950 	struct debug_obj *obj;
718951 	unsigned long flags;
952
+	bool print_object = false;
719953 
720954 	if (!debug_objects_enabled)
721955 		return;
....@@ -731,22 +965,25 @@
731965 			if (obj->astate == expect)
732966 				obj->astate = next;
733967 			else
734
-				debug_print_object(obj, "active_state");
968
+				print_object = true;
735969 			break;
736970 
737971 		default:
738
-			debug_print_object(obj, "active_state");
972
+			print_object = true;
739973 			break;
740974 		}
741
-	} else {
975
+	}
976
+
977
+	raw_spin_unlock_irqrestore(&db->lock, flags);
978
+	if (!obj) {
742979 		struct debug_obj o = { .object = addr,
743980 				       .state = ODEBUG_STATE_NOTAVAILABLE,
744981 				       .descr = descr };
745982 
746983 		debug_print_object(&o, "active_state");
984
+	} else if (print_object) {
985
+		debug_print_object(obj, "active_state");
747986 	}
748
-
749
-	raw_spin_unlock_irqrestore(&db->lock, flags);
750987 }
751988 EXPORT_SYMBOL_GPL(debug_object_active_state);
752989 
....@@ -754,13 +991,12 @@
754991 static void __debug_check_no_obj_freed(const void *address, unsigned long size)
755992 {
756993 	unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
757
-	struct debug_obj_descr *descr;
994
+	const struct debug_obj_descr *descr;
758995 	enum debug_obj_state state;
759996 	struct debug_bucket *db;
760997 	struct hlist_node *tmp;
761998 	struct debug_obj *obj;
762999 	int cnt, objs_checked = 0;
763
-	bool work = false;
7641000 
7651001 	saddr = (unsigned long) address;
7661002 	eaddr = saddr + size;
....@@ -782,16 +1018,16 @@
7821018 
7831019 			switch (obj->state) {
7841020 			case ODEBUG_STATE_ACTIVE:
785
-				debug_print_object(obj, "free");
7861021 				descr = obj->descr;
7871022 				state = obj->state;
7881023 				raw_spin_unlock_irqrestore(&db->lock, flags);
1024
+				debug_print_object(obj, "free");
7891025 				debug_object_fixup(descr->fixup_free,
7901026 						   (void *) oaddr, state);
7911027 				goto repeat;
7921028 			default:
7931029 				hlist_del(&obj->node);
794
-				work |= __free_object(obj);
1030
+				__free_object(obj);
7951031 				break;
7961032 			}
7971033 		}
....@@ -807,8 +1043,10 @@
8071043 		debug_objects_maxchecked = objs_checked;
8081044 
8091045 	/* Schedule work to actually kmem_cache_free() objects */
810
-	if (work)
811
-		schedule_work(&debug_obj_work);
1046
+	if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
1047
+		WRITE_ONCE(obj_freeing, true);
1048
+		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
1049
+	}
8121050 }
8131051 
8141052 void debug_check_no_obj_freed(const void *address, unsigned long size)
....@@ -822,54 +1060,39 @@
8221060 
8231061 static int debug_stats_show(struct seq_file *m, void *v)
8241062 {
1063
+	int cpu, obj_percpu_free = 0;
1064
+
1065
+	for_each_possible_cpu(cpu)
1066
+		obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
1067
+
8251068 	seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
8261069 	seq_printf(m, "max_checked   :%d\n", debug_objects_maxchecked);
8271070 	seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
8281071 	seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
829
-	seq_printf(m, "pool_free     :%d\n", obj_pool_free);
1072
+	seq_printf(m, "pool_free     :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
1073
+	seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
8301074 	seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
831
-	seq_printf(m, "pool_used     :%d\n", obj_pool_used);
1075
+	seq_printf(m, "pool_used     :%d\n", obj_pool_used - obj_percpu_free);
8321076 	seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
833
-	seq_printf(m, "on_free_list  :%d\n", obj_nr_tofree);
1077
+	seq_printf(m, "on_free_list  :%d\n", READ_ONCE(obj_nr_tofree));
8341078 	seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
8351079 	seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
8361080 	return 0;
8371081 }
838
-
839
-static int debug_stats_open(struct inode *inode, struct file *filp)
840
-{
841
-	return single_open(filp, debug_stats_show, NULL);
842
-}
843
-
844
-static const struct file_operations debug_stats_fops = {
845
-	.open		= debug_stats_open,
846
-	.read		= seq_read,
847
-	.llseek		= seq_lseek,
848
-	.release	= single_release,
849
-};
1082
+DEFINE_SHOW_ATTRIBUTE(debug_stats);
8501083 
8511084 static int __init debug_objects_init_debugfs(void)
8521085 {
853
-	struct dentry *dbgdir, *dbgstats;
1086
+	struct dentry *dbgdir;
8541087 
8551088 	if (!debug_objects_enabled)
8561089 		return 0;
8571090 
8581091 	dbgdir = debugfs_create_dir("debug_objects", NULL);
859
-	if (!dbgdir)
860
-		return -ENOMEM;
8611092 
862
-	dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL,
863
-				       &debug_stats_fops);
864
-	if (!dbgstats)
865
-		goto err;
1093
+	debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
8661094 
8671095 	return 0;
868
-
869
-err:
870
-	debugfs_remove(dbgdir);
871
-
872
-	return -ENOMEM;
8731096 }
8741097 __initcall(debug_objects_init_debugfs);
8751098 
....@@ -886,7 +1109,7 @@
8861109 	unsigned long	dummy2[3];
8871110 };
8881111 
889
-static __initdata struct debug_obj_descr descr_type_test;
1112
+static __initconst const struct debug_obj_descr descr_type_test;
8901113 
8911114 static bool __init is_static_object(void *addr)
8921115 {
....@@ -1011,7 +1234,7 @@
10111234 	return res;
10121235 }
10131236 
1014
-static __initdata struct debug_obj_descr descr_type_test = {
1237
+static __initconst const struct debug_obj_descr descr_type_test = {
10151238 	.name			= "selftest",
10161239 	.is_static_object	= is_static_object,
10171240 	.fixup_init		= fixup_init,
....@@ -1130,12 +1353,13 @@
11301353 		hlist_add_head(&obj->node, &objects);
11311354 	}
11321355 
1356
+	debug_objects_allocated += i;
1357
+
11331358 	/*
1134
-	 * When debug_objects_mem_init() is called we know that only
1135
-	 * one CPU is up, so disabling interrupts is enough
1136
-	 * protection. This avoids the lockdep hell of lock ordering.
1359
+	 * debug_objects_mem_init() is now called early that only one CPU is up
1360
+	 * and interrupts have been disabled, so it is safe to replace the
1361
+	 * active object references.
11371362 	 */
1138
-	local_irq_disable();
11391363 
11401364 	/* Remove the statically allocated objects from the pool */
11411365 	hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
....@@ -1156,7 +1380,6 @@
11561380 			cnt++;
11571381 		}
11581382 	}
1159
-	local_irq_enable();
11601383 
11611384 	pr_debug("%d of %d active objects replaced\n",
11621385 		 cnt, obj_pool_used);
....@@ -1177,8 +1400,19 @@
11771400  */
11781401 void __init debug_objects_mem_init(void)
11791402 {
1403
+	int cpu, extras;
1404
+
11801405 	if (!debug_objects_enabled)
11811406 		return;
1407
+
1408
+	/*
1409
+	 * Initialize the percpu object pools
1410
+	 *
1411
+	 * Initialization is not strictly necessary, but was done for
1412
+	 * completeness.
1413
+	 */
1414
+	for_each_possible_cpu(cpu)
1415
+		INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
11821416 
11831417 	obj_cache = kmem_cache_create("debug_objects_cache",
11841418 				      sizeof (struct debug_obj), 0,
....@@ -1189,13 +1423,20 @@
11891423 		debug_objects_enabled = 0;
11901424 		kmem_cache_destroy(obj_cache);
11911425 		pr_warn("out of memory.\n");
1426
+		return;
11921427 	} else
11931428 		debug_objects_selftest();
1429
+
1430
+#ifdef CONFIG_HOTPLUG_CPU
1431
+	cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
1432
+					object_cpu_offline);
1433
+#endif
11941434 
11951435 	/*
11961436 	 * Increase the thresholds for allocating and freeing objects
11971437 	 * according to the number of possible CPUs available in the system.
11981438 	 */
1199
-	debug_objects_pool_size += num_possible_cpus() * 32;
1200
-	debug_objects_pool_min_level += num_possible_cpus() * 4;
1439
+	extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
1440
+	debug_objects_pool_size += extras;
1441
+	debug_objects_pool_min_level += extras;
12011442 }