forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-06 08f87f769b595151be1afeff53e144f543faa614 
 kernel/lib/debugobjects.c
....@@ -19,21 +19,43 @@
1919 #include <linux/slab.h>
2020 #include <linux/hash.h>
2121 #include <linux/kmemleak.h>
22
+#include <linux/cpu.h>
2223 
2324 #define ODEBUG_HASH_BITS	14
2425 #define ODEBUG_HASH_SIZE	(1 << ODEBUG_HASH_BITS)
2526 
2627 #define ODEBUG_POOL_SIZE	1024
2728 #define ODEBUG_POOL_MIN_LEVEL	256
29
+#define ODEBUG_POOL_PERCPU_SIZE	64
30
+#define ODEBUG_BATCH_SIZE	16
2831 
2932 #define ODEBUG_CHUNK_SHIFT	PAGE_SHIFT
3033 #define ODEBUG_CHUNK_SIZE	(1 << ODEBUG_CHUNK_SHIFT)
3134 #define ODEBUG_CHUNK_MASK	(~(ODEBUG_CHUNK_SIZE - 1))
3235 
36
+/*
37
+ * We limit the freeing of debug objects via workqueue at a maximum
38
+ * frequency of 10Hz and about 1024 objects for each freeing operation.
39
+ * So it is freeing at most 10k debug objects per second.
40
+ */
41
+#define ODEBUG_FREE_WORK_MAX	1024
42
+#define ODEBUG_FREE_WORK_DELAY	DIV_ROUND_UP(HZ, 10)
43
+
3344 struct debug_bucket {
3445 	struct hlist_head	list;
3546 	raw_spinlock_t		lock;
3647 };
48
+
49
+/*
50
+ * Debug object percpu free list
51
+ * Access is protected by disabling irq
52
+ */
53
+struct debug_percpu_free {
54
+	struct hlist_head	free_objs;
55
+	int			obj_free;
56
+};
57
+
58
+static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
3759 
3860 static struct debug_bucket	obj_hash[ODEBUG_HASH_SIZE];
3961 
....@@ -44,13 +66,20 @@
4466 static HLIST_HEAD(obj_pool);
4567 static HLIST_HEAD(obj_to_free);
4668 
69
+/*
70
+ * Because of the presence of percpu free pools, obj_pool_free will
71
+ * under-count those in the percpu free pools. Similarly, obj_pool_used
72
+ * will over-count those in the percpu free pools. Adjustments will be
73
+ * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
74
+ * can be off.
75
+ */
4776 static int			obj_pool_min_free = ODEBUG_POOL_SIZE;
4877 static int			obj_pool_free = ODEBUG_POOL_SIZE;
4978 static int			obj_pool_used;
5079 static int			obj_pool_max_used;
80
+static bool			obj_freeing;
5181 /* The number of objs on the global free list */
5282 static int			obj_nr_tofree;
53
-static struct kmem_cache	*obj_cache;
5483 
5584 static int			debug_objects_maxchain __read_mostly;
5685 static int __maybe_unused	debug_objects_maxchecked __read_mostly;
....@@ -62,7 +91,8 @@
6291 				= ODEBUG_POOL_SIZE;
6392 static int			debug_objects_pool_min_level __read_mostly
6493 				= ODEBUG_POOL_MIN_LEVEL;
65
-static struct debug_obj_descr	*descr_test  __read_mostly;
94
+static const struct debug_obj_descr *descr_test  __read_mostly;
95
+static struct kmem_cache	*obj_cache __read_mostly;
6696 
6797 /*
6898  * Track numbers of kmem_cache_alloc()/free() calls done.
....@@ -71,7 +101,7 @@
71101 static int			debug_objects_freed;
72102 
73103 static void free_obj_work(struct work_struct *work);
74
-static DECLARE_WORK(debug_obj_work, free_obj_work);
104
+static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
75105 
76106 static int __init enable_object_debug(char *str)
77107 {
....@@ -100,28 +130,32 @@
100130 static void fill_pool(void)
101131 {
102132 	gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
103
-	struct debug_obj *new, *obj;
133
+	struct debug_obj *obj;
104134 	unsigned long flags;
105135 
106
-	if (likely(obj_pool_free >= debug_objects_pool_min_level))
136
+	if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
107137 		return;
108138 
109139 	/*
110140 	 * Reuse objs from the global free list; they will be reinitialized
111141 	 * when allocating.
142
+	 *
143
+	 * Both obj_nr_tofree and obj_pool_free are checked locklessly; the
144
+	 * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical
145
+	 * sections.
112146 	 */
113
-	while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
147
+	while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) {
114148 		raw_spin_lock_irqsave(&pool_lock, flags);
115149 		/*
116150 		 * Recheck with the lock held as the worker thread might have
117151 		 * won the race and freed the global free list already.
118152 		 */
119
-		if (obj_nr_tofree) {
153
+		while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
120154 			obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
121155 			hlist_del(&obj->node);
122
-			obj_nr_tofree--;
156
+			WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
123157 			hlist_add_head(&obj->node, &obj_pool);
124
-			obj_pool_free++;
158
+			WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
125159 		}
126160 		raw_spin_unlock_irqrestore(&pool_lock, flags);
127161 	}
....@@ -129,16 +163,24 @@
129163 	if (unlikely(!obj_cache))
130164 		return;
131165 
132
-	while (obj_pool_free < debug_objects_pool_min_level) {
166
+	while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
167
+		struct debug_obj *new[ODEBUG_BATCH_SIZE];
168
+		int cnt;
133169 
134
-		new = kmem_cache_zalloc(obj_cache, gfp);
135
-		if (!new)
170
+		for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
171
+			new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
172
+			if (!new[cnt])
173
+				break;
174
+		}
175
+		if (!cnt)
136176 			return;
137177 
138178 		raw_spin_lock_irqsave(&pool_lock, flags);
139
-		hlist_add_head(&new->node, &obj_pool);
140
-		debug_objects_allocated++;
141
-		obj_pool_free++;
179
+		while (cnt) {
180
+			hlist_add_head(&new[--cnt]->node, &obj_pool);
181
+			debug_objects_allocated++;
182
+			WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
183
+		}
142184 		raw_spin_unlock_irqrestore(&pool_lock, flags);
143185 	}
144186 }
....@@ -163,36 +205,81 @@
163205 }
164206 
165207 /*
208
+ * Allocate a new object from the hlist
209
+ */
210
+static struct debug_obj *__alloc_object(struct hlist_head *list)
211
+{
212
+	struct debug_obj *obj = NULL;
213
+
214
+	if (list->first) {
215
+		obj = hlist_entry(list->first, typeof(*obj), node);
216
+		hlist_del(&obj->node);
217
+	}
218
+
219
+	return obj;
220
+}
221
+
222
+/*
166223  * Allocate a new object. If the pool is empty, switch off the debugger.
167224  * Must be called with interrupts disabled.
168225  */
169226 static struct debug_obj *
170
-alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
227
+alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
171228 {
172
-	struct debug_obj *obj = NULL;
229
+	struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
230
+	struct debug_obj *obj;
231
+
232
+	if (likely(obj_cache)) {
233
+		obj = __alloc_object(&percpu_pool->free_objs);
234
+		if (obj) {
235
+			percpu_pool->obj_free--;
236
+			goto init_obj;
237
+		}
238
+	}
173239 
174240 	raw_spin_lock(&pool_lock);
175
-	if (obj_pool.first) {
176
-		obj	    = hlist_entry(obj_pool.first, typeof(*obj), node);
177
-
178
-		obj->object = addr;
179
-		obj->descr  = descr;
180
-		obj->state  = ODEBUG_STATE_NONE;
181
-		obj->astate = 0;
182
-		hlist_del(&obj->node);
183
-
184
-		hlist_add_head(&obj->node, &b->list);
185
-
241
+	obj = __alloc_object(&obj_pool);
242
+	if (obj) {
186243 		obj_pool_used++;
244
+		WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
245
+
246
+		/*
247
+		 * Looking ahead, allocate one batch of debug objects and
248
+		 * put them into the percpu free pool.
249
+		 */
250
+		if (likely(obj_cache)) {
251
+			int i;
252
+
253
+			for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
254
+				struct debug_obj *obj2;
255
+
256
+				obj2 = __alloc_object(&obj_pool);
257
+				if (!obj2)
258
+					break;
259
+				hlist_add_head(&obj2->node,
260
+					       &percpu_pool->free_objs);
261
+				percpu_pool->obj_free++;
262
+				obj_pool_used++;
263
+				WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
264
+			}
265
+		}
266
+
187267 		if (obj_pool_used > obj_pool_max_used)
188268 			obj_pool_max_used = obj_pool_used;
189269 
190
-		obj_pool_free--;
191270 		if (obj_pool_free < obj_pool_min_free)
192271 			obj_pool_min_free = obj_pool_free;
193272 	}
194273 	raw_spin_unlock(&pool_lock);
195274 
275
+init_obj:
276
+	if (obj) {
277
+		obj->object = addr;
278
+		obj->descr  = descr;
279
+		obj->state  = ODEBUG_STATE_NONE;
280
+		obj->astate = 0;
281
+		hlist_add_head(&obj->node, &b->list);
282
+	}
196283 	return obj;
197284 }
198285 
....@@ -209,22 +296,31 @@
209296 	unsigned long flags;
210297 	HLIST_HEAD(tofree);
211298 
299
+	WRITE_ONCE(obj_freeing, false);
212300 	if (!raw_spin_trylock_irqsave(&pool_lock, flags))
213301 		return;
302
+
303
+	if (obj_pool_free >= debug_objects_pool_size)
304
+		goto free_objs;
214305 
215306 	/*
216307 	 * The objs on the pool list might be allocated before the work is
217308 	 * run, so recheck if pool list it full or not, if not fill pool
218
-	 * list from the global free list
309
+	 * list from the global free list. As it is likely that a workload
310
+	 * may be gearing up to use more and more objects, don't free any
311
+	 * of them until the next round.
219312 	 */
220313 	while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
221314 		obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
222315 		hlist_del(&obj->node);
223316 		hlist_add_head(&obj->node, &obj_pool);
224
-		obj_pool_free++;
225
-		obj_nr_tofree--;
317
+		WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
318
+		WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
226319 	}
320
+	raw_spin_unlock_irqrestore(&pool_lock, flags);
321
+	return;
227322 
323
+free_objs:
228324 	/*
229325 	 * Pool list is already full and there are still objs on the free
230326 	 * list. Move remaining free objs to a temporary list to free the
....@@ -233,7 +329,7 @@
233329 	if (obj_nr_tofree) {
234330 		hlist_move_list(&obj_to_free, &tofree);
235331 		debug_objects_freed += obj_nr_tofree;
236
-		obj_nr_tofree = 0;
332
+		WRITE_ONCE(obj_nr_tofree, 0);
237333 	}
238334 	raw_spin_unlock_irqrestore(&pool_lock, flags);
239335 
....@@ -243,24 +339,86 @@
243339 	}
244340 }
245341 
246
-static bool __free_object(struct debug_obj *obj)
342
+static void __free_object(struct debug_obj *obj)
247343 {
344
+	struct debug_obj *objs[ODEBUG_BATCH_SIZE];
345
+	struct debug_percpu_free *percpu_pool;
346
+	int lookahead_count = 0;
248347 	unsigned long flags;
249348 	bool work;
250349 
251
-	raw_spin_lock_irqsave(&pool_lock, flags);
252
-	work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
350
+	local_irq_save(flags);
351
+	if (!obj_cache)
352
+		goto free_to_obj_pool;
353
+
354
+	/*
355
+	 * Try to free it into the percpu pool first.
356
+	 */
357
+	percpu_pool = this_cpu_ptr(&percpu_obj_pool);
358
+	if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
359
+		hlist_add_head(&obj->node, &percpu_pool->free_objs);
360
+		percpu_pool->obj_free++;
361
+		local_irq_restore(flags);
362
+		return;
363
+	}
364
+
365
+	/*
366
+	 * As the percpu pool is full, look ahead and pull out a batch
367
+	 * of objects from the percpu pool and free them as well.
368
+	 */
369
+	for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
370
+		objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
371
+		if (!objs[lookahead_count])
372
+			break;
373
+		percpu_pool->obj_free--;
374
+	}
375
+
376
+free_to_obj_pool:
377
+	raw_spin_lock(&pool_lock);
378
+	work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
379
+	       (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
253380 	obj_pool_used--;
254381 
255382 	if (work) {
256
-		obj_nr_tofree++;
383
+		WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
257384 		hlist_add_head(&obj->node, &obj_to_free);
385
+		if (lookahead_count) {
386
+			WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
387
+			obj_pool_used -= lookahead_count;
388
+			while (lookahead_count) {
389
+				hlist_add_head(&objs[--lookahead_count]->node,
390
+					       &obj_to_free);
391
+			}
392
+		}
393
+
394
+		if ((obj_pool_free > debug_objects_pool_size) &&
395
+		    (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
396
+			int i;
397
+
398
+			/*
399
+			 * Free one more batch of objects from obj_pool.
400
+			 */
401
+			for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
402
+				obj = __alloc_object(&obj_pool);
403
+				hlist_add_head(&obj->node, &obj_to_free);
404
+				WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
405
+				WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
406
+			}
407
+		}
258408 	} else {
259
-		obj_pool_free++;
409
+		WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
260410 		hlist_add_head(&obj->node, &obj_pool);
411
+		if (lookahead_count) {
412
+			WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
413
+			obj_pool_used -= lookahead_count;
414
+			while (lookahead_count) {
415
+				hlist_add_head(&objs[--lookahead_count]->node,
416
+					       &obj_pool);
417
+			}
418
+		}
261419 	}
262
-	raw_spin_unlock_irqrestore(&pool_lock, flags);
263
-	return work;
420
+	raw_spin_unlock(&pool_lock);
421
+	local_irq_restore(flags);
264422 }
265423 
266424 /*
....@@ -269,9 +427,31 @@
269427  */
270428 static void free_object(struct debug_obj *obj)
271429 {
272
-	if (__free_object(obj))
273
-		schedule_work(&debug_obj_work);
430
+	__free_object(obj);
431
+	if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
432
+		WRITE_ONCE(obj_freeing, true);
433
+		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
434
+	}
274435 }
436
+
437
+#ifdef CONFIG_HOTPLUG_CPU
438
+static int object_cpu_offline(unsigned int cpu)
439
+{
440
+	struct debug_percpu_free *percpu_pool;
441
+	struct hlist_node *tmp;
442
+	struct debug_obj *obj;
443
+
444
+	/* Remote access is safe as the CPU is dead already */
445
+	percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu);
446
+	hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) {
447
+		hlist_del(&obj->node);
448
+		kmem_cache_free(obj_cache, obj);
449
+	}
450
+	percpu_pool->obj_free = 0;
451
+
452
+	return 0;
453
+}
454
+#endif
275455 
276456 /*
277457  * We run out of memory. That means we probably have tons of objects
....@@ -315,7 +495,7 @@
315495 
316496 static void debug_print_object(struct debug_obj *obj, char *msg)
317497 {
318
-	struct debug_obj_descr *descr = obj->descr;
498
+	const struct debug_obj_descr *descr = obj->descr;
319499 	static int limit;
320500 
321501 	if (limit < 5 && descr != descr_test) {
....@@ -369,17 +549,15 @@
369549 }
370550 
371551 static void
372
-__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
552
+__debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack)
373553 {
374554 	enum debug_obj_state state;
555
+	bool check_stack = false;
375556 	struct debug_bucket *db;
376557 	struct debug_obj *obj;
377558 	unsigned long flags;
378559 
379
-#ifdef CONFIG_PREEMPT_RT_FULL
380
-	if (preempt_count() == 0 && !irqs_disabled())
381
-#endif
382
-		fill_pool();
560
+	fill_pool();
383561 
384562 	db = get_bucket((unsigned long) addr);
385563 
....@@ -394,7 +572,7 @@
394572 			debug_objects_oom();
395573 			return;
396574 		}
397
-		debug_object_is_on_stack(addr, onstack);
575
+		check_stack = true;
398576 	}
399577 
400578 	switch (obj->state) {
....@@ -405,20 +583,23 @@
405583 		break;
406584 
407585 	case ODEBUG_STATE_ACTIVE:
408
-		debug_print_object(obj, "init");
409586 		state = obj->state;
410587 		raw_spin_unlock_irqrestore(&db->lock, flags);
588
+		debug_print_object(obj, "init");
411589 		debug_object_fixup(descr->fixup_init, addr, state);
412590 		return;
413591 
414592 	case ODEBUG_STATE_DESTROYED:
593
+		raw_spin_unlock_irqrestore(&db->lock, flags);
415594 		debug_print_object(obj, "init");
416
-		break;
595
+		return;
417596 	default:
418597 		break;
419598 	}
420599 
421600 	raw_spin_unlock_irqrestore(&db->lock, flags);
601
+	if (check_stack)
602
+		debug_object_is_on_stack(addr, onstack);
422603 }
423604 
424605 /**
....@@ -426,7 +607,7 @@
426607  * @addr:	address of the object
427608  * @descr:	pointer to an object specific debug description structure
428609  */
429
-void debug_object_init(void *addr, struct debug_obj_descr *descr)
610
+void debug_object_init(void *addr, const struct debug_obj_descr *descr)
430611 {
431612 	if (!debug_objects_enabled)
432613 		return;
....@@ -441,7 +622,7 @@
441622  * @addr:	address of the object
442623  * @descr:	pointer to an object specific debug description structure
443624  */
444
-void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
625
+void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr)
445626 {
446627 	if (!debug_objects_enabled)
447628 		return;
....@@ -456,7 +637,7 @@
456637  * @descr:	pointer to an object specific debug description structure
457638  * Returns 0 for success, -EINVAL for check failed.
458639  */
459
-int debug_object_activate(void *addr, struct debug_obj_descr *descr)
640
+int debug_object_activate(void *addr, const struct debug_obj_descr *descr)
460641 {
461642 	enum debug_obj_state state;
462643 	struct debug_bucket *db;
....@@ -476,6 +657,8 @@
476657 
477658 	obj = lookup_object(addr, db);
478659 	if (obj) {
660
+		bool print_object = false;
661
+
479662 		switch (obj->state) {
480663 		case ODEBUG_STATE_INIT:
481664 		case ODEBUG_STATE_INACTIVE:
....@@ -484,14 +667,14 @@
484667 			break;
485668 
486669 		case ODEBUG_STATE_ACTIVE:
487
-			debug_print_object(obj, "activate");
488670 			state = obj->state;
489671 			raw_spin_unlock_irqrestore(&db->lock, flags);
672
+			debug_print_object(obj, "activate");
490673 			ret = debug_object_fixup(descr->fixup_activate, addr, state);
491674 			return ret ? 0 : -EINVAL;
492675 
493676 		case ODEBUG_STATE_DESTROYED:
494
-			debug_print_object(obj, "activate");
677
+			print_object = true;
495678 			ret = -EINVAL;
496679 			break;
497680 		default:
....@@ -499,10 +682,13 @@
499682 			break;
500683 		}
501684 		raw_spin_unlock_irqrestore(&db->lock, flags);
685
+		if (print_object)
686
+			debug_print_object(obj, "activate");
502687 		return ret;
503688 	}
504689 
505690 	raw_spin_unlock_irqrestore(&db->lock, flags);
691
+
506692 	/*
507693 	 * We are here when a static object is activated. We
508694 	 * let the type specific code confirm whether this is
....@@ -529,11 +715,12 @@
529715  * @addr:	address of the object
530716  * @descr:	pointer to an object specific debug description structure
531717  */
532
-void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
718
+void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr)
533719 {
534720 	struct debug_bucket *db;
535721 	struct debug_obj *obj;
536722 	unsigned long flags;
723
+	bool print_object = false;
537724 
538725 	if (!debug_objects_enabled)
539726 		return;
....@@ -551,24 +738,27 @@
551738 			if (!obj->astate)
552739 				obj->state = ODEBUG_STATE_INACTIVE;
553740 			else
554
-				debug_print_object(obj, "deactivate");
741
+				print_object = true;
555742 			break;
556743 
557744 		case ODEBUG_STATE_DESTROYED:
558
-			debug_print_object(obj, "deactivate");
745
+			print_object = true;
559746 			break;
560747 		default:
561748 			break;
562749 		}
563
-	} else {
750
+	}
751
+
752
+	raw_spin_unlock_irqrestore(&db->lock, flags);
753
+	if (!obj) {
564754 		struct debug_obj o = { .object = addr,
565755 				       .state = ODEBUG_STATE_NOTAVAILABLE,
566756 				       .descr = descr };
567757 
568758 		debug_print_object(&o, "deactivate");
759
+	} else if (print_object) {
760
+		debug_print_object(obj, "deactivate");
569761 	}
570
-
571
-	raw_spin_unlock_irqrestore(&db->lock, flags);
572762 }
573763 EXPORT_SYMBOL_GPL(debug_object_deactivate);
574764 
....@@ -577,12 +767,13 @@
577767  * @addr:	address of the object
578768  * @descr:	pointer to an object specific debug description structure
579769  */
580
-void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
770
+void debug_object_destroy(void *addr, const struct debug_obj_descr *descr)
581771 {
582772 	enum debug_obj_state state;
583773 	struct debug_bucket *db;
584774 	struct debug_obj *obj;
585775 	unsigned long flags;
776
+	bool print_object = false;
586777 
587778 	if (!debug_objects_enabled)
588779 		return;
....@@ -602,20 +793,22 @@
602793 		obj->state = ODEBUG_STATE_DESTROYED;
603794 		break;
604795 	case ODEBUG_STATE_ACTIVE:
605
-		debug_print_object(obj, "destroy");
606796 		state = obj->state;
607797 		raw_spin_unlock_irqrestore(&db->lock, flags);
798
+		debug_print_object(obj, "destroy");
608799 		debug_object_fixup(descr->fixup_destroy, addr, state);
609800 		return;
610801 
611802 	case ODEBUG_STATE_DESTROYED:
612
-		debug_print_object(obj, "destroy");
803
+		print_object = true;
613804 		break;
614805 	default:
615806 		break;
616807 	}
617808 out_unlock:
618809 	raw_spin_unlock_irqrestore(&db->lock, flags);
810
+	if (print_object)
811
+		debug_print_object(obj, "destroy");
619812 }
620813 EXPORT_SYMBOL_GPL(debug_object_destroy);
621814 
....@@ -624,7 +817,7 @@
624817  * @addr:	address of the object
625818  * @descr:	pointer to an object specific debug description structure
626819  */
627
-void debug_object_free(void *addr, struct debug_obj_descr *descr)
820
+void debug_object_free(void *addr, const struct debug_obj_descr *descr)
628821 {
629822 	enum debug_obj_state state;
630823 	struct debug_bucket *db;
....@@ -644,9 +837,9 @@
644837 
645838 	switch (obj->state) {
646839 	case ODEBUG_STATE_ACTIVE:
647
-		debug_print_object(obj, "free");
648840 		state = obj->state;
649841 		raw_spin_unlock_irqrestore(&db->lock, flags);
842
+		debug_print_object(obj, "free");
650843 		debug_object_fixup(descr->fixup_free, addr, state);
651844 		return;
652845 	default:
....@@ -665,7 +858,7 @@
665858  * @addr:	address of the object
666859  * @descr:	pointer to an object specific debug description structure
667860  */
668
-void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
861
+void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr)
669862 {
670863 	struct debug_bucket *db;
671864 	struct debug_obj *obj;
....@@ -713,12 +906,13 @@
713906  * @next:	state to move to if expected state is found
714907  */
715908 void
716
-debug_object_active_state(void *addr, struct debug_obj_descr *descr,
909
+debug_object_active_state(void *addr, const struct debug_obj_descr *descr,
717910 			  unsigned int expect, unsigned int next)
718911 {
719912 	struct debug_bucket *db;
720913 	struct debug_obj *obj;
721914 	unsigned long flags;
915
+	bool print_object = false;
722916 
723917 	if (!debug_objects_enabled)
724918 		return;
....@@ -734,22 +928,25 @@
734928 			if (obj->astate == expect)
735929 				obj->astate = next;
736930 			else
737
-				debug_print_object(obj, "active_state");
931
+				print_object = true;
738932 			break;
739933 
740934 		default:
741
-			debug_print_object(obj, "active_state");
935
+			print_object = true;
742936 			break;
743937 		}
744
-	} else {
938
+	}
939
+
940
+	raw_spin_unlock_irqrestore(&db->lock, flags);
941
+	if (!obj) {
745942 		struct debug_obj o = { .object = addr,
746943 				       .state = ODEBUG_STATE_NOTAVAILABLE,
747944 				       .descr = descr };
748945 
749946 		debug_print_object(&o, "active_state");
947
+	} else if (print_object) {
948
+		debug_print_object(obj, "active_state");
750949 	}
751
-
752
-	raw_spin_unlock_irqrestore(&db->lock, flags);
753950 }
754951 EXPORT_SYMBOL_GPL(debug_object_active_state);
755952 
....@@ -757,13 +954,12 @@
757954 static void __debug_check_no_obj_freed(const void *address, unsigned long size)
758955 {
759956 	unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
760
-	struct debug_obj_descr *descr;
957
+	const struct debug_obj_descr *descr;
761958 	enum debug_obj_state state;
762959 	struct debug_bucket *db;
763960 	struct hlist_node *tmp;
764961 	struct debug_obj *obj;
765962 	int cnt, objs_checked = 0;
766
-	bool work = false;
767963 
768964 	saddr = (unsigned long) address;
769965 	eaddr = saddr + size;
....@@ -785,16 +981,16 @@
785981 
786982 			switch (obj->state) {
787983 			case ODEBUG_STATE_ACTIVE:
788
-				debug_print_object(obj, "free");
789984 				descr = obj->descr;
790985 				state = obj->state;
791986 				raw_spin_unlock_irqrestore(&db->lock, flags);
987
+				debug_print_object(obj, "free");
792988 				debug_object_fixup(descr->fixup_free,
793989 						   (void *) oaddr, state);
794990 				goto repeat;
795991 			default:
796992 				hlist_del(&obj->node);
797
-				work |= __free_object(obj);
993
+				__free_object(obj);
798994 				break;
799995 			}
800996 		}
....@@ -810,8 +1006,10 @@
8101006 		debug_objects_maxchecked = objs_checked;
8111007 
8121008 	/* Schedule work to actually kmem_cache_free() objects */
813
-	if (work)
814
-		schedule_work(&debug_obj_work);
1009
+	if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
1010
+		WRITE_ONCE(obj_freeing, true);
1011
+		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
1012
+	}
8151013 }
8161014 
8171015 void debug_check_no_obj_freed(const void *address, unsigned long size)
....@@ -825,54 +1023,39 @@
8251023 
8261024 static int debug_stats_show(struct seq_file *m, void *v)
8271025 {
1026
+	int cpu, obj_percpu_free = 0;
1027
+
1028
+	for_each_possible_cpu(cpu)
1029
+		obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
1030
+
8281031 	seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
8291032 	seq_printf(m, "max_checked   :%d\n", debug_objects_maxchecked);
8301033 	seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
8311034 	seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
832
-	seq_printf(m, "pool_free     :%d\n", obj_pool_free);
1035
+	seq_printf(m, "pool_free     :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
1036
+	seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
8331037 	seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
834
-	seq_printf(m, "pool_used     :%d\n", obj_pool_used);
1038
+	seq_printf(m, "pool_used     :%d\n", obj_pool_used - obj_percpu_free);
8351039 	seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
836
-	seq_printf(m, "on_free_list  :%d\n", obj_nr_tofree);
1040
+	seq_printf(m, "on_free_list  :%d\n", READ_ONCE(obj_nr_tofree));
8371041 	seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
8381042 	seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
8391043 	return 0;
8401044 }
841
-
842
-static int debug_stats_open(struct inode *inode, struct file *filp)
843
-{
844
-	return single_open(filp, debug_stats_show, NULL);
845
-}
846
-
847
-static const struct file_operations debug_stats_fops = {
848
-	.open		= debug_stats_open,
849
-	.read		= seq_read,
850
-	.llseek		= seq_lseek,
851
-	.release	= single_release,
852
-};
1045
+DEFINE_SHOW_ATTRIBUTE(debug_stats);
8531046 
8541047 static int __init debug_objects_init_debugfs(void)
8551048 {
856
-	struct dentry *dbgdir, *dbgstats;
1049
+	struct dentry *dbgdir;
8571050 
8581051 	if (!debug_objects_enabled)
8591052 		return 0;
8601053 
8611054 	dbgdir = debugfs_create_dir("debug_objects", NULL);
862
-	if (!dbgdir)
863
-		return -ENOMEM;
8641055 
865
-	dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL,
866
-				       &debug_stats_fops);
867
-	if (!dbgstats)
868
-		goto err;
1056
+	debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
8691057 
8701058 	return 0;
871
-
872
-err:
873
-	debugfs_remove(dbgdir);
874
-
875
-	return -ENOMEM;
8761059 }
8771060 __initcall(debug_objects_init_debugfs);
8781061 
....@@ -889,7 +1072,7 @@
8891072 	unsigned long	dummy2[3];
8901073 };
8911074 
892
-static __initdata struct debug_obj_descr descr_type_test;
1075
+static __initconst const struct debug_obj_descr descr_type_test;
8931076 
8941077 static bool __init is_static_object(void *addr)
8951078 {
....@@ -1014,7 +1197,7 @@
10141197 	return res;
10151198 }
10161199 
1017
-static __initdata struct debug_obj_descr descr_type_test = {
1200
+static __initconst const struct debug_obj_descr descr_type_test = {
10181201 	.name			= "selftest",
10191202 	.is_static_object	= is_static_object,
10201203 	.fixup_init		= fixup_init,
....@@ -1134,11 +1317,10 @@
11341317 	}
11351318 
11361319 	/*
1137
-	 * When debug_objects_mem_init() is called we know that only
1138
-	 * one CPU is up, so disabling interrupts is enough
1139
-	 * protection. This avoids the lockdep hell of lock ordering.
1320
+	 * debug_objects_mem_init() is now called early that only one CPU is up
1321
+	 * and interrupts have been disabled, so it is safe to replace the
1322
+	 * active object references.
11401323 	 */
1141
-	local_irq_disable();
11421324 
11431325 	/* Remove the statically allocated objects from the pool */
11441326 	hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
....@@ -1159,7 +1341,6 @@
11591341 			cnt++;
11601342 		}
11611343 	}
1162
-	local_irq_enable();
11631344 
11641345 	pr_debug("%d of %d active objects replaced\n",
11651346 		 cnt, obj_pool_used);
....@@ -1180,8 +1361,19 @@
11801361  */
11811362 void __init debug_objects_mem_init(void)
11821363 {
1364
+	int cpu, extras;
1365
+
11831366 	if (!debug_objects_enabled)
11841367 		return;
1368
+
1369
+	/*
1370
+	 * Initialize the percpu object pools
1371
+	 *
1372
+	 * Initialization is not strictly necessary, but was done for
1373
+	 * completeness.
1374
+	 */
1375
+	for_each_possible_cpu(cpu)
1376
+		INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
11851377 
11861378 	obj_cache = kmem_cache_create("debug_objects_cache",
11871379 				      sizeof (struct debug_obj), 0,
....@@ -1195,10 +1387,16 @@
11951387 	} else
11961388 		debug_objects_selftest();
11971389 
1390
+#ifdef CONFIG_HOTPLUG_CPU
1391
+	cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
1392
+					object_cpu_offline);
1393
+#endif
1394
+
11981395 	/*
11991396 	 * Increase the thresholds for allocating and freeing objects
12001397 	 * according to the number of possible CPUs available in the system.
12011398 	 */
1202
-	debug_objects_pool_size += num_possible_cpus() * 32;
1203
-	debug_objects_pool_min_level += num_possible_cpus() * 4;
1399
+	extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
1400
+	debug_objects_pool_size += extras;
1401
+	debug_objects_pool_min_level += extras;
12041402 }