forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-06 08f87f769b595151be1afeff53e144f543faa614 
 kernel/lib/debugobjects.c
....@@ -19,21 +19,43 @@
1919 #include <linux/slab.h>
2020 #include <linux/hash.h>
2121 #include <linux/kmemleak.h>
22
+#include <linux/cpu.h>
2223 
2324 #define ODEBUG_HASH_BITS	14
2425 #define ODEBUG_HASH_SIZE	(1 << ODEBUG_HASH_BITS)
2526 
2627 #define ODEBUG_POOL_SIZE	1024
2728 #define ODEBUG_POOL_MIN_LEVEL	256
29
+#define ODEBUG_POOL_PERCPU_SIZE	64
30
+#define ODEBUG_BATCH_SIZE	16
2831 
2932 #define ODEBUG_CHUNK_SHIFT	PAGE_SHIFT
3033 #define ODEBUG_CHUNK_SIZE	(1 << ODEBUG_CHUNK_SHIFT)
3134 #define ODEBUG_CHUNK_MASK	(~(ODEBUG_CHUNK_SIZE - 1))
3235 
36
+/*
37
+ * We limit the freeing of debug objects via workqueue at a maximum
38
+ * frequency of 10Hz and about 1024 objects for each freeing operation.
39
+ * So it is freeing at most 10k debug objects per second.
40
+ */
41
+#define ODEBUG_FREE_WORK_MAX	1024
42
+#define ODEBUG_FREE_WORK_DELAY	DIV_ROUND_UP(HZ, 10)
43
+
3344 struct debug_bucket {
3445 	struct hlist_head	list;
3546 	raw_spinlock_t		lock;
3647 };
48
+
49
+/*
50
+ * Debug object percpu free list
51
+ * Access is protected by disabling irq
52
+ */
53
+struct debug_percpu_free {
54
+	struct hlist_head	free_objs;
55
+	int			obj_free;
56
+};
57
+
58
+static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
3759 
3860 static struct debug_bucket	obj_hash[ODEBUG_HASH_SIZE];
3961 
....@@ -44,13 +66,20 @@
4466 static HLIST_HEAD(obj_pool);
4567 static HLIST_HEAD(obj_to_free);
4668 
69
+/*
70
+ * Because of the presence of percpu free pools, obj_pool_free will
71
+ * under-count those in the percpu free pools. Similarly, obj_pool_used
72
+ * will over-count those in the percpu free pools. Adjustments will be
73
+ * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
74
+ * can be off.
75
+ */
4776 static int			obj_pool_min_free = ODEBUG_POOL_SIZE;
4877 static int			obj_pool_free = ODEBUG_POOL_SIZE;
4978 static int			obj_pool_used;
5079 static int			obj_pool_max_used;
80
+static bool			obj_freeing;
5181 /* The number of objs on the global free list */
5282 static int			obj_nr_tofree;
53
-static struct kmem_cache	*obj_cache;
5483 
5584 static int			debug_objects_maxchain __read_mostly;
5685 static int __maybe_unused	debug_objects_maxchecked __read_mostly;
....@@ -62,7 +91,8 @@
6291 				= ODEBUG_POOL_SIZE;
6392 static int			debug_objects_pool_min_level __read_mostly
6493 				= ODEBUG_POOL_MIN_LEVEL;
65
-static struct debug_obj_descr	*descr_test  __read_mostly;
94
+static const struct debug_obj_descr *descr_test  __read_mostly;
95
+static struct kmem_cache	*obj_cache __read_mostly;
6696 
6797 /*
6898  * Track numbers of kmem_cache_alloc()/free() calls done.
....@@ -71,7 +101,7 @@
71101 static int			debug_objects_freed;
72102 
73103 static void free_obj_work(struct work_struct *work);
74
-static DECLARE_WORK(debug_obj_work, free_obj_work);
104
+static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
75105 
76106 static int __init enable_object_debug(char *str)
77107 {
....@@ -100,28 +130,32 @@
100130 static void fill_pool(void)
101131 {
102132 	gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
103
-	struct debug_obj *new, *obj;
133
+	struct debug_obj *obj;
104134 	unsigned long flags;
105135 
106
-	if (likely(obj_pool_free >= debug_objects_pool_min_level))
136
+	if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
107137 		return;
108138 
109139 	/*
110140 	 * Reuse objs from the global free list; they will be reinitialized
111141 	 * when allocating.
142
+	 *
143
+	 * Both obj_nr_tofree and obj_pool_free are checked locklessly; the
144
+	 * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical
145
+	 * sections.
112146 	 */
113
-	while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
147
+	while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) {
114148 		raw_spin_lock_irqsave(&pool_lock, flags);
115149 		/*
116150 		 * Recheck with the lock held as the worker thread might have
117151 		 * won the race and freed the global free list already.
118152 		 */
119
-		if (obj_nr_tofree) {
153
+		while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
120154 			obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
121155 			hlist_del(&obj->node);
122
-			obj_nr_tofree--;
156
+			WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
123157 			hlist_add_head(&obj->node, &obj_pool);
124
-			obj_pool_free++;
158
+			WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
125159 		}
126160 		raw_spin_unlock_irqrestore(&pool_lock, flags);
127161 	}
....@@ -129,16 +163,24 @@
129163 	if (unlikely(!obj_cache))
130164 		return;
131165 
132
-	while (obj_pool_free < debug_objects_pool_min_level) {
166
+	while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
167
+		struct debug_obj *new[ODEBUG_BATCH_SIZE];
168
+		int cnt;
133169 
134
-		new = kmem_cache_zalloc(obj_cache, gfp);
135
-		if (!new)
170
+		for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
171
+			new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
172
+			if (!new[cnt])
173
+				break;
174
+		}
175
+		if (!cnt)
136176 			return;
137177 
138178 		raw_spin_lock_irqsave(&pool_lock, flags);
139
-		hlist_add_head(&new->node, &obj_pool);
140
-		debug_objects_allocated++;
141
-		obj_pool_free++;
179
+		while (cnt) {
180
+			hlist_add_head(&new[--cnt]->node, &obj_pool);
181
+			debug_objects_allocated++;
182
+			WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
183
+		}
142184 		raw_spin_unlock_irqrestore(&pool_lock, flags);
143185 	}
144186 }
....@@ -163,36 +205,81 @@
163205 }
164206 
165207 /*
208
+ * Allocate a new object from the hlist
209
+ */
210
+static struct debug_obj *__alloc_object(struct hlist_head *list)
211
+{
212
+	struct debug_obj *obj = NULL;
213
+
214
+	if (list->first) {
215
+		obj = hlist_entry(list->first, typeof(*obj), node);
216
+		hlist_del(&obj->node);
217
+	}
218
+
219
+	return obj;
220
+}
221
+
222
+/*
166223  * Allocate a new object. If the pool is empty, switch off the debugger.
167224  * Must be called with interrupts disabled.
168225  */
169226 static struct debug_obj *
170
-alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
227
+alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
171228 {
172
-	struct debug_obj *obj = NULL;
229
+	struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
230
+	struct debug_obj *obj;
231
+
232
+	if (likely(obj_cache)) {
233
+		obj = __alloc_object(&percpu_pool->free_objs);
234
+		if (obj) {
235
+			percpu_pool->obj_free--;
236
+			goto init_obj;
237
+		}
238
+	}
173239 
174240 	raw_spin_lock(&pool_lock);
175
-	if (obj_pool.first) {
176
-		obj	    = hlist_entry(obj_pool.first, typeof(*obj), node);
177
-
178
-		obj->object = addr;
179
-		obj->descr  = descr;
180
-		obj->state  = ODEBUG_STATE_NONE;
181
-		obj->astate = 0;
182
-		hlist_del(&obj->node);
183
-
184
-		hlist_add_head(&obj->node, &b->list);
185
-
241
+	obj = __alloc_object(&obj_pool);
242
+	if (obj) {
186243 		obj_pool_used++;
244
+		WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
245
+
246
+		/*
247
+		 * Looking ahead, allocate one batch of debug objects and
248
+		 * put them into the percpu free pool.
249
+		 */
250
+		if (likely(obj_cache)) {
251
+			int i;
252
+
253
+			for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
254
+				struct debug_obj *obj2;
255
+
256
+				obj2 = __alloc_object(&obj_pool);
257
+				if (!obj2)
258
+					break;
259
+				hlist_add_head(&obj2->node,
260
+					       &percpu_pool->free_objs);
261
+				percpu_pool->obj_free++;
262
+				obj_pool_used++;
263
+				WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
264
+			}
265
+		}
266
+
187267 		if (obj_pool_used > obj_pool_max_used)
188268 			obj_pool_max_used = obj_pool_used;
189269 
190
-		obj_pool_free--;
191270 		if (obj_pool_free < obj_pool_min_free)
192271 			obj_pool_min_free = obj_pool_free;
193272 	}
194273 	raw_spin_unlock(&pool_lock);
195274 
275
+init_obj:
276
+	if (obj) {
277
+		obj->object = addr;
278
+		obj->descr  = descr;
279
+		obj->state  = ODEBUG_STATE_NONE;
280
+		obj->astate = 0;
281
+		hlist_add_head(&obj->node, &b->list);
282
+	}
196283 	return obj;
197284 }
198285 
....@@ -209,22 +296,31 @@
209296 	unsigned long flags;
210297 	HLIST_HEAD(tofree);
211298 
299
+	WRITE_ONCE(obj_freeing, false);
212300 	if (!raw_spin_trylock_irqsave(&pool_lock, flags))
213301 		return;
302
+
303
+	if (obj_pool_free >= debug_objects_pool_size)
304
+		goto free_objs;
214305 
215306 	/*
216307 	 * The objs on the pool list might be allocated before the work is
217308 	 * run, so recheck if pool list it full or not, if not fill pool
218
-	 * list from the global free list
309
+	 * list from the global free list. As it is likely that a workload
310
+	 * may be gearing up to use more and more objects, don't free any
311
+	 * of them until the next round.
219312 	 */
220313 	while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
221314 		obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
222315 		hlist_del(&obj->node);
223316 		hlist_add_head(&obj->node, &obj_pool);
224
-		obj_pool_free++;
225
-		obj_nr_tofree--;
317
+		WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
318
+		WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
226319 	}
320
+	raw_spin_unlock_irqrestore(&pool_lock, flags);
321
+	return;
227322 
323
+free_objs:
228324 	/*
229325 	 * Pool list is already full and there are still objs on the free
230326 	 * list. Move remaining free objs to a temporary list to free the
....@@ -233,7 +329,7 @@
233329 	if (obj_nr_tofree) {
234330 		hlist_move_list(&obj_to_free, &tofree);
235331 		debug_objects_freed += obj_nr_tofree;
236
-		obj_nr_tofree = 0;
332
+		WRITE_ONCE(obj_nr_tofree, 0);
237333 	}
238334 	raw_spin_unlock_irqrestore(&pool_lock, flags);
239335 
....@@ -243,24 +339,86 @@
243339 	}
244340 }
245341 
246
-static bool __free_object(struct debug_obj *obj)
342
+static void __free_object(struct debug_obj *obj)
247343 {
344
+	struct debug_obj *objs[ODEBUG_BATCH_SIZE];
345
+	struct debug_percpu_free *percpu_pool;
346
+	int lookahead_count = 0;
248347 	unsigned long flags;
249348 	bool work;
250349 
251
-	raw_spin_lock_irqsave(&pool_lock, flags);
252
-	work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
350
+	local_irq_save(flags);
351
+	if (!obj_cache)
352
+		goto free_to_obj_pool;
353
+
354
+	/*
355
+	 * Try to free it into the percpu pool first.
356
+	 */
357
+	percpu_pool = this_cpu_ptr(&percpu_obj_pool);
358
+	if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
359
+		hlist_add_head(&obj->node, &percpu_pool->free_objs);
360
+		percpu_pool->obj_free++;
361
+		local_irq_restore(flags);
362
+		return;
363
+	}
364
+
365
+	/*
366
+	 * As the percpu pool is full, look ahead and pull out a batch
367
+	 * of objects from the percpu pool and free them as well.
368
+	 */
369
+	for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
370
+		objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
371
+		if (!objs[lookahead_count])
372
+			break;
373
+		percpu_pool->obj_free--;
374
+	}
375
+
376
+free_to_obj_pool:
377
+	raw_spin_lock(&pool_lock);
378
+	work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
379
+	       (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
253380 	obj_pool_used--;
254381 
255382 	if (work) {
256
-		obj_nr_tofree++;
383
+		WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
257384 		hlist_add_head(&obj->node, &obj_to_free);
385
+		if (lookahead_count) {
386
+			WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
387
+			obj_pool_used -= lookahead_count;
388
+			while (lookahead_count) {
389
+				hlist_add_head(&objs[--lookahead_count]->node,
390
+					       &obj_to_free);
391
+			}
392
+		}
393
+
394
+		if ((obj_pool_free > debug_objects_pool_size) &&
395
+		    (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
396
+			int i;
397
+
398
+			/*
399
+			 * Free one more batch of objects from obj_pool.
400
+			 */
401
+			for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
402
+				obj = __alloc_object(&obj_pool);
403
+				hlist_add_head(&obj->node, &obj_to_free);
404
+				WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
405
+				WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
406
+			}
407
+		}
258408 	} else {
259
-		obj_pool_free++;
409
+		WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
260410 		hlist_add_head(&obj->node, &obj_pool);
411
+		if (lookahead_count) {
412
+			WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
413
+			obj_pool_used -= lookahead_count;
414
+			while (lookahead_count) {
415
+				hlist_add_head(&objs[--lookahead_count]->node,
416
+					       &obj_pool);
417
+			}
418
+		}
261419 	}
262
-	raw_spin_unlock_irqrestore(&pool_lock, flags);
263
-	return work;
420
+	raw_spin_unlock(&pool_lock);
421
+	local_irq_restore(flags);
264422 }
265423 
266424 /*
....@@ -269,9 +427,31 @@
269427  */
270428 static void free_object(struct debug_obj *obj)
271429 {
272
-	if (__free_object(obj))
273
-		schedule_work(&debug_obj_work);
430
+	__free_object(obj);
431
+	if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
432
+		WRITE_ONCE(obj_freeing, true);
433
+		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
434
+	}
274435 }
436
+
437
+#ifdef CONFIG_HOTPLUG_CPU
438
+static int object_cpu_offline(unsigned int cpu)
439
+{
440
+	struct debug_percpu_free *percpu_pool;
441
+	struct hlist_node *tmp;
442
+	struct debug_obj *obj;
443
+
444
+	/* Remote access is safe as the CPU is dead already */
445
+	percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu);
446
+	hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) {
447
+		hlist_del(&obj->node);
448
+		kmem_cache_free(obj_cache, obj);
449
+	}
450
+	percpu_pool->obj_free = 0;
451
+
452
+	return 0;
453
+}
454
+#endif
275455 
276456 /*
277457  * We run out of memory. That means we probably have tons of objects
....@@ -315,7 +495,7 @@
315495 
316496 static void debug_print_object(struct debug_obj *obj, char *msg)
317497 {
318
-	struct debug_obj_descr *descr = obj->descr;
498
+	const struct debug_obj_descr *descr = obj->descr;
319499 	static int limit;
320500 
321501 	if (limit < 5 && descr != descr_test) {
....@@ -369,9 +549,10 @@
369549 }
370550 
371551 static void
372
-__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
552
+__debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack)
373553 {
374554 	enum debug_obj_state state;
555
+	bool check_stack = false;
375556 	struct debug_bucket *db;
376557 	struct debug_obj *obj;
377558 	unsigned long flags;
....@@ -391,7 +572,7 @@
391572 			debug_objects_oom();
392573 			return;
393574 		}
394
-		debug_object_is_on_stack(addr, onstack);
575
+		check_stack = true;
395576 	}
396577 
397578 	switch (obj->state) {
....@@ -402,20 +583,23 @@
402583 		break;
403584 
404585 	case ODEBUG_STATE_ACTIVE:
405
-		debug_print_object(obj, "init");
406586 		state = obj->state;
407587 		raw_spin_unlock_irqrestore(&db->lock, flags);
588
+		debug_print_object(obj, "init");
408589 		debug_object_fixup(descr->fixup_init, addr, state);
409590 		return;
410591 
411592 	case ODEBUG_STATE_DESTROYED:
593
+		raw_spin_unlock_irqrestore(&db->lock, flags);
412594 		debug_print_object(obj, "init");
413
-		break;
595
+		return;
414596 	default:
415597 		break;
416598 	}
417599 
418600 	raw_spin_unlock_irqrestore(&db->lock, flags);
601
+	if (check_stack)
602
+		debug_object_is_on_stack(addr, onstack);
419603 }
420604 
421605 /**
....@@ -423,7 +607,7 @@
423607  * @addr:	address of the object
424608  * @descr:	pointer to an object specific debug description structure
425609  */
426
-void debug_object_init(void *addr, struct debug_obj_descr *descr)
610
+void debug_object_init(void *addr, const struct debug_obj_descr *descr)
427611 {
428612 	if (!debug_objects_enabled)
429613 		return;
....@@ -438,7 +622,7 @@
438622  * @addr:	address of the object
439623  * @descr:	pointer to an object specific debug description structure
440624  */
441
-void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
625
+void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr)
442626 {
443627 	if (!debug_objects_enabled)
444628 		return;
....@@ -453,7 +637,7 @@
453637  * @descr:	pointer to an object specific debug description structure
454638  * Returns 0 for success, -EINVAL for check failed.
455639  */
456
-int debug_object_activate(void *addr, struct debug_obj_descr *descr)
640
+int debug_object_activate(void *addr, const struct debug_obj_descr *descr)
457641 {
458642 	enum debug_obj_state state;
459643 	struct debug_bucket *db;
....@@ -473,6 +657,8 @@
473657 
474658 	obj = lookup_object(addr, db);
475659 	if (obj) {
660
+		bool print_object = false;
661
+
476662 		switch (obj->state) {
477663 		case ODEBUG_STATE_INIT:
478664 		case ODEBUG_STATE_INACTIVE:
....@@ -481,14 +667,14 @@
481667 			break;
482668 
483669 		case ODEBUG_STATE_ACTIVE:
484
-			debug_print_object(obj, "activate");
485670 			state = obj->state;
486671 			raw_spin_unlock_irqrestore(&db->lock, flags);
672
+			debug_print_object(obj, "activate");
487673 			ret = debug_object_fixup(descr->fixup_activate, addr, state);
488674 			return ret ? 0 : -EINVAL;
489675 
490676 		case ODEBUG_STATE_DESTROYED:
491
-			debug_print_object(obj, "activate");
677
+			print_object = true;
492678 			ret = -EINVAL;
493679 			break;
494680 		default:
....@@ -496,10 +682,13 @@
496682 			break;
497683 		}
498684 		raw_spin_unlock_irqrestore(&db->lock, flags);
685
+		if (print_object)
686
+			debug_print_object(obj, "activate");
499687 		return ret;
500688 	}
501689 
502690 	raw_spin_unlock_irqrestore(&db->lock, flags);
691
+
503692 	/*
504693 	 * We are here when a static object is activated. We
505694 	 * let the type specific code confirm whether this is
....@@ -526,11 +715,12 @@
526715  * @addr:	address of the object
527716  * @descr:	pointer to an object specific debug description structure
528717  */
529
-void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
718
+void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr)
530719 {
531720 	struct debug_bucket *db;
532721 	struct debug_obj *obj;
533722 	unsigned long flags;
723
+	bool print_object = false;
534724 
535725 	if (!debug_objects_enabled)
536726 		return;
....@@ -548,24 +738,27 @@
548738 			if (!obj->astate)
549739 				obj->state = ODEBUG_STATE_INACTIVE;
550740 			else
551
-				debug_print_object(obj, "deactivate");
741
+				print_object = true;
552742 			break;
553743 
554744 		case ODEBUG_STATE_DESTROYED:
555
-			debug_print_object(obj, "deactivate");
745
+			print_object = true;
556746 			break;
557747 		default:
558748 			break;
559749 		}
560
-	} else {
750
+	}
751
+
752
+	raw_spin_unlock_irqrestore(&db->lock, flags);
753
+	if (!obj) {
561754 		struct debug_obj o = { .object = addr,
562755 				       .state = ODEBUG_STATE_NOTAVAILABLE,
563756 				       .descr = descr };
564757 
565758 		debug_print_object(&o, "deactivate");
759
+	} else if (print_object) {
760
+		debug_print_object(obj, "deactivate");
566761 	}
567
-
568
-	raw_spin_unlock_irqrestore(&db->lock, flags);
569762 }
570763 EXPORT_SYMBOL_GPL(debug_object_deactivate);
571764 
....@@ -574,12 +767,13 @@
574767  * @addr:	address of the object
575768  * @descr:	pointer to an object specific debug description structure
576769  */
577
-void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
770
+void debug_object_destroy(void *addr, const struct debug_obj_descr *descr)
578771 {
579772 	enum debug_obj_state state;
580773 	struct debug_bucket *db;
581774 	struct debug_obj *obj;
582775 	unsigned long flags;
776
+	bool print_object = false;
583777 
584778 	if (!debug_objects_enabled)
585779 		return;
....@@ -599,20 +793,22 @@
599793 		obj->state = ODEBUG_STATE_DESTROYED;
600794 		break;
601795 	case ODEBUG_STATE_ACTIVE:
602
-		debug_print_object(obj, "destroy");
603796 		state = obj->state;
604797 		raw_spin_unlock_irqrestore(&db->lock, flags);
798
+		debug_print_object(obj, "destroy");
605799 		debug_object_fixup(descr->fixup_destroy, addr, state);
606800 		return;
607801 
608802 	case ODEBUG_STATE_DESTROYED:
609
-		debug_print_object(obj, "destroy");
803
+		print_object = true;
610804 		break;
611805 	default:
612806 		break;
613807 	}
614808 out_unlock:
615809 	raw_spin_unlock_irqrestore(&db->lock, flags);
810
+	if (print_object)
811
+		debug_print_object(obj, "destroy");
616812 }
617813 EXPORT_SYMBOL_GPL(debug_object_destroy);
618814 
....@@ -621,7 +817,7 @@
621817  * @addr:	address of the object
622818  * @descr:	pointer to an object specific debug description structure
623819  */
624
-void debug_object_free(void *addr, struct debug_obj_descr *descr)
820
+void debug_object_free(void *addr, const struct debug_obj_descr *descr)
625821 {
626822 	enum debug_obj_state state;
627823 	struct debug_bucket *db;
....@@ -641,9 +837,9 @@
641837 
642838 	switch (obj->state) {
643839 	case ODEBUG_STATE_ACTIVE:
644
-		debug_print_object(obj, "free");
645840 		state = obj->state;
646841 		raw_spin_unlock_irqrestore(&db->lock, flags);
842
+		debug_print_object(obj, "free");
647843 		debug_object_fixup(descr->fixup_free, addr, state);
648844 		return;
649845 	default:
....@@ -662,7 +858,7 @@
662858  * @addr:	address of the object
663859  * @descr:	pointer to an object specific debug description structure
664860  */
665
-void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
861
+void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr)
666862 {
667863 	struct debug_bucket *db;
668864 	struct debug_obj *obj;
....@@ -710,12 +906,13 @@
710906  * @next:	state to move to if expected state is found
711907  */
712908 void
713
-debug_object_active_state(void *addr, struct debug_obj_descr *descr,
909
+debug_object_active_state(void *addr, const struct debug_obj_descr *descr,
714910 			  unsigned int expect, unsigned int next)
715911 {
716912 	struct debug_bucket *db;
717913 	struct debug_obj *obj;
718914 	unsigned long flags;
915
+	bool print_object = false;
719916 
720917 	if (!debug_objects_enabled)
721918 		return;
....@@ -731,22 +928,25 @@
731928 			if (obj->astate == expect)
732929 				obj->astate = next;
733930 			else
734
-				debug_print_object(obj, "active_state");
931
+				print_object = true;
735932 			break;
736933 
737934 		default:
738
-			debug_print_object(obj, "active_state");
935
+			print_object = true;
739936 			break;
740937 		}
741
-	} else {
938
+	}
939
+
940
+	raw_spin_unlock_irqrestore(&db->lock, flags);
941
+	if (!obj) {
742942 		struct debug_obj o = { .object = addr,
743943 				       .state = ODEBUG_STATE_NOTAVAILABLE,
744944 				       .descr = descr };
745945 
746946 		debug_print_object(&o, "active_state");
947
+	} else if (print_object) {
948
+		debug_print_object(obj, "active_state");
747949 	}
748
-
749
-	raw_spin_unlock_irqrestore(&db->lock, flags);
750950 }
751951 EXPORT_SYMBOL_GPL(debug_object_active_state);
752952 
....@@ -754,13 +954,12 @@
754954 static void __debug_check_no_obj_freed(const void *address, unsigned long size)
755955 {
756956 	unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
757
-	struct debug_obj_descr *descr;
957
+	const struct debug_obj_descr *descr;
758958 	enum debug_obj_state state;
759959 	struct debug_bucket *db;
760960 	struct hlist_node *tmp;
761961 	struct debug_obj *obj;
762962 	int cnt, objs_checked = 0;
763
-	bool work = false;
764963 
765964 	saddr = (unsigned long) address;
766965 	eaddr = saddr + size;
....@@ -782,16 +981,16 @@
782981 
783982 			switch (obj->state) {
784983 			case ODEBUG_STATE_ACTIVE:
785
-				debug_print_object(obj, "free");
786984 				descr = obj->descr;
787985 				state = obj->state;
788986 				raw_spin_unlock_irqrestore(&db->lock, flags);
987
+				debug_print_object(obj, "free");
789988 				debug_object_fixup(descr->fixup_free,
790989 						   (void *) oaddr, state);
791990 				goto repeat;
792991 			default:
793992 				hlist_del(&obj->node);
794
-				work |= __free_object(obj);
993
+				__free_object(obj);
795994 				break;
796995 			}
797996 		}
....@@ -807,8 +1006,10 @@
8071006 		debug_objects_maxchecked = objs_checked;
8081007 
8091008 	/* Schedule work to actually kmem_cache_free() objects */
810
-	if (work)
811
-		schedule_work(&debug_obj_work);
1009
+	if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
1010
+		WRITE_ONCE(obj_freeing, true);
1011
+		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
1012
+	}
8121013 }
8131014 
8141015 void debug_check_no_obj_freed(const void *address, unsigned long size)
....@@ -822,54 +1023,39 @@
8221023 
8231024 static int debug_stats_show(struct seq_file *m, void *v)
8241025 {
1026
+	int cpu, obj_percpu_free = 0;
1027
+
1028
+	for_each_possible_cpu(cpu)
1029
+		obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
1030
+
8251031 	seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
8261032 	seq_printf(m, "max_checked   :%d\n", debug_objects_maxchecked);
8271033 	seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
8281034 	seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
829
-	seq_printf(m, "pool_free     :%d\n", obj_pool_free);
1035
+	seq_printf(m, "pool_free     :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
1036
+	seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
8301037 	seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
831
-	seq_printf(m, "pool_used     :%d\n", obj_pool_used);
1038
+	seq_printf(m, "pool_used     :%d\n", obj_pool_used - obj_percpu_free);
8321039 	seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
833
-	seq_printf(m, "on_free_list  :%d\n", obj_nr_tofree);
1040
+	seq_printf(m, "on_free_list  :%d\n", READ_ONCE(obj_nr_tofree));
8341041 	seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
8351042 	seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
8361043 	return 0;
8371044 }
838
-
839
-static int debug_stats_open(struct inode *inode, struct file *filp)
840
-{
841
-	return single_open(filp, debug_stats_show, NULL);
842
-}
843
-
844
-static const struct file_operations debug_stats_fops = {
845
-	.open		= debug_stats_open,
846
-	.read		= seq_read,
847
-	.llseek		= seq_lseek,
848
-	.release	= single_release,
849
-};
1045
+DEFINE_SHOW_ATTRIBUTE(debug_stats);
8501046 
8511047 static int __init debug_objects_init_debugfs(void)
8521048 {
853
-	struct dentry *dbgdir, *dbgstats;
1049
+	struct dentry *dbgdir;
8541050 
8551051 	if (!debug_objects_enabled)
8561052 		return 0;
8571053 
8581054 	dbgdir = debugfs_create_dir("debug_objects", NULL);
859
-	if (!dbgdir)
860
-		return -ENOMEM;
8611055 
862
-	dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL,
863
-				       &debug_stats_fops);
864
-	if (!dbgstats)
865
-		goto err;
1056
+	debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
8661057 
8671058 	return 0;
868
-
869
-err:
870
-	debugfs_remove(dbgdir);
871
-
872
-	return -ENOMEM;
8731059 }
8741060 __initcall(debug_objects_init_debugfs);
8751061 
....@@ -886,7 +1072,7 @@
8861072 	unsigned long	dummy2[3];
8871073 };
8881074 
889
-static __initdata struct debug_obj_descr descr_type_test;
1075
+static __initconst const struct debug_obj_descr descr_type_test;
8901076 
8911077 static bool __init is_static_object(void *addr)
8921078 {
....@@ -1011,7 +1197,7 @@
10111197 	return res;
10121198 }
10131199 
1014
-static __initdata struct debug_obj_descr descr_type_test = {
1200
+static __initconst const struct debug_obj_descr descr_type_test = {
10151201 	.name			= "selftest",
10161202 	.is_static_object	= is_static_object,
10171203 	.fixup_init		= fixup_init,
....@@ -1131,11 +1317,10 @@
11311317 	}
11321318 
11331319 	/*
1134
-	 * When debug_objects_mem_init() is called we know that only
1135
-	 * one CPU is up, so disabling interrupts is enough
1136
-	 * protection. This avoids the lockdep hell of lock ordering.
1320
+	 * debug_objects_mem_init() is now called early that only one CPU is up
1321
+	 * and interrupts have been disabled, so it is safe to replace the
1322
+	 * active object references.
11371323 	 */
1138
-	local_irq_disable();
11391324 
11401325 	/* Remove the statically allocated objects from the pool */
11411326 	hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
....@@ -1156,7 +1341,6 @@
11561341 			cnt++;
11571342 		}
11581343 	}
1159
-	local_irq_enable();
11601344 
11611345 	pr_debug("%d of %d active objects replaced\n",
11621346 		 cnt, obj_pool_used);
....@@ -1177,8 +1361,19 @@
11771361  */
11781362 void __init debug_objects_mem_init(void)
11791363 {
1364
+	int cpu, extras;
1365
+
11801366 	if (!debug_objects_enabled)
11811367 		return;
1368
+
1369
+	/*
1370
+	 * Initialize the percpu object pools
1371
+	 *
1372
+	 * Initialization is not strictly necessary, but was done for
1373
+	 * completeness.
1374
+	 */
1375
+	for_each_possible_cpu(cpu)
1376
+		INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
11821377 
11831378 	obj_cache = kmem_cache_create("debug_objects_cache",
11841379 				      sizeof (struct debug_obj), 0,
....@@ -1192,10 +1387,16 @@
11921387 	} else
11931388 		debug_objects_selftest();
11941389 
1390
+#ifdef CONFIG_HOTPLUG_CPU
1391
+	cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
1392
+					object_cpu_offline);
1393
+#endif
1394
+
11951395 	/*
11961396 	 * Increase the thresholds for allocating and freeing objects
11971397 	 * according to the number of possible CPUs available in the system.
11981398 	 */
1199
-	debug_objects_pool_size += num_possible_cpus() * 32;
1200
-	debug_objects_pool_min_level += num_possible_cpus() * 4;
1399
+	extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
1400
+	debug_objects_pool_size += extras;
1401
+	debug_objects_pool_min_level += extras;
12011402 }