forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771 
 kernel/lib/debugobjects.c
....@@ -19,21 +19,43 @@
1919 #include <linux/slab.h>
2020 #include <linux/hash.h>
2121 #include <linux/kmemleak.h>
22
+#include <linux/cpu.h>
2223 
2324 #define ODEBUG_HASH_BITS	14
2425 #define ODEBUG_HASH_SIZE	(1 << ODEBUG_HASH_BITS)
2526 
2627 #define ODEBUG_POOL_SIZE	1024
2728 #define ODEBUG_POOL_MIN_LEVEL	256
29
+#define ODEBUG_POOL_PERCPU_SIZE	64
30
+#define ODEBUG_BATCH_SIZE	16
2831 
2932 #define ODEBUG_CHUNK_SHIFT	PAGE_SHIFT
3033 #define ODEBUG_CHUNK_SIZE	(1 << ODEBUG_CHUNK_SHIFT)
3134 #define ODEBUG_CHUNK_MASK	(~(ODEBUG_CHUNK_SIZE - 1))
3235 
36
+/*
37
+ * We limit the freeing of debug objects via workqueue at a maximum
38
+ * frequency of 10Hz and about 1024 objects for each freeing operation.
39
+ * So it is freeing at most 10k debug objects per second.
40
+ */
41
+#define ODEBUG_FREE_WORK_MAX	1024
42
+#define ODEBUG_FREE_WORK_DELAY	DIV_ROUND_UP(HZ, 10)
43
+
3344 struct debug_bucket {
3445 	struct hlist_head	list;
3546 	raw_spinlock_t		lock;
3647 };
48
+
49
+/*
50
+ * Debug object percpu free list
51
+ * Access is protected by disabling irq
52
+ */
53
+struct debug_percpu_free {
54
+	struct hlist_head	free_objs;
55
+	int			obj_free;
56
+};
57
+
58
+static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
3759 
3860 static struct debug_bucket	obj_hash[ODEBUG_HASH_SIZE];
3961 
....@@ -44,13 +66,20 @@
4466 static HLIST_HEAD(obj_pool);
4567 static HLIST_HEAD(obj_to_free);
4668 
69
+/*
70
+ * Because of the presence of percpu free pools, obj_pool_free will
71
+ * under-count those in the percpu free pools. Similarly, obj_pool_used
72
+ * will over-count those in the percpu free pools. Adjustments will be
73
+ * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
74
+ * can be off.
75
+ */
4776 static int			obj_pool_min_free = ODEBUG_POOL_SIZE;
4877 static int			obj_pool_free = ODEBUG_POOL_SIZE;
4978 static int			obj_pool_used;
5079 static int			obj_pool_max_used;
80
+static bool			obj_freeing;
5181 /* The number of objs on the global free list */
5282 static int			obj_nr_tofree;
53
-static struct kmem_cache	*obj_cache;
5483 
5584 static int			debug_objects_maxchain __read_mostly;
5685 static int __maybe_unused	debug_objects_maxchecked __read_mostly;
....@@ -62,7 +91,8 @@
6291 				= ODEBUG_POOL_SIZE;
6392 static int			debug_objects_pool_min_level __read_mostly
6493 				= ODEBUG_POOL_MIN_LEVEL;
65
-static struct debug_obj_descr	*descr_test  __read_mostly;
94
+static const struct debug_obj_descr *descr_test  __read_mostly;
95
+static struct kmem_cache	*obj_cache __read_mostly;
6696 
6797 /*
6898  * Track numbers of kmem_cache_alloc()/free() calls done.
....@@ -71,7 +101,7 @@
71101 static int			debug_objects_freed;
72102 
73103 static void free_obj_work(struct work_struct *work);
74
-static DECLARE_WORK(debug_obj_work, free_obj_work);
104
+static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
75105 
76106 static int __init enable_object_debug(char *str)
77107 {
....@@ -99,29 +129,33 @@
99129 
100130 static void fill_pool(void)
101131 {
102
-	gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
103
-	struct debug_obj *new, *obj;
132
+	gfp_t gfp = __GFP_HIGH | __GFP_NOWARN;
133
+	struct debug_obj *obj;
104134 	unsigned long flags;
105135 
106
-	if (likely(obj_pool_free >= debug_objects_pool_min_level))
136
+	if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
107137 		return;
108138 
109139 	/*
110140 	 * Reuse objs from the global free list; they will be reinitialized
111141 	 * when allocating.
142
+	 *
143
+	 * Both obj_nr_tofree and obj_pool_free are checked locklessly; the
144
+	 * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical
145
+	 * sections.
112146 	 */
113
-	while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
147
+	while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) {
114148 		raw_spin_lock_irqsave(&pool_lock, flags);
115149 		/*
116150 		 * Recheck with the lock held as the worker thread might have
117151 		 * won the race and freed the global free list already.
118152 		 */
119
-		if (obj_nr_tofree) {
153
+		while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
120154 			obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
121155 			hlist_del(&obj->node);
122
-			obj_nr_tofree--;
156
+			WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
123157 			hlist_add_head(&obj->node, &obj_pool);
124
-			obj_pool_free++;
158
+			WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
125159 		}
126160 		raw_spin_unlock_irqrestore(&pool_lock, flags);
127161 	}
....@@ -129,16 +163,24 @@
129163 	if (unlikely(!obj_cache))
130164 		return;
131165 
132
-	while (obj_pool_free < debug_objects_pool_min_level) {
166
+	while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
167
+		struct debug_obj *new[ODEBUG_BATCH_SIZE];
168
+		int cnt;
133169 
134
-		new = kmem_cache_zalloc(obj_cache, gfp);
135
-		if (!new)
170
+		for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
171
+			new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
172
+			if (!new[cnt])
173
+				break;
174
+		}
175
+		if (!cnt)
136176 			return;
137177 
138178 		raw_spin_lock_irqsave(&pool_lock, flags);
139
-		hlist_add_head(&new->node, &obj_pool);
140
-		debug_objects_allocated++;
141
-		obj_pool_free++;
179
+		while (cnt) {
180
+			hlist_add_head(&new[--cnt]->node, &obj_pool);
181
+			debug_objects_allocated++;
182
+			WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
183
+		}
142184 		raw_spin_unlock_irqrestore(&pool_lock, flags);
143185 	}
144186 }
....@@ -163,36 +205,77 @@
163205 }
164206 
165207 /*
166
- * Allocate a new object. If the pool is empty, switch off the debugger.
167
- * Must be called with interrupts disabled.
208
+ * Allocate a new object from the hlist
168209  */
169
-static struct debug_obj *
170
-alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
210
+static struct debug_obj *__alloc_object(struct hlist_head *list)
171211 {
172212 	struct debug_obj *obj = NULL;
173213 
174
-	raw_spin_lock(&pool_lock);
175
-	if (obj_pool.first) {
176
-		obj	    = hlist_entry(obj_pool.first, typeof(*obj), node);
177
-
178
-		obj->object = addr;
179
-		obj->descr  = descr;
180
-		obj->state  = ODEBUG_STATE_NONE;
181
-		obj->astate = 0;
214
+	if (list->first) {
215
+		obj = hlist_entry(list->first, typeof(*obj), node);
182216 		hlist_del(&obj->node);
217
+	}
183218 
184
-		hlist_add_head(&obj->node, &b->list);
219
+	return obj;
220
+}
185221 
222
+static struct debug_obj *
223
+alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
224
+{
225
+	struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
226
+	struct debug_obj *obj;
227
+
228
+	if (likely(obj_cache)) {
229
+		obj = __alloc_object(&percpu_pool->free_objs);
230
+		if (obj) {
231
+			percpu_pool->obj_free--;
232
+			goto init_obj;
233
+		}
234
+	}
235
+
236
+	raw_spin_lock(&pool_lock);
237
+	obj = __alloc_object(&obj_pool);
238
+	if (obj) {
186239 		obj_pool_used++;
240
+		WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
241
+
242
+		/*
243
+		 * Looking ahead, allocate one batch of debug objects and
244
+		 * put them into the percpu free pool.
245
+		 */
246
+		if (likely(obj_cache)) {
247
+			int i;
248
+
249
+			for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
250
+				struct debug_obj *obj2;
251
+
252
+				obj2 = __alloc_object(&obj_pool);
253
+				if (!obj2)
254
+					break;
255
+				hlist_add_head(&obj2->node,
256
+					       &percpu_pool->free_objs);
257
+				percpu_pool->obj_free++;
258
+				obj_pool_used++;
259
+				WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
260
+			}
261
+		}
262
+
187263 		if (obj_pool_used > obj_pool_max_used)
188264 			obj_pool_max_used = obj_pool_used;
189265 
190
-		obj_pool_free--;
191266 		if (obj_pool_free < obj_pool_min_free)
192267 			obj_pool_min_free = obj_pool_free;
193268 	}
194269 	raw_spin_unlock(&pool_lock);
195270 
271
+init_obj:
272
+	if (obj) {
273
+		obj->object = addr;
274
+		obj->descr  = descr;
275
+		obj->state  = ODEBUG_STATE_NONE;
276
+		obj->astate = 0;
277
+		hlist_add_head(&obj->node, &b->list);
278
+	}
196279 	return obj;
197280 }
198281 
....@@ -209,22 +292,31 @@
209292 	unsigned long flags;
210293 	HLIST_HEAD(tofree);
211294 
295
+	WRITE_ONCE(obj_freeing, false);
212296 	if (!raw_spin_trylock_irqsave(&pool_lock, flags))
213297 		return;
298
+
299
+	if (obj_pool_free >= debug_objects_pool_size)
300
+		goto free_objs;
214301 
215302 	/*
216303 	 * The objs on the pool list might be allocated before the work is
217304 	 * run, so recheck if pool list it full or not, if not fill pool
218
-	 * list from the global free list
305
+	 * list from the global free list. As it is likely that a workload
306
+	 * may be gearing up to use more and more objects, don't free any
307
+	 * of them until the next round.
219308 	 */
220309 	while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
221310 		obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
222311 		hlist_del(&obj->node);
223312 		hlist_add_head(&obj->node, &obj_pool);
224
-		obj_pool_free++;
225
-		obj_nr_tofree--;
313
+		WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
314
+		WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
226315 	}
316
+	raw_spin_unlock_irqrestore(&pool_lock, flags);
317
+	return;
227318 
319
+free_objs:
228320 	/*
229321 	 * Pool list is already full and there are still objs on the free
230322 	 * list. Move remaining free objs to a temporary list to free the
....@@ -233,7 +325,7 @@
233325 	if (obj_nr_tofree) {
234326 		hlist_move_list(&obj_to_free, &tofree);
235327 		debug_objects_freed += obj_nr_tofree;
236
-		obj_nr_tofree = 0;
328
+		WRITE_ONCE(obj_nr_tofree, 0);
237329 	}
238330 	raw_spin_unlock_irqrestore(&pool_lock, flags);
239331 
....@@ -243,24 +335,86 @@
243335 	}
244336 }
245337 
246
-static bool __free_object(struct debug_obj *obj)
338
+static void __free_object(struct debug_obj *obj)
247339 {
340
+	struct debug_obj *objs[ODEBUG_BATCH_SIZE];
341
+	struct debug_percpu_free *percpu_pool;
342
+	int lookahead_count = 0;
248343 	unsigned long flags;
249344 	bool work;
250345 
251
-	raw_spin_lock_irqsave(&pool_lock, flags);
252
-	work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
346
+	local_irq_save(flags);
347
+	if (!obj_cache)
348
+		goto free_to_obj_pool;
349
+
350
+	/*
351
+	 * Try to free it into the percpu pool first.
352
+	 */
353
+	percpu_pool = this_cpu_ptr(&percpu_obj_pool);
354
+	if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
355
+		hlist_add_head(&obj->node, &percpu_pool->free_objs);
356
+		percpu_pool->obj_free++;
357
+		local_irq_restore(flags);
358
+		return;
359
+	}
360
+
361
+	/*
362
+	 * As the percpu pool is full, look ahead and pull out a batch
363
+	 * of objects from the percpu pool and free them as well.
364
+	 */
365
+	for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
366
+		objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
367
+		if (!objs[lookahead_count])
368
+			break;
369
+		percpu_pool->obj_free--;
370
+	}
371
+
372
+free_to_obj_pool:
373
+	raw_spin_lock(&pool_lock);
374
+	work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
375
+	       (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
253376 	obj_pool_used--;
254377 
255378 	if (work) {
256
-		obj_nr_tofree++;
379
+		WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
257380 		hlist_add_head(&obj->node, &obj_to_free);
381
+		if (lookahead_count) {
382
+			WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
383
+			obj_pool_used -= lookahead_count;
384
+			while (lookahead_count) {
385
+				hlist_add_head(&objs[--lookahead_count]->node,
386
+					       &obj_to_free);
387
+			}
388
+		}
389
+
390
+		if ((obj_pool_free > debug_objects_pool_size) &&
391
+		    (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
392
+			int i;
393
+
394
+			/*
395
+			 * Free one more batch of objects from obj_pool.
396
+			 */
397
+			for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
398
+				obj = __alloc_object(&obj_pool);
399
+				hlist_add_head(&obj->node, &obj_to_free);
400
+				WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
401
+				WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
402
+			}
403
+		}
258404 	} else {
259
-		obj_pool_free++;
405
+		WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
260406 		hlist_add_head(&obj->node, &obj_pool);
407
+		if (lookahead_count) {
408
+			WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
409
+			obj_pool_used -= lookahead_count;
410
+			while (lookahead_count) {
411
+				hlist_add_head(&objs[--lookahead_count]->node,
412
+					       &obj_pool);
413
+			}
414
+		}
261415 	}
262
-	raw_spin_unlock_irqrestore(&pool_lock, flags);
263
-	return work;
416
+	raw_spin_unlock(&pool_lock);
417
+	local_irq_restore(flags);
264418 }
265419 
266420 /*
....@@ -269,9 +423,38 @@
269423  */
270424 static void free_object(struct debug_obj *obj)
271425 {
272
-	if (__free_object(obj))
273
-		schedule_work(&debug_obj_work);
426
+	__free_object(obj);
427
+	if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
428
+		WRITE_ONCE(obj_freeing, true);
429
+		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
430
+	}
274431 }
432
+
433
+#ifdef CONFIG_HOTPLUG_CPU
434
+static int object_cpu_offline(unsigned int cpu)
435
+{
436
+	struct debug_percpu_free *percpu_pool;
437
+	struct hlist_node *tmp;
438
+	struct debug_obj *obj;
439
+	unsigned long flags;
440
+
441
+	/* Remote access is safe as the CPU is dead already */
442
+	percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu);
443
+	hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) {
444
+		hlist_del(&obj->node);
445
+		kmem_cache_free(obj_cache, obj);
446
+	}
447
+
448
+	raw_spin_lock_irqsave(&pool_lock, flags);
449
+	obj_pool_used -= percpu_pool->obj_free;
450
+	debug_objects_freed += percpu_pool->obj_free;
451
+	raw_spin_unlock_irqrestore(&pool_lock, flags);
452
+
453
+	percpu_pool->obj_free = 0;
454
+
455
+	return 0;
456
+}
457
+#endif
275458 
276459 /*
277460  * We run out of memory. That means we probably have tons of objects
....@@ -315,8 +498,17 @@
315498 
316499 static void debug_print_object(struct debug_obj *obj, char *msg)
317500 {
318
-	struct debug_obj_descr *descr = obj->descr;
501
+	const struct debug_obj_descr *descr = obj->descr;
319502 	static int limit;
503
+
504
+	/*
505
+	 * Don't report if lookup_object_or_alloc() by the current thread
506
+	 * failed because lookup_object_or_alloc()/debug_objects_oom() by a
507
+	 * concurrent thread turned off debug_objects_enabled and cleared
508
+	 * the hash buckets.
509
+	 */
510
+	if (!debug_objects_enabled)
511
+		return;
320512 
321513 	if (limit < 5 && descr != descr_test) {
322514 		void *hint = descr->debug_hint ?
....@@ -368,33 +560,74 @@
368560 	WARN_ON(1);
369561 }
370562 
563
+static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket *b,
564
+						const struct debug_obj_descr *descr,
565
+						bool onstack, bool alloc_ifstatic)
566
+{
567
+	struct debug_obj *obj = lookup_object(addr, b);
568
+	enum debug_obj_state state = ODEBUG_STATE_NONE;
569
+
570
+	if (likely(obj))
571
+		return obj;
572
+
573
+	/*
574
+	 * debug_object_init() unconditionally allocates untracked
575
+	 * objects. It does not matter whether it is a static object or
576
+	 * not.
577
+	 *
578
+	 * debug_object_assert_init() and debug_object_activate() allow
579
+	 * allocation only if the descriptor callback confirms that the
580
+	 * object is static and considered initialized. For non-static
581
+	 * objects the allocation needs to be done from the fixup callback.
582
+	 */
583
+	if (unlikely(alloc_ifstatic)) {
584
+		if (!descr->is_static_object || !descr->is_static_object(addr))
585
+			return ERR_PTR(-ENOENT);
586
+		/* Statically allocated objects are considered initialized */
587
+		state = ODEBUG_STATE_INIT;
588
+	}
589
+
590
+	obj = alloc_object(addr, b, descr);
591
+	if (likely(obj)) {
592
+		obj->state = state;
593
+		debug_object_is_on_stack(addr, onstack);
594
+		return obj;
595
+	}
596
+
597
+	/* Out of memory. Do the cleanup outside of the locked region */
598
+	debug_objects_enabled = 0;
599
+	return NULL;
600
+}
601
+
602
+static void debug_objects_fill_pool(void)
603
+{
604
+	/*
605
+	 * On RT enabled kernels the pool refill must happen in preemptible
606
+	 * context:
607
+	 */
608
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible())
609
+		fill_pool();
610
+}
611
+
371612 static void
372
-__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
613
+__debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack)
373614 {
374615 	enum debug_obj_state state;
375616 	struct debug_bucket *db;
376617 	struct debug_obj *obj;
377618 	unsigned long flags;
378619 
379
-#ifdef CONFIG_PREEMPT_RT_FULL
380
-	if (preempt_count() == 0 && !irqs_disabled())
381
-#endif
382
-		fill_pool();
620
+	debug_objects_fill_pool();
383621 
384622 	db = get_bucket((unsigned long) addr);
385623 
386624 	raw_spin_lock_irqsave(&db->lock, flags);
387625 
388
-	obj = lookup_object(addr, db);
389
-	if (!obj) {
390
-		obj = alloc_object(addr, db, descr);
391
-		if (!obj) {
392
-			debug_objects_enabled = 0;
393
-			raw_spin_unlock_irqrestore(&db->lock, flags);
394
-			debug_objects_oom();
395
-			return;
396
-		}
397
-		debug_object_is_on_stack(addr, onstack);
626
+	obj = lookup_object_or_alloc(addr, db, descr, onstack, false);
627
+	if (unlikely(!obj)) {
628
+		raw_spin_unlock_irqrestore(&db->lock, flags);
629
+		debug_objects_oom();
630
+		return;
398631 	}
399632 
400633 	switch (obj->state) {
....@@ -405,15 +638,16 @@
405638 		break;
406639 
407640 	case ODEBUG_STATE_ACTIVE:
408
-		debug_print_object(obj, "init");
409641 		state = obj->state;
410642 		raw_spin_unlock_irqrestore(&db->lock, flags);
643
+		debug_print_object(obj, "init");
411644 		debug_object_fixup(descr->fixup_init, addr, state);
412645 		return;
413646 
414647 	case ODEBUG_STATE_DESTROYED:
648
+		raw_spin_unlock_irqrestore(&db->lock, flags);
415649 		debug_print_object(obj, "init");
416
-		break;
650
+		return;
417651 	default:
418652 		break;
419653 	}
....@@ -426,7 +660,7 @@
426660  * @addr:	address of the object
427661  * @descr:	pointer to an object specific debug description structure
428662  */
429
-void debug_object_init(void *addr, struct debug_obj_descr *descr)
663
+void debug_object_init(void *addr, const struct debug_obj_descr *descr)
430664 {
431665 	if (!debug_objects_enabled)
432666 		return;
....@@ -441,7 +675,7 @@
441675  * @addr:	address of the object
442676  * @descr:	pointer to an object specific debug description structure
443677  */
444
-void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
678
+void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr)
445679 {
446680 	if (!debug_objects_enabled)
447681 		return;
....@@ -456,26 +690,28 @@
456690  * @descr:	pointer to an object specific debug description structure
457691  * Returns 0 for success, -EINVAL for check failed.
458692  */
459
-int debug_object_activate(void *addr, struct debug_obj_descr *descr)
693
+int debug_object_activate(void *addr, const struct debug_obj_descr *descr)
460694 {
695
+	struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr };
461696 	enum debug_obj_state state;
462697 	struct debug_bucket *db;
463698 	struct debug_obj *obj;
464699 	unsigned long flags;
465700 	int ret;
466
-	struct debug_obj o = { .object = addr,
467
-			       .state = ODEBUG_STATE_NOTAVAILABLE,
468
-			       .descr = descr };
469701 
470702 	if (!debug_objects_enabled)
471703 		return 0;
704
+
705
+	debug_objects_fill_pool();
472706 
473707 	db = get_bucket((unsigned long) addr);
474708 
475709 	raw_spin_lock_irqsave(&db->lock, flags);
476710 
477
-	obj = lookup_object(addr, db);
478
-	if (obj) {
711
+	obj = lookup_object_or_alloc(addr, db, descr, false, true);
712
+	if (likely(!IS_ERR_OR_NULL(obj))) {
713
+		bool print_object = false;
714
+
479715 		switch (obj->state) {
480716 		case ODEBUG_STATE_INIT:
481717 		case ODEBUG_STATE_INACTIVE:
....@@ -484,14 +720,14 @@
484720 			break;
485721 
486722 		case ODEBUG_STATE_ACTIVE:
487
-			debug_print_object(obj, "activate");
488723 			state = obj->state;
489724 			raw_spin_unlock_irqrestore(&db->lock, flags);
725
+			debug_print_object(obj, "activate");
490726 			ret = debug_object_fixup(descr->fixup_activate, addr, state);
491727 			return ret ? 0 : -EINVAL;
492728 
493729 		case ODEBUG_STATE_DESTROYED:
494
-			debug_print_object(obj, "activate");
730
+			print_object = true;
495731 			ret = -EINVAL;
496732 			break;
497733 		default:
....@@ -499,28 +735,23 @@
499735 			break;
500736 		}
501737 		raw_spin_unlock_irqrestore(&db->lock, flags);
738
+		if (print_object)
739
+			debug_print_object(obj, "activate");
502740 		return ret;
503741 	}
504742 
505743 	raw_spin_unlock_irqrestore(&db->lock, flags);
506
-	/*
507
-	 * We are here when a static object is activated. We
508
-	 * let the type specific code confirm whether this is
509
-	 * true or not. if true, we just make sure that the
510
-	 * static object is tracked in the object tracker. If
511
-	 * not, this must be a bug, so we try to fix it up.
512
-	 */
513
-	if (descr->is_static_object && descr->is_static_object(addr)) {
514
-		/* track this static object */
515
-		debug_object_init(addr, descr);
516
-		debug_object_activate(addr, descr);
517
-	} else {
518
-		debug_print_object(&o, "activate");
519
-		ret = debug_object_fixup(descr->fixup_activate, addr,
520
-					ODEBUG_STATE_NOTAVAILABLE);
521
-		return ret ? 0 : -EINVAL;
744
+
745
+	/* If NULL the allocation has hit OOM */
746
+	if (!obj) {
747
+		debug_objects_oom();
748
+		return 0;
522749 	}
523
-	return 0;
750
+
751
+	/* Object is neither static nor tracked. It's not initialized */
752
+	debug_print_object(&o, "activate");
753
+	ret = debug_object_fixup(descr->fixup_activate, addr, ODEBUG_STATE_NOTAVAILABLE);
754
+	return ret ? 0 : -EINVAL;
524755 }
525756 EXPORT_SYMBOL_GPL(debug_object_activate);
526757 
....@@ -529,11 +760,12 @@
529760  * @addr:	address of the object
530761  * @descr:	pointer to an object specific debug description structure
531762  */
532
-void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
763
+void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr)
533764 {
534765 	struct debug_bucket *db;
535766 	struct debug_obj *obj;
536767 	unsigned long flags;
768
+	bool print_object = false;
537769 
538770 	if (!debug_objects_enabled)
539771 		return;
....@@ -551,24 +783,27 @@
551783 			if (!obj->astate)
552784 				obj->state = ODEBUG_STATE_INACTIVE;
553785 			else
554
-				debug_print_object(obj, "deactivate");
786
+				print_object = true;
555787 			break;
556788 
557789 		case ODEBUG_STATE_DESTROYED:
558
-			debug_print_object(obj, "deactivate");
790
+			print_object = true;
559791 			break;
560792 		default:
561793 			break;
562794 		}
563
-	} else {
795
+	}
796
+
797
+	raw_spin_unlock_irqrestore(&db->lock, flags);
798
+	if (!obj) {
564799 		struct debug_obj o = { .object = addr,
565800 				       .state = ODEBUG_STATE_NOTAVAILABLE,
566801 				       .descr = descr };
567802 
568803 		debug_print_object(&o, "deactivate");
804
+	} else if (print_object) {
805
+		debug_print_object(obj, "deactivate");
569806 	}
570
-
571
-	raw_spin_unlock_irqrestore(&db->lock, flags);
572807 }
573808 EXPORT_SYMBOL_GPL(debug_object_deactivate);
574809 
....@@ -577,12 +812,13 @@
577812  * @addr:	address of the object
578813  * @descr:	pointer to an object specific debug description structure
579814  */
580
-void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
815
+void debug_object_destroy(void *addr, const struct debug_obj_descr *descr)
581816 {
582817 	enum debug_obj_state state;
583818 	struct debug_bucket *db;
584819 	struct debug_obj *obj;
585820 	unsigned long flags;
821
+	bool print_object = false;
586822 
587823 	if (!debug_objects_enabled)
588824 		return;
....@@ -602,20 +838,22 @@
602838 		obj->state = ODEBUG_STATE_DESTROYED;
603839 		break;
604840 	case ODEBUG_STATE_ACTIVE:
605
-		debug_print_object(obj, "destroy");
606841 		state = obj->state;
607842 		raw_spin_unlock_irqrestore(&db->lock, flags);
843
+		debug_print_object(obj, "destroy");
608844 		debug_object_fixup(descr->fixup_destroy, addr, state);
609845 		return;
610846 
611847 	case ODEBUG_STATE_DESTROYED:
612
-		debug_print_object(obj, "destroy");
848
+		print_object = true;
613849 		break;
614850 	default:
615851 		break;
616852 	}
617853 out_unlock:
618854 	raw_spin_unlock_irqrestore(&db->lock, flags);
855
+	if (print_object)
856
+		debug_print_object(obj, "destroy");
619857 }
620858 EXPORT_SYMBOL_GPL(debug_object_destroy);
621859 
....@@ -624,7 +862,7 @@
624862  * @addr:	address of the object
625863  * @descr:	pointer to an object specific debug description structure
626864  */
627
-void debug_object_free(void *addr, struct debug_obj_descr *descr)
865
+void debug_object_free(void *addr, const struct debug_obj_descr *descr)
628866 {
629867 	enum debug_obj_state state;
630868 	struct debug_bucket *db;
....@@ -644,9 +882,9 @@
644882 
645883 	switch (obj->state) {
646884 	case ODEBUG_STATE_ACTIVE:
647
-		debug_print_object(obj, "free");
648885 		state = obj->state;
649886 		raw_spin_unlock_irqrestore(&db->lock, flags);
887
+		debug_print_object(obj, "free");
650888 		debug_object_fixup(descr->fixup_free, addr, state);
651889 		return;
652890 	default:
....@@ -665,8 +903,9 @@
665903  * @addr:	address of the object
666904  * @descr:	pointer to an object specific debug description structure
667905  */
668
-void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
906
+void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr)
669907 {
908
+	struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr };
670909 	struct debug_bucket *db;
671910 	struct debug_obj *obj;
672911 	unsigned long flags;
....@@ -674,34 +913,25 @@
674913 	if (!debug_objects_enabled)
675914 		return;
676915 
916
+	debug_objects_fill_pool();
917
+
677918 	db = get_bucket((unsigned long) addr);
678919 
679920 	raw_spin_lock_irqsave(&db->lock, flags);
921
+	obj = lookup_object_or_alloc(addr, db, descr, false, true);
922
+	raw_spin_unlock_irqrestore(&db->lock, flags);
923
+	if (likely(!IS_ERR_OR_NULL(obj)))
924
+		return;
680925 
681
-	obj = lookup_object(addr, db);
926
+	/* If NULL the allocation has hit OOM */
682927 	if (!obj) {
683
-		struct debug_obj o = { .object = addr,
684
-				       .state = ODEBUG_STATE_NOTAVAILABLE,
685
-				       .descr = descr };
686
-
687
-		raw_spin_unlock_irqrestore(&db->lock, flags);
688
-		/*
689
-		 * Maybe the object is static, and we let the type specific
690
-		 * code confirm. Track this static object if true, else invoke
691
-		 * fixup.
692
-		 */
693
-		if (descr->is_static_object && descr->is_static_object(addr)) {
694
-			/* Track this static object */
695
-			debug_object_init(addr, descr);
696
-		} else {
697
-			debug_print_object(&o, "assert_init");
698
-			debug_object_fixup(descr->fixup_assert_init, addr,
699
-					   ODEBUG_STATE_NOTAVAILABLE);
700
-		}
928
+		debug_objects_oom();
701929 		return;
702930 	}
703931 
704
-	raw_spin_unlock_irqrestore(&db->lock, flags);
932
+	/* Object is neither tracked nor static. It's not initialized. */
933
+	debug_print_object(&o, "assert_init");
934
+	debug_object_fixup(descr->fixup_assert_init, addr, ODEBUG_STATE_NOTAVAILABLE);
705935 }
706936 EXPORT_SYMBOL_GPL(debug_object_assert_init);
707937 
....@@ -713,12 +943,13 @@
713943  * @next:	state to move to if expected state is found
714944  */
715945 void
716
-debug_object_active_state(void *addr, struct debug_obj_descr *descr,
946
+debug_object_active_state(void *addr, const struct debug_obj_descr *descr,
717947 			  unsigned int expect, unsigned int next)
718948 {
719949 	struct debug_bucket *db;
720950 	struct debug_obj *obj;
721951 	unsigned long flags;
952
+	bool print_object = false;
722953 
723954 	if (!debug_objects_enabled)
724955 		return;
....@@ -734,22 +965,25 @@
734965 			if (obj->astate == expect)
735966 				obj->astate = next;
736967 			else
737
-				debug_print_object(obj, "active_state");
968
+				print_object = true;
738969 			break;
739970 
740971 		default:
741
-			debug_print_object(obj, "active_state");
972
+			print_object = true;
742973 			break;
743974 		}
744
-	} else {
975
+	}
976
+
977
+	raw_spin_unlock_irqrestore(&db->lock, flags);
978
+	if (!obj) {
745979 		struct debug_obj o = { .object = addr,
746980 				       .state = ODEBUG_STATE_NOTAVAILABLE,
747981 				       .descr = descr };
748982 
749983 		debug_print_object(&o, "active_state");
984
+	} else if (print_object) {
985
+		debug_print_object(obj, "active_state");
750986 	}
751
-
752
-	raw_spin_unlock_irqrestore(&db->lock, flags);
753987 }
754988 EXPORT_SYMBOL_GPL(debug_object_active_state);
755989 
....@@ -757,13 +991,12 @@
757991 static void __debug_check_no_obj_freed(const void *address, unsigned long size)
758992 {
759993 	unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
760
-	struct debug_obj_descr *descr;
994
+	const struct debug_obj_descr *descr;
761995 	enum debug_obj_state state;
762996 	struct debug_bucket *db;
763997 	struct hlist_node *tmp;
764998 	struct debug_obj *obj;
765999 	int cnt, objs_checked = 0;
766
-	bool work = false;
7671000 
7681001 	saddr = (unsigned long) address;
7691002 	eaddr = saddr + size;
....@@ -785,16 +1018,16 @@
7851018 
7861019 			switch (obj->state) {
7871020 			case ODEBUG_STATE_ACTIVE:
788
-				debug_print_object(obj, "free");
7891021 				descr = obj->descr;
7901022 				state = obj->state;
7911023 				raw_spin_unlock_irqrestore(&db->lock, flags);
1024
+				debug_print_object(obj, "free");
7921025 				debug_object_fixup(descr->fixup_free,
7931026 						   (void *) oaddr, state);
7941027 				goto repeat;
7951028 			default:
7961029 				hlist_del(&obj->node);
797
-				work |= __free_object(obj);
1030
+				__free_object(obj);
7981031 				break;
7991032 			}
8001033 		}
....@@ -810,8 +1043,10 @@
8101043 		debug_objects_maxchecked = objs_checked;
8111044 
8121045 	/* Schedule work to actually kmem_cache_free() objects */
813
-	if (work)
814
-		schedule_work(&debug_obj_work);
1046
+	if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
1047
+		WRITE_ONCE(obj_freeing, true);
1048
+		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
1049
+	}
8151050 }
8161051 
8171052 void debug_check_no_obj_freed(const void *address, unsigned long size)
....@@ -825,54 +1060,39 @@
8251060 
8261061 static int debug_stats_show(struct seq_file *m, void *v)
8271062 {
1063
+	int cpu, obj_percpu_free = 0;
1064
+
1065
+	for_each_possible_cpu(cpu)
1066
+		obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
1067
+
8281068 	seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
8291069 	seq_printf(m, "max_checked   :%d\n", debug_objects_maxchecked);
8301070 	seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
8311071 	seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
832
-	seq_printf(m, "pool_free     :%d\n", obj_pool_free);
1072
+	seq_printf(m, "pool_free     :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
1073
+	seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
8331074 	seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
834
-	seq_printf(m, "pool_used     :%d\n", obj_pool_used);
1075
+	seq_printf(m, "pool_used     :%d\n", obj_pool_used - obj_percpu_free);
8351076 	seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
836
-	seq_printf(m, "on_free_list  :%d\n", obj_nr_tofree);
1077
+	seq_printf(m, "on_free_list  :%d\n", READ_ONCE(obj_nr_tofree));
8371078 	seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
8381079 	seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
8391080 	return 0;
8401081 }
841
-
842
-static int debug_stats_open(struct inode *inode, struct file *filp)
843
-{
844
-	return single_open(filp, debug_stats_show, NULL);
845
-}
846
-
847
-static const struct file_operations debug_stats_fops = {
848
-	.open		= debug_stats_open,
849
-	.read		= seq_read,
850
-	.llseek		= seq_lseek,
851
-	.release	= single_release,
852
-};
1082
+DEFINE_SHOW_ATTRIBUTE(debug_stats);
8531083 
8541084 static int __init debug_objects_init_debugfs(void)
8551085 {
856
-	struct dentry *dbgdir, *dbgstats;
1086
+	struct dentry *dbgdir;
8571087 
8581088 	if (!debug_objects_enabled)
8591089 		return 0;
8601090 
8611091 	dbgdir = debugfs_create_dir("debug_objects", NULL);
862
-	if (!dbgdir)
863
-		return -ENOMEM;
8641092 
865
-	dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL,
866
-				       &debug_stats_fops);
867
-	if (!dbgstats)
868
-		goto err;
1093
+	debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
8691094 
8701095 	return 0;
871
-
872
-err:
873
-	debugfs_remove(dbgdir);
874
-
875
-	return -ENOMEM;
8761096 }
8771097 __initcall(debug_objects_init_debugfs);
8781098 
....@@ -889,7 +1109,7 @@
8891109 	unsigned long	dummy2[3];
8901110 };
8911111 
892
-static __initdata struct debug_obj_descr descr_type_test;
1112
+static __initconst const struct debug_obj_descr descr_type_test;
8931113 
8941114 static bool __init is_static_object(void *addr)
8951115 {
....@@ -1014,7 +1234,7 @@
10141234 	return res;
10151235 }
10161236 
1017
-static __initdata struct debug_obj_descr descr_type_test = {
1237
+static __initconst const struct debug_obj_descr descr_type_test = {
10181238 	.name			= "selftest",
10191239 	.is_static_object	= is_static_object,
10201240 	.fixup_init		= fixup_init,
....@@ -1133,12 +1353,13 @@
11331353 		hlist_add_head(&obj->node, &objects);
11341354 	}
11351355 
1356
+	debug_objects_allocated += i;
1357
+
11361358 	/*
1137
-	 * When debug_objects_mem_init() is called we know that only
1138
-	 * one CPU is up, so disabling interrupts is enough
1139
-	 * protection. This avoids the lockdep hell of lock ordering.
1359
+	 * debug_objects_mem_init() is now called early that only one CPU is up
1360
+	 * and interrupts have been disabled, so it is safe to replace the
1361
+	 * active object references.
11401362 	 */
1141
-	local_irq_disable();
11421363 
11431364 	/* Remove the statically allocated objects from the pool */
11441365 	hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
....@@ -1159,7 +1380,6 @@
11591380 			cnt++;
11601381 		}
11611382 	}
1162
-	local_irq_enable();
11631383 
11641384 	pr_debug("%d of %d active objects replaced\n",
11651385 		 cnt, obj_pool_used);
....@@ -1180,8 +1400,19 @@
11801400  */
11811401 void __init debug_objects_mem_init(void)
11821402 {
1403
+	int cpu, extras;
1404
+
11831405 	if (!debug_objects_enabled)
11841406 		return;
1407
+
1408
+	/*
1409
+	 * Initialize the percpu object pools
1410
+	 *
1411
+	 * Initialization is not strictly necessary, but was done for
1412
+	 * completeness.
1413
+	 */
1414
+	for_each_possible_cpu(cpu)
1415
+		INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
11851416 
11861417 	obj_cache = kmem_cache_create("debug_objects_cache",
11871418 				      sizeof (struct debug_obj), 0,
....@@ -1192,13 +1423,20 @@
11921423 		debug_objects_enabled = 0;
11931424 		kmem_cache_destroy(obj_cache);
11941425 		pr_warn("out of memory.\n");
1426
+		return;
11951427 	} else
11961428 		debug_objects_selftest();
1429
+
1430
+#ifdef CONFIG_HOTPLUG_CPU
1431
+	cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
1432
+					object_cpu_offline);
1433
+#endif
11971434 
11981435 	/*
11991436 	 * Increase the thresholds for allocating and freeing objects
12001437 	 * according to the number of possible CPUs available in the system.
12011438 	 */
1202
-	debug_objects_pool_size += num_possible_cpus() * 32;
1203
-	debug_objects_pool_min_level += num_possible_cpus() * 4;
1439
+	extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
1440
+	debug_objects_pool_size += extras;
1441
+	debug_objects_pool_min_level += extras;
12041442 }