forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/mm/slob.c
....@@ -112,13 +112,13 @@
112112 
113113 static void set_slob_page_free(struct page *sp, struct list_head *list)
114114 {
115
-	list_add(&sp->lru, list);
115
+	list_add(&sp->slab_list, list);
116116 	__SetPageSlobFree(sp);
117117 }
118118 
119119 static inline void clear_slob_page_free(struct page *sp)
120120 {
121
-	list_del(&sp->lru);
121
+	list_del(&sp->slab_list);
122122 	__ClearPageSlobFree(sp);
123123 }
124124 
....@@ -190,7 +190,7 @@
190190 
191191 static void *slob_new_pages(gfp_t gfp, int order, int node)
192192 {
193
-	void *page;
193
+	struct page *page;
194194 
195195 #ifdef CONFIG_NUMA
196196 	if (node != NUMA_NO_NODE)
....@@ -202,29 +202,59 @@
202202 	if (!page)
203203 		return NULL;
204204 
205
+	mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B,
206
+			    PAGE_SIZE << order);
205207 	return page_address(page);
206208 }
207209 
208210 static void slob_free_pages(void *b, int order)
209211 {
212
+	struct page *sp = virt_to_page(b);
213
+
210214 	if (current->reclaim_state)
211215 		current->reclaim_state->reclaimed_slab += 1 << order;
212
-	free_pages((unsigned long)b, order);
216
+
217
+	mod_node_page_state(page_pgdat(sp), NR_SLAB_UNRECLAIMABLE_B,
218
+			    -(PAGE_SIZE << order));
219
+	__free_pages(sp, order);
213220 }
214221 
215222 /*
216
- * Allocate a slob block within a given slob_page sp.
223
+ * slob_page_alloc() - Allocate a slob block within a given slob_page sp.
224
+ * @sp: Page to look in.
225
+ * @size: Size of the allocation.
226
+ * @align: Allocation alignment.
227
+ * @align_offset: Offset in the allocated block that will be aligned.
228
+ * @page_removed_from_list: Return parameter.
229
+ *
230
+ * Tries to find a chunk of memory at least @size bytes big within @page.
231
+ *
232
+ * Return: Pointer to memory if allocated, %NULL otherwise.  If the
233
+ *         allocation fills up @page then the page is removed from the
234
+ *         freelist, in this case @page_removed_from_list will be set to
235
+ *         true (set to false otherwise).
217236  */
218
-static void *slob_page_alloc(struct page *sp, size_t size, int align)
237
+static void *slob_page_alloc(struct page *sp, size_t size, int align,
238
+			      int align_offset, bool *page_removed_from_list)
219239 {
220240 	slob_t *prev, *cur, *aligned = NULL;
221241 	int delta = 0, units = SLOB_UNITS(size);
222242 
243
+	*page_removed_from_list = false;
223244 	for (prev = NULL, cur = sp->freelist; ; prev = cur, cur = slob_next(cur)) {
224245 		slobidx_t avail = slob_units(cur);
225246 
247
+		/*
248
+		 * 'aligned' will hold the address of the slob block so that the
249
+		 * address 'aligned'+'align_offset' is aligned according to the
250
+		 * 'align' parameter. This is for kmalloc() which prepends the
251
+		 * allocated block with its size, so that the block itself is
252
+		 * aligned when needed.
253
+		 */
226254 		if (align) {
227
-			aligned = (slob_t *)ALIGN((unsigned long)cur, align);
255
+			aligned = (slob_t *)
256
+				(ALIGN((unsigned long)cur + align_offset, align)
257
+				 - align_offset);
228258 			delta = aligned - cur;
229259 		}
230260 		if (avail >= units + delta) { /* room enough? */
....@@ -254,8 +284,10 @@
254284 			}
255285 
256286 			sp->units -= units;
257
-			if (!sp->units)
287
+			if (!sp->units) {
258288 				clear_slob_page_free(sp);
289
+				*page_removed_from_list = true;
290
+			}
259291 			return cur;
260292 		}
261293 		if (slob_last(cur))
....@@ -266,13 +298,14 @@
266298 /*
267299  * slob_alloc: entry point into the slob allocator.
268300  */
269
-static void *slob_alloc(size_t size, gfp_t gfp, int align, int node)
301
+static void *slob_alloc(size_t size, gfp_t gfp, int align, int node,
302
+							int align_offset)
270303 {
271304 	struct page *sp;
272
-	struct list_head *prev;
273305 	struct list_head *slob_list;
274306 	slob_t *b = NULL;
275307 	unsigned long flags;
308
+	bool _unused;
276309 
277310 	if (size < SLOB_BREAK1)
278311 		slob_list = &free_slob_small;
....@@ -283,7 +316,8 @@
283316 
284317 	spin_lock_irqsave(&slob_lock, flags);
285318 	/* Iterate through each partially free page, try to find room */
286
-	list_for_each_entry(sp, slob_list, lru) {
319
+	list_for_each_entry(sp, slob_list, slab_list) {
320
+		bool page_removed_from_list = false;
287321 #ifdef CONFIG_NUMA
288322 		/*
289323 		 * If there's a node specification, search for a partial
....@@ -296,18 +330,25 @@
296330 		if (sp->units < SLOB_UNITS(size))
297331 			continue;
298332 
299
-		/* Attempt to alloc */
300
-		prev = sp->lru.prev;
301
-		b = slob_page_alloc(sp, size, align);
333
+		b = slob_page_alloc(sp, size, align, align_offset, &page_removed_from_list);
302334 		if (!b)
303335 			continue;
304336 
305
-		/* Improve fragment distribution and reduce our average
306
-		 * search time by starting our next search here. (see
307
-		 * Knuth vol 1, sec 2.5, pg 449) */
308
-		if (prev != slob_list->prev &&
309
-				slob_list->next != prev->next)
310
-			list_move_tail(slob_list, prev->next);
337
+		/*
338
+		 * If slob_page_alloc() removed sp from the list then we
339
+		 * cannot call list functions on sp.  If so allocation
340
+		 * did not fragment the page anyway so optimisation is
341
+		 * unnecessary.
342
+		 */
343
+		if (!page_removed_from_list) {
344
+			/*
345
+			 * Improve fragment distribution and reduce our average
346
+			 * search time by starting our next search here. (see
347
+			 * Knuth vol 1, sec 2.5, pg 449)
348
+			 */
349
+			if (!list_is_first(&sp->slab_list, slob_list))
350
+				list_rotate_to_front(&sp->slab_list, slob_list);
351
+		}
311352 		break;
312353 	}
313354 	spin_unlock_irqrestore(&slob_lock, flags);
....@@ -323,10 +364,10 @@
323364 		spin_lock_irqsave(&slob_lock, flags);
324365 		sp->units = SLOB_UNITS(PAGE_SIZE);
325366 		sp->freelist = b;
326
-		INIT_LIST_HEAD(&sp->lru);
367
+		INIT_LIST_HEAD(&sp->slab_list);
327368 		set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE));
328369 		set_slob_page_free(sp, slob_list);
329
-		b = slob_page_alloc(sp, size, align);
370
+		b = slob_page_alloc(sp, size, align, align_offset, &_unused);
330371 		BUG_ON(!b);
331372 		spin_unlock_irqrestore(&slob_lock, flags);
332373 	}
....@@ -428,27 +469,38 @@
428469 __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
429470 {
430471 	unsigned int *m;
431
-	int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
472
+	unsigned int minalign;
432473 	void *ret;
433474 
475
+	minalign = max_t(unsigned int, ARCH_KMALLOC_MINALIGN,
476
+			 arch_slab_minalign());
434477 	gfp &= gfp_allowed_mask;
435478 
436479 	fs_reclaim_acquire(gfp);
437480 	fs_reclaim_release(gfp);
438481 
439
-	if (size < PAGE_SIZE - align) {
482
+	if (size < PAGE_SIZE - minalign) {
483
+		int align = minalign;
484
+
485
+		/*
486
+		 * For power of two sizes, guarantee natural alignment for
487
+		 * kmalloc()'d objects.
488
+		 */
489
+		if (is_power_of_2(size))
490
+			align = max_t(unsigned int, minalign, size);
491
+
440492 		if (!size)
441493 			return ZERO_SIZE_PTR;
442494 
443
-		m = slob_alloc(size + align, gfp, align, node);
495
+		m = slob_alloc(size + minalign, gfp, align, node, minalign);
444496 
445497 		if (!m)
446498 			return NULL;
447499 		*m = size;
448
-		ret = (void *)m + align;
500
+		ret = (void *)m + minalign;
449501 
450502 		trace_kmalloc_node(caller, ret,
451
-				   size, size + align, gfp, node);
503
+				   size, size + minalign, gfp, node);
452504 	} else {
453505 		unsigned int order = get_order(size);
454506 
....@@ -497,19 +549,27 @@
497549 
498550 	sp = virt_to_page(block);
499551 	if (PageSlab(sp)) {
500
-		int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
552
+		unsigned int align = max_t(unsigned int,
553
+					   ARCH_KMALLOC_MINALIGN,
554
+					   arch_slab_minalign());
501555 		unsigned int *m = (unsigned int *)(block - align);
556
+
502557 		slob_free(m, *m + align);
503
-	} else
504
-		__free_pages(sp, compound_order(sp));
558
+	} else {
559
+		unsigned int order = compound_order(sp);
560
+		mod_node_page_state(page_pgdat(sp), NR_SLAB_UNRECLAIMABLE_B,
561
+				    -(PAGE_SIZE << order));
562
+		__free_pages(sp, order);
563
+
564
+	}
505565 }
506566 EXPORT_SYMBOL(kfree);
507567 
508568 /* can't use ksize for kmem_cache_alloc memory, only kmalloc */
509
-size_t ksize(const void *block)
569
+size_t __ksize(const void *block)
510570 {
511571 	struct page *sp;
512
-	int align;
572
+	unsigned int align;
513573 	unsigned int *m;
514574 
515575 	BUG_ON(!block);
....@@ -518,13 +578,14 @@
518578 
519579 	sp = virt_to_page(block);
520580 	if (unlikely(!PageSlab(sp)))
521
-		return PAGE_SIZE << compound_order(sp);
581
+		return page_size(sp);
522582 
523
-	align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
583
+	align = max_t(unsigned int, ARCH_KMALLOC_MINALIGN,
584
+		      arch_slab_minalign());
524585 	m = (unsigned int *)(block - align);
525586 	return SLOB_UNITS(*m) * SLOB_UNIT;
526587 }
527
-EXPORT_SYMBOL(ksize);
588
+EXPORT_SYMBOL(__ksize);
528589 
529590 int __kmem_cache_create(struct kmem_cache *c, slab_flags_t flags)
530591 {
....@@ -546,7 +607,7 @@
546607 	fs_reclaim_release(flags);
547608 
548609 	if (c->size < PAGE_SIZE) {
549
-		b = slob_alloc(c->size, flags, c->align, node);
610
+		b = slob_alloc(c->size, flags, c->align, node, 0);
550611 		trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
551612 					    SLOB_UNITS(c->size) * SLOB_UNIT,
552613 					    flags, node);