forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/mm/page_alloc.c
....@@ -61,7 +61,6 @@
6161 #include <linux/hugetlb.h>
6262 #include <linux/sched/rt.h>
6363 #include <linux/sched/mm.h>
64
-#include <linux/local_lock.h>
6564 #include <linux/page_owner.h>
6665 #include <linux/page_pinner.h>
6766 #include <linux/kthread.h>
....@@ -385,13 +384,6 @@
385384 EXPORT_SYMBOL(nr_node_ids);
386385 EXPORT_SYMBOL(nr_online_nodes);
387386 #endif
388
-
389
-struct pa_lock {
390
-	local_lock_t l;
391
-};
392
-static DEFINE_PER_CPU(struct pa_lock, pa_lock) = {
393
-	.l	= INIT_LOCAL_LOCK(l),
394
-};
395387 
396388 int page_group_by_mobility_disabled __read_mostly;
397389 
....@@ -1430,7 +1422,7 @@
14301422 }
14311423 
14321424 /*
1433
- * Frees a number of pages which have been collected from the pcp lists.
1425
+ * Frees a number of pages from the PCP lists
14341426  * Assumes all pages on list are in same zone, and of same order.
14351427  * count is the number of pages to free.
14361428  *
....@@ -1440,56 +1432,15 @@
14401432  * And clear the zone's pages_scanned counter, to hold off the "all pages are
14411433  * pinned" detection logic.
14421434  */
1443
-static void free_pcppages_bulk(struct zone *zone, struct list_head *head,
1444
-			       bool zone_retry)
1445
-{
1446
-	bool isolated_pageblocks;
1447
-	struct page *page, *tmp;
1448
-	unsigned long flags;
1449
-
1450
-	spin_lock_irqsave(&zone->lock, flags);
1451
-	isolated_pageblocks = has_isolate_pageblock(zone);
1452
-
1453
-	/*
1454
-	 * Use safe version since after __free_one_page(),
1455
-	 * page->lru.next will not point to original list.
1456
-	 */
1457
-	list_for_each_entry_safe(page, tmp, head, lru) {
1458
-		int mt = get_pcppage_migratetype(page);
1459
-
1460
-		if (page_zone(page) != zone) {
1461
-			/*
1462
-			 * free_unref_page_list() sorts pages by zone. If we end
1463
-			 * up with pages from a different NUMA nodes belonging
1464
-			 * to the same ZONE index then we need to redo with the
1465
-			 * correct ZONE pointer. Skip the page for now, redo it
1466
-			 * on the next iteration.
1467
-			 */
1468
-			WARN_ON_ONCE(zone_retry == false);
1469
-			if (zone_retry)
1470
-				continue;
1471
-		}
1472
-
1473
-		/* MIGRATE_ISOLATE page should not go to pcplists */
1474
-		VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1475
-		/* Pageblock could have been isolated meanwhile */
1476
-		if (unlikely(isolated_pageblocks))
1477
-			mt = get_pageblock_migratetype(page);
1478
-
1479
-		list_del(&page->lru);
1480
-		__free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE);
1481
-		trace_mm_page_pcpu_drain(page, 0, mt);
1482
-	}
1483
-	spin_unlock_irqrestore(&zone->lock, flags);
1484
-}
1485
-
1486
-static void isolate_pcp_pages(int count, struct per_cpu_pages *pcp,
1487
-			      struct list_head *dst)
1435
+static void free_pcppages_bulk(struct zone *zone, int count,
1436
+					struct per_cpu_pages *pcp)
14881437 {
14891438 	int migratetype = 0;
14901439 	int batch_free = 0;
14911440 	int prefetch_nr = 0;
1492
-	struct page *page;
1441
+	bool isolated_pageblocks;
1442
+	struct page *page, *tmp;
1443
+	LIST_HEAD(head);
14931444 
14941445 	/*
14951446 	 * Ensure proper count is passed which otherwise would stuck in the
....@@ -1526,7 +1477,7 @@
15261477 			if (bulkfree_pcp_prepare(page))
15271478 				continue;
15281479 
1529
-			list_add_tail(&page->lru, dst);
1480
+			list_add_tail(&page->lru, &head);
15301481 
15311482 			/*
15321483 			 * We are going to put the page back to the global
....@@ -1541,6 +1492,26 @@
15411492 				prefetch_buddy(page);
15421493 		} while (--count && --batch_free && !list_empty(list));
15431494 	}
1495
+
1496
+	spin_lock(&zone->lock);
1497
+	isolated_pageblocks = has_isolate_pageblock(zone);
1498
+
1499
+	/*
1500
+	 * Use safe version since after __free_one_page(),
1501
+	 * page->lru.next will not point to original list.
1502
+	 */
1503
+	list_for_each_entry_safe(page, tmp, &head, lru) {
1504
+		int mt = get_pcppage_migratetype(page);
1505
+		/* MIGRATE_ISOLATE page should not go to pcplists */
1506
+		VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1507
+		/* Pageblock could have been isolated meanwhile */
1508
+		if (unlikely(isolated_pageblocks))
1509
+			mt = get_pageblock_migratetype(page);
1510
+
1511
+		__free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE);
1512
+		trace_mm_page_pcpu_drain(page, 0, mt);
1513
+	}
1514
+	spin_unlock(&zone->lock);
15441515 }
15451516 
15461517 static void free_one_page(struct zone *zone,
....@@ -1648,11 +1619,11 @@
16481619 		return;
16491620 
16501621 	migratetype = get_pfnblock_migratetype(page, pfn);
1651
-	local_lock_irqsave(&pa_lock.l, flags);
1622
+	local_irq_save(flags);
16521623 	__count_vm_events(PGFREE, 1 << order);
16531624 	free_one_page(page_zone(page), page, pfn, order, migratetype,
16541625 		      fpi_flags);
1655
-	local_unlock_irqrestore(&pa_lock.l, flags);
1626
+	local_irq_restore(flags);
16561627 }
16571628 
16581629 void __free_pages_core(struct page *page, unsigned int order)
....@@ -3105,18 +3076,13 @@
31053076 {
31063077 	unsigned long flags;
31073078 	int to_drain, batch;
3108
-	LIST_HEAD(dst);
31093079 
3110
-	local_lock_irqsave(&pa_lock.l, flags);
3080
+	local_irq_save(flags);
31113081 	batch = READ_ONCE(pcp->batch);
31123082 	to_drain = min(pcp->count, batch);
31133083 	if (to_drain > 0)
3114
-		isolate_pcp_pages(to_drain, pcp, &dst);
3115
-
3116
-	local_unlock_irqrestore(&pa_lock.l, flags);
3117
-
3118
-	if (to_drain > 0)
3119
-		free_pcppages_bulk(zone, &dst, false);
3084
+		free_pcppages_bulk(zone, to_drain, pcp);
3085
+	local_irq_restore(flags);
31203086 }
31213087 #endif
31223088 
....@@ -3132,21 +3098,14 @@
31323098 	unsigned long flags;
31333099 	struct per_cpu_pageset *pset;
31343100 	struct per_cpu_pages *pcp;
3135
-	LIST_HEAD(dst);
3136
-	int count;
31373101 
3138
-	local_lock_irqsave(&pa_lock.l, flags);
3102
+	local_irq_save(flags);
31393103 	pset = per_cpu_ptr(zone->pageset, cpu);
31403104 
31413105 	pcp = &pset->pcp;
3142
-	count = pcp->count;
3143
-	if (count)
3144
-		isolate_pcp_pages(count, pcp, &dst);
3145
-
3146
-	local_unlock_irqrestore(&pa_lock.l, flags);
3147
-
3148
-	if (count)
3149
-		free_pcppages_bulk(zone, &dst, false);
3106
+	if (pcp->count)
3107
+		free_pcppages_bulk(zone, pcp->count, pcp);
3108
+	local_irq_restore(flags);
31503109 }
31513110 
31523111 /*
....@@ -3194,9 +3153,9 @@
31943153 	 * cpu which is allright but we also have to make sure to not move to
31953154 	 * a different one.
31963155 	 */
3197
-	migrate_disable();
3156
+	preempt_disable();
31983157 	drain_local_pages(drain->zone);
3199
-	migrate_enable();
3158
+	preempt_enable();
32003159 }
32013160 
32023161 /*
....@@ -3345,8 +3304,7 @@
33453304 	return true;
33463305 }
33473306 
3348
-static void free_unref_page_commit(struct page *page, unsigned long pfn,
3349
-				   struct list_head *dst)
3307
+static void free_unref_page_commit(struct page *page, unsigned long pfn)
33503308 {
33513309 	struct zone *zone = page_zone(page);
33523310 	struct per_cpu_pages *pcp;
....@@ -3380,8 +3338,7 @@
33803338 	pcp->count++;
33813339 	if (pcp->count >= pcp->high) {
33823340 		unsigned long batch = READ_ONCE(pcp->batch);
3383
-
3384
-		isolate_pcp_pages(batch, pcp, dst);
3341
+		free_pcppages_bulk(zone, batch, pcp);
33853342 	}
33863343 }
33873344 
....@@ -3392,17 +3349,13 @@
33923349 {
33933350 	unsigned long flags;
33943351 	unsigned long pfn = page_to_pfn(page);
3395
-	struct zone *zone = page_zone(page);
3396
-	LIST_HEAD(dst);
33973352 
33983353 	if (!free_unref_page_prepare(page, pfn))
33993354 		return;
34003355 
3401
-	local_lock_irqsave(&pa_lock.l, flags);
3402
-	free_unref_page_commit(page, pfn, &dst);
3403
-	local_unlock_irqrestore(&pa_lock.l, flags);
3404
-	if (!list_empty(&dst))
3405
-		free_pcppages_bulk(zone, &dst, false);
3356
+	local_irq_save(flags);
3357
+	free_unref_page_commit(page, pfn);
3358
+	local_irq_restore(flags);
34063359 }
34073360 
34083361 /*
....@@ -3413,11 +3366,6 @@
34133366 	struct page *page, *next;
34143367 	unsigned long flags, pfn;
34153368 	int batch_count = 0;
3416
-	struct list_head dsts[__MAX_NR_ZONES];
3417
-	int i;
3418
-
3419
-	for (i = 0; i < __MAX_NR_ZONES; i++)
3420
-		INIT_LIST_HEAD(&dsts[i]);
34213369 
34223370 	/* Prepare pages for freeing */
34233371 	list_for_each_entry_safe(page, next, list, lru) {
....@@ -3427,42 +3375,25 @@
34273375 		set_page_private(page, pfn);
34283376 	}
34293377 
3430
-	local_lock_irqsave(&pa_lock.l, flags);
3378
+	local_irq_save(flags);
34313379 	list_for_each_entry_safe(page, next, list, lru) {
34323380 		unsigned long pfn = page_private(page);
3433
-		enum zone_type type;
34343381 
34353382 		set_page_private(page, 0);
34363383 		trace_mm_page_free_batched(page);
3437
-		type = page_zonenum(page);
3438
-		free_unref_page_commit(page, pfn, &dsts[type]);
3384
+		free_unref_page_commit(page, pfn);
34393385 
34403386 		/*
34413387 		 * Guard against excessive IRQ disabled times when we get
34423388 		 * a large list of pages to free.
34433389 		 */
34443390 		if (++batch_count == SWAP_CLUSTER_MAX) {
3445
-			local_unlock_irqrestore(&pa_lock.l, flags);
3391
+			local_irq_restore(flags);
34463392 			batch_count = 0;
3447
-			local_lock_irqsave(&pa_lock.l, flags);
3393
+			local_irq_save(flags);
34483394 		}
34493395 	}
3450
-	local_unlock_irqrestore(&pa_lock.l, flags);
3451
-
3452
-	for (i = 0; i < __MAX_NR_ZONES; ) {
3453
-		struct page *page;
3454
-		struct zone *zone;
3455
-
3456
-		if (list_empty(&dsts[i])) {
3457
-			i++;
3458
-			continue;
3459
-		}
3460
-
3461
-		page = list_first_entry(&dsts[i], struct page, lru);
3462
-		zone = page_zone(page);
3463
-
3464
-		free_pcppages_bulk(zone, &dsts[i], true);
3465
-	}
3396
+	local_irq_restore(flags);
34663397 }
34673398 
34683399 /*
....@@ -3629,7 +3560,7 @@
36293560 	struct page *page;
36303561 	unsigned long flags;
36313562 
3632
-	local_lock_irqsave(&pa_lock.l, flags);
3563
+	local_irq_save(flags);
36333564 	pcp = &this_cpu_ptr(zone->pageset)->pcp;
36343565 	page = __rmqueue_pcplist(zone,  migratetype, alloc_flags, pcp,
36353566 				 gfp_flags);
....@@ -3637,7 +3568,7 @@
36373568 		__count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
36383569 		zone_statistics(preferred_zone, zone);
36393570 	}
3640
-	local_unlock_irqrestore(&pa_lock.l, flags);
3571
+	local_irq_restore(flags);
36413572 	return page;
36423573 }
36433574 
....@@ -3664,8 +3595,7 @@
36643595 	 * allocate greater than order-1 page units with __GFP_NOFAIL.
36653596 	 */
36663597 	WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3667
-	local_lock_irqsave(&pa_lock.l, flags);
3668
-	spin_lock(&zone->lock);
3598
+	spin_lock_irqsave(&zone->lock, flags);
36693599 
36703600 	do {
36713601 		page = NULL;
....@@ -3700,7 +3630,7 @@
37003630 	zone_statistics(preferred_zone, zone);
37013631 	trace_android_vh_rmqueue(preferred_zone, zone, order,
37023632 			gfp_flags, alloc_flags, migratetype);
3703
-	local_unlock_irqrestore(&pa_lock.l, flags);
3633
+	local_irq_restore(flags);
37043634 
37053635 out:
37063636 	/* Separate test+clear to avoid unnecessary atomics */
....@@ -3713,7 +3643,7 @@
37133643 	return page;
37143644 
37153645 failed:
3716
-	local_unlock_irqrestore(&pa_lock.l, flags);
3646
+	local_irq_restore(flags);
37173647 	return NULL;
37183648 }
37193649 
....@@ -9141,7 +9071,7 @@
91419071 	struct per_cpu_pageset *pset;
91429072 
91439073 	/* avoid races with drain_pages()  */
9144
-	local_lock_irqsave(&pa_lock.l, flags);
9074
+	local_irq_save(flags);
91459075 	if (zone->pageset != &boot_pageset) {
91469076 		for_each_online_cpu(cpu) {
91479077 			pset = per_cpu_ptr(zone->pageset, cpu);
....@@ -9150,7 +9080,7 @@
91509080 		free_percpu(zone->pageset);
91519081 		zone->pageset = &boot_pageset;
91529082 	}
9153
-	local_unlock_irqrestore(&pa_lock.l, flags);
9083
+	local_irq_restore(flags);
91549084 }
91559085 
91569086 #ifdef CONFIG_MEMORY_HOTREMOVE