~hc/RK356X_SDK_RELEASE.git

..	..	@@ -63,6 +63,12 @@
63	63	#define CREATE_TRACE_POINTS
64	64	#include <trace/events/vmscan.h>
65	65
	66	+#undef CREATE_TRACE_POINTS
	67	+#include <trace/hooks/vmscan.h>
	68	+
	69	+EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin);
	70	+EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end);
	71	+
66	72	struct scan_control {
67	73	/* How many pages shrink_list() should reclaim */
68	74	unsigned long nr_to_reclaim;
..	..	@@ -79,6 +85,19 @@
79	85	*/
80	86	struct mem_cgroup *target_mem_cgroup;
81	87
	88	+ /*
	89	+ * Scan pressure balancing between anon and file LRUs
	90	+ */
	91	+ unsigned long anon_cost;
	92	+ unsigned long file_cost;
	93	+
	94	+ /* Can active pages be deactivated as part of reclaim? */
	95	+#define DEACTIVATE_ANON 1
	96	+#define DEACTIVATE_FILE 2
	97	+ unsigned int may_deactivate:2;
	98	+ unsigned int force_deactivate:1;
	99	+ unsigned int skipped_deactivate:1;
	100	+
82	101	/* Writepage batching in laptop mode; RECLAIM_WRITE */
83	102	unsigned int may_writepage:1;
84	103
..	..	@@ -89,9 +108,12 @@
89	108	unsigned int may_swap:1;
90	109
91	110	/*
92		- * Cgroups are not reclaimed below their configured memory.low,
93		- * unless we threaten to OOM. If any cgroups are skipped due to
94		- * memory.low and nothing was reclaimed, go back for memory.low.
	111	+ * Cgroup memory below memory.low is protected as long as we
	112	+ * don't threaten to OOM. If any cgroup is reclaimed at
	113	+ * reduced force or passed over entirely due to its memory.low
	114	+ * setting (memcg_low_skipped), and nothing is reclaimed as a
	115	+ * result, then go back for one more cycle that reclaims the protected
	116	+ * memory (memcg_low_reclaim) to avert OOM.
95	117	*/
96	118	unsigned int memcg_low_reclaim:1;
97	119	unsigned int memcg_low_skipped:1;
..	..	@@ -100,6 +122,12 @@
100	122
101	123	/* One of the zones is ready for compaction */
102	124	unsigned int compaction_ready:1;
	125	+
	126	+ /* There is easily reclaimable cold cache in the current node */
	127	+ unsigned int cache_trim_mode:1;
	128	+
	129	+ /* The file pages on the current node are dangerously low */
	130	+ unsigned int file_is_tiny:1;
103	131
104	132	/* Allocation order */
105	133	s8 order;
..	..	@@ -128,21 +156,10 @@
128	156	unsigned int file_taken;
129	157	unsigned int taken;
130	158	} nr;
131		-};
132	159
133		-#ifdef ARCH_HAS_PREFETCH
134		-#define prefetch_prev_lru_page(_page, _base, _field) \
135		- do { \
136		- if ((_page)->lru.prev != _base) { \
137		- struct page *prev; \
138		- \
139		- prev = lru_to_page(&(_page->lru)); \
140		- prefetch(&prev->_field); \
141		- } \
142		- } while (0)
143		-#else
144		-#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
145		-#endif
	160	+ /* for recording the reclaimed slab by now */
	161	+ struct reclaim_state reclaim_state;
	162	+};
146	163
147	164	#ifdef ARCH_HAS_PREFETCHW
148	165	#define prefetchw_prev_lru_page(_page, _base, _field) \
..	..	@@ -159,20 +176,43 @@
159	176	#endif
160	177
161	178	/*
162		- * From 0 .. 100. Higher means more swappy.
	179	+ * From 0 .. 200. Higher means more swappy.
163	180	*/
164	181	int vm_swappiness = 60;
165		-/*
166		- * The total number of pages which are beyond the high watermark within all
167		- * zones.
168		- */
169		-unsigned long vm_total_pages;
	182	+
	183	+#define DEF_KSWAPD_THREADS_PER_NODE 1
	184	+static int kswapd_threads = DEF_KSWAPD_THREADS_PER_NODE;
	185	+static int __init kswapd_per_node_setup(char *str)
	186	+{
	187	+ int tmp;
	188	+
	189	+ if (kstrtoint(str, 0, &tmp) < 0)
	190	+ return 0;
	191	+
	192	+ if (tmp > MAX_KSWAPD_THREADS \|\| tmp <= 0)
	193	+ return 0;
	194	+
	195	+ kswapd_threads = tmp;
	196	+ return 1;
	197	+}
	198	+__setup("kswapd_per_node=", kswapd_per_node_setup);
	199	+
	200	+static void set_task_reclaim_state(struct task_struct *task,
	201	+ struct reclaim_state *rs)
	202	+{
	203	+ /* Check for an overwrite */
	204	+ WARN_ON_ONCE(rs && task->reclaim_state);
	205	+
	206	+ /* Check for the nulling of an already-nulled member */
	207	+ WARN_ON_ONCE(!rs && !task->reclaim_state);
	208	+
	209	+ task->reclaim_state = rs;
	210	+}
170	211
171	212	static LIST_HEAD(shrinker_list);
172	213	static DECLARE_RWSEM(shrinker_rwsem);
173	214
174		-#ifdef CONFIG_MEMCG_KMEM
175		-
	215	+#ifdef CONFIG_MEMCG
176	216	/*
177	217	* We allow subsystems to populate their shrinker-related
178	218	* LRU lists before register_shrinker_prepared() is called
..	..	@@ -224,25 +264,14 @@
224	264	idr_remove(&shrinker_idr, id);
225	265	up_write(&shrinker_rwsem);
226	266	}
227		-#else /* CONFIG_MEMCG_KMEM */
228		-static int prealloc_memcg_shrinker(struct shrinker *shrinker)
229		-{
230		- return 0;
231		-}
232	267
233		-static void unregister_memcg_shrinker(struct shrinker *shrinker)
	268	+static bool cgroup_reclaim(struct scan_control *sc)
234	269	{
235		-}
236		-#endif /* CONFIG_MEMCG_KMEM */
237		-
238		-#ifdef CONFIG_MEMCG
239		-static bool global_reclaim(struct scan_control *sc)
240		-{
241		- return !sc->target_mem_cgroup;
	270	+ return sc->target_mem_cgroup;
242	271	}
243	272
244	273	/**
245		- * sane_reclaim - is the usual dirty throttling mechanism operational?
	274	+ * writeback_throttling_sane - is the usual dirty throttling mechanism available?
246	275	* @sc: scan_control in question
247	276	*
248	277	* The normal page dirty throttling mechanism in balance_dirty_pages() is
..	..	@@ -254,11 +283,9 @@
254	283	* This function tests whether the vmscan currently in progress can assume
255	284	* that the normal dirty throttling mechanism is operational.
256	285	*/
257		-static bool sane_reclaim(struct scan_control *sc)
	286	+static bool writeback_throttling_sane(struct scan_control *sc)
258	287	{
259		- struct mem_cgroup *memcg = sc->target_mem_cgroup;
260		-
261		- if (!memcg)
	288	+ if (!cgroup_reclaim(sc))
262	289	return true;
263	290	#ifdef CONFIG_CGROUP_WRITEBACK
264	291	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
..	..	@@ -266,50 +293,24 @@
266	293	#endif
267	294	return false;
268	295	}
269		-
270		-static void set_memcg_congestion(pg_data_t *pgdat,
271		- struct mem_cgroup *memcg,
272		- bool congested)
273		-{
274		- struct mem_cgroup_per_node *mn;
275		-
276		- if (!memcg)
277		- return;
278		-
279		- mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
280		- WRITE_ONCE(mn->congested, congested);
281		-}
282		-
283		-static bool memcg_congested(pg_data_t *pgdat,
284		- struct mem_cgroup *memcg)
285		-{
286		- struct mem_cgroup_per_node *mn;
287		-
288		- mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
289		- return READ_ONCE(mn->congested);
290		-
291		-}
292	296	#else
293		-static bool global_reclaim(struct scan_control *sc)
	297	+static int prealloc_memcg_shrinker(struct shrinker *shrinker)
294	298	{
295		- return true;
	299	+ return 0;
296	300	}
297	301
298		-static bool sane_reclaim(struct scan_control *sc)
299		-{
300		- return true;
301		-}
302		-
303		-static inline void set_memcg_congestion(struct pglist_data *pgdat,
304		- struct mem_cgroup *memcg, bool congested)
	302	+static void unregister_memcg_shrinker(struct shrinker *shrinker)
305	303	{
306	304	}
307	305
308		-static inline bool memcg_congested(struct pglist_data *pgdat,
309		- struct mem_cgroup *memcg)
	306	+static bool cgroup_reclaim(struct scan_control *sc)
310	307	{
311	308	return false;
	309	+}
312	310
	311	+static bool writeback_throttling_sane(struct scan_control *sc)
	312	+{
	313	+ return true;
313	314	}
314	315	#endif
315	316
..	..	@@ -339,31 +340,21 @@
339	340	*/
340	341	unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
341	342	{
342		- unsigned long lru_size;
	343	+ unsigned long size = 0;
343	344	int zid;
344	345
345		- if (!mem_cgroup_disabled())
346		- lru_size = mem_cgroup_get_lru_size(lruvec, lru);
347		- else
348		- lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
349		-
350		- for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
	346	+ for (zid = 0; zid <= zone_idx && zid < MAX_NR_ZONES; zid++) {
351	347	struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
352		- unsigned long size;
353	348
354	349	if (!managed_zone(zone))
355	350	continue;
356	351
357	352	if (!mem_cgroup_disabled())
358		- size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
	353	+ size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
359	354	else
360		- size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],
361		- NR_ZONE_LRU_BASE + lru);
362		- lru_size -= min(size, lru_size);
	355	+ size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
363	356	}
364		-
365		- return lru_size;
366		-
	357	+ return size;
367	358	}
368	359
369	360	/*
..	..	@@ -371,7 +362,7 @@
371	362	*/
372	363	int prealloc_shrinker(struct shrinker *shrinker)
373	364	{
374		- size_t size = sizeof(*shrinker->nr_deferred);
	365	+ unsigned int size = sizeof(*shrinker->nr_deferred);
375	366
376	367	if (shrinker->flags & SHRINKER_NUMA_AWARE)
377	368	size *= nr_node_ids;
..	..	@@ -409,7 +400,7 @@
409	400	{
410	401	down_write(&shrinker_rwsem);
411	402	list_add_tail(&shrinker->list, &shrinker_list);
412		-#ifdef CONFIG_MEMCG_KMEM
	403	+#ifdef CONFIG_MEMCG
413	404	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
414	405	idr_replace(&shrinker_idr, shrinker, shrinker->id);
415	406	#endif
..	..	@@ -475,13 +466,22 @@
475	466	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
476	467
477	468	total_scan = nr;
478		- delta = freeable >> priority;
479		- delta *= 4;
480		- do_div(delta, shrinker->seeks);
	469	+ if (shrinker->seeks) {
	470	+ delta = freeable >> priority;
	471	+ delta *= 4;
	472	+ do_div(delta, shrinker->seeks);
	473	+ } else {
	474	+ /*
	475	+ * These objects don't require any IO to create. Trim
	476	+ * them aggressively under memory pressure to keep
	477	+ * them from causing refetches in the IO caches.
	478	+ */
	479	+ delta = freeable / 2;
	480	+ }
481	481
482	482	total_scan += delta;
483	483	if (total_scan < 0) {
484		- pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n",
	484	+ pr_err("shrink_slab: %pS negative objects to delete nr=%ld\n",
485	485	shrinker->scan_objects, total_scan);
486	486	total_scan = freeable;
487	487	next_deferred = nr;
..	..	@@ -567,7 +567,7 @@
567	567	return freed;
568	568	}
569	569
570		-#ifdef CONFIG_MEMCG_KMEM
	570	+#ifdef CONFIG_MEMCG
571	571	static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
572	572	struct mem_cgroup *memcg, int priority)
573	573	{
..	..	@@ -575,7 +575,7 @@
575	575	unsigned long ret, freed = 0;
576	576	int i;
577	577
578		- if (!memcg_kmem_enabled() \|\| !mem_cgroup_online(memcg))
	578	+ if (!mem_cgroup_online(memcg))
579	579	return 0;
580	580
581	581	if (!down_read_trylock(&shrinker_rwsem))
..	..	@@ -600,6 +600,11 @@
600	600	clear_bit(i, map->map);
601	601	continue;
602	602	}
	603	+
	604	+ /* Call non-slab shrinkers even though kmem is disabled */
	605	+ if (!memcg_kmem_enabled() &&
	606	+ !(shrinker->flags & SHRINKER_NONSLAB))
	607	+ continue;
603	608
604	609	ret = do_shrink_slab(&sc, shrinker, priority);
605	610	if (ret == SHRINK_EMPTY) {
..	..	@@ -637,13 +642,13 @@
637	642	up_read(&shrinker_rwsem);
638	643	return freed;
639	644	}
640		-#else /* CONFIG_MEMCG_KMEM */
	645	+#else /* CONFIG_MEMCG */
641	646	static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
642	647	struct mem_cgroup *memcg, int priority)
643	648	{
644	649	return 0;
645	650	}
646		-#endif /* CONFIG_MEMCG_KMEM */
	651	+#endif /* CONFIG_MEMCG */
647	652
648	653	/**
649	654	* shrink_slab - shrink slab caches
..	..	@@ -665,12 +670,17 @@
665	670	*
666	671	* Returns the number of reclaimed slab objects.
667	672	*/
668		-static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
	673	+unsigned long shrink_slab(gfp_t gfp_mask, int nid,
669	674	struct mem_cgroup *memcg,
670	675	int priority)
671	676	{
672	677	unsigned long ret, freed = 0;
673	678	struct shrinker *shrinker;
	679	+ bool bypass = false;
	680	+
	681	+ trace_android_vh_shrink_slab_bypass(gfp_mask, nid, memcg, priority, &bypass);
	682	+ if (bypass)
	683	+ return 0;
674	684
675	685	/*
676	686	* The root memcg might be allocated even though memcg is disabled
..	..	@@ -698,7 +708,7 @@
698	708	freed += ret;
699	709	/*
700	710	* Bail out if someone want to register a new shrinker to
701		- * prevent the regsitration from being stalled for long periods
	711	+ * prevent the registration from being stalled for long periods
702	712	* by parallel ongoing shrinking.
703	713	*/
704	714	if (rwsem_is_contended(&shrinker_rwsem)) {
..	..	@@ -712,6 +722,7 @@
712	722	cond_resched();
713	723	return freed;
714	724	}
	725	+EXPORT_SYMBOL_GPL(shrink_slab);
715	726
716	727	void drop_slab_node(int nid)
717	728	{
..	..	@@ -719,6 +730,9 @@
719	730
720	731	do {
721	732	struct mem_cgroup *memcg = NULL;
	733	+
	734	+ if (fatal_signal_pending(current))
	735	+ return;
722	736
723	737	freed = 0;
724	738	memcg = mem_cgroup_iter(NULL, NULL, NULL);
..	..	@@ -740,15 +754,14 @@
740	754	{
741	755	/*
742	756	* A freeable page cache page is referenced only by the caller
743		- * that isolated the page, the page cache radix tree and
744		- * optional buffer heads at page->private.
	757	+ * that isolated the page, the page cache and optional buffer
	758	+ * heads at page->private.
745	759	*/
746		- int radix_pins = PageTransHuge(page) && PageSwapCache(page) ?
747		- HPAGE_PMD_NR : 1;
748		- return page_count(page) - page_has_private(page) == 1 + radix_pins;
	760	+ int page_cache_pins = thp_nr_pages(page);
	761	+ return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
749	762	}
750	763
751		-static int may_write_to_inode(struct inode inode, struct scan_control sc)
	764	+static int may_write_to_inode(struct inode *inode)
752	765	{
753	766	if (current->flags & PF_SWAPWRITE)
754	767	return 1;
..	..	@@ -796,8 +809,7 @@
796	809	* pageout is called by shrink_page_list() for each dirty page.
797	810	* Calls ->writepage().
798	811	*/
799		-static pageout_t pageout(struct page page, struct address_space mapping,
800		- struct scan_control *sc)
	812	+static pageout_t pageout(struct page page, struct address_space mapping)
801	813	{
802	814	/*
803	815	* If the page is dirty, only perform writeback if that write
..	..	@@ -833,7 +845,7 @@
833	845	}
834	846	if (mapping->a_ops->writepage == NULL)
835	847	return PAGE_ACTIVATE;
836		- if (!may_write_to_inode(mapping->host, sc))
	848	+ if (!may_write_to_inode(mapping->host))
837	849	return PAGE_KEEP;
838	850
839	851	if (clear_page_dirty_for_io(page)) {
..	..	@@ -872,10 +884,11 @@
872	884	* gets returned with a refcount of 0.
873	885	*/
874	886	static int __remove_mapping(struct address_space mapping, struct page page,
875		- bool reclaimed)
	887	+ bool reclaimed, struct mem_cgroup *target_memcg)
876	888	{
877	889	unsigned long flags;
878	890	int refcount;
	891	+ void *shadow = NULL;
879	892
880	893	BUG_ON(!PageLocked(page));
881	894	BUG_ON(mapping != page_mapping(page));
..	..	@@ -906,10 +919,7 @@
906	919	* Note that if SetPageDirty is always performed via set_page_dirty,
907	920	* and thus under the i_pages lock, then this ordering is not required.
908	921	*/
909		- if (unlikely(PageTransHuge(page)) && PageSwapCache(page))
910		- refcount = 1 + HPAGE_PMD_NR;
911		- else
912		- refcount = 2;
	922	+ refcount = 1 + compound_nr(page);
913	923	if (!page_ref_freeze(page, refcount))
914	924	goto cannot_free;
915	925	/* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
..	..	@@ -921,12 +931,13 @@
921	931	if (PageSwapCache(page)) {
922	932	swp_entry_t swap = { .val = page_private(page) };
923	933	mem_cgroup_swapout(page, swap);
924		- __delete_from_swap_cache(page);
	934	+ if (reclaimed && !mapping_exiting(mapping))
	935	+ shadow = workingset_eviction(page, target_memcg);
	936	+ __delete_from_swap_cache(page, swap, shadow);
925	937	xa_unlock_irqrestore(&mapping->i_pages, flags);
926	938	put_swap_page(page, swap);
927	939	} else {
928	940	void (freepage)(struct page );
929		- void *shadow = NULL;
930	941
931	942	freepage = mapping->a_ops->freepage;
932	943	/*
..	..	@@ -934,7 +945,7 @@
934	945	* order to detect refaults, thus thrashing, later on.
935	946	*
936	947	* But don't store shadows in an address space that is
937		- * already exiting. This is not just an optizimation,
	948	+ * already exiting. This is not just an optimization,
938	949	* inode reclaim needs to empty out the radix tree or
939	950	* the nodes are lost. Don't plant shadows behind its
940	951	* back.
..	..	@@ -945,9 +956,9 @@
945	956	* exceptional entries and shadow exceptional entries in the
946	957	* same address_space.
947	958	*/
948		- if (reclaimed && page_is_file_cache(page) &&
	959	+ if (reclaimed && page_is_file_lru(page) &&
949	960	!mapping_exiting(mapping) && !dax_mapping(mapping))
950		- shadow = workingset_eviction(mapping, page);
	961	+ shadow = workingset_eviction(page, target_memcg);
951	962	__delete_from_page_cache(page, shadow);
952	963	xa_unlock_irqrestore(&mapping->i_pages, flags);
953	964
..	..	@@ -970,7 +981,7 @@
970	981	*/
971	982	int remove_mapping(struct address_space mapping, struct page page)
972	983	{
973		- if (__remove_mapping(mapping, page, false)) {
	984	+ if (__remove_mapping(mapping, page, false, NULL)) {
974	985	/*
975	986	* Unfreezing the refcount with 1 rather than 2 effectively
976	987	* drops the pagecache ref for us without requiring another
..	..	@@ -1009,11 +1020,24 @@
1009	1020	{
1010	1021	int referenced_ptes, referenced_page;
1011	1022	unsigned long vm_flags;
	1023	+ bool should_protect = false;
	1024	+ bool trylock_fail = false;
	1025	+ int ret = 0;
1012	1026
	1027	+ trace_android_vh_page_should_be_protected(page, &should_protect);
	1028	+ if (unlikely(should_protect))
	1029	+ return PAGEREF_ACTIVATE;
	1030	+
	1031	+ trace_android_vh_page_trylock_set(page);
	1032	+ trace_android_vh_check_page_look_around_ref(page, &ret);
	1033	+ if (ret)
	1034	+ return ret;
1013	1035	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
1014	1036	&vm_flags);
1015	1037	referenced_page = TestClearPageReferenced(page);
1016		-
	1038	+ trace_android_vh_page_trylock_get_result(page, &trylock_fail);
	1039	+ if (trylock_fail)
	1040	+ return PAGEREF_KEEP;
1017	1041	/*
1018	1042	* Mlock lost the isolation race with us. Let try_to_unmap()
1019	1043	* move the page to the unevictable list.
..	..	@@ -1021,9 +1045,11 @@
1021	1045	if (vm_flags & VM_LOCKED)
1022	1046	return PAGEREF_RECLAIM;
1023	1047
	1048	+ /* rmap lock contention: rotate */
	1049	+ if (referenced_ptes == -1)
	1050	+ return PAGEREF_KEEP;
	1051	+
1024	1052	if (referenced_ptes) {
1025		- if (PageSwapBacked(page))
1026		- return PAGEREF_ACTIVATE;
1027	1053	/*
1028	1054	* All mapped pages start out with page table
1029	1055	* references from the instantiating fault, so we need
..	..	@@ -1046,7 +1072,7 @@
1046	1072	/*
1047	1073	* Activate file-backed executable pages after first usage.
1048	1074	*/
1049		- if (vm_flags & VM_EXEC)
	1075	+ if ((vm_flags & VM_EXEC) && !PageSwapBacked(page))
1050	1076	return PAGEREF_ACTIVATE;
1051	1077
1052	1078	return PAGEREF_KEEP;
..	..	@@ -1069,7 +1095,7 @@
1069	1095	* Anonymous pages are not handled by flushers and must be written
1070	1096	* from reclaim context. Do not stall reclaim based on them
1071	1097	*/
1072		- if (!page_is_file_cache(page) \|\|
	1098	+ if (!page_is_file_lru(page) \|\|
1073	1099	(PageAnon(page) && !PageSwapBacked(page))) {
1074	1100	*dirty = false;
1075	1101	*writeback = false;
..	..	@@ -1092,33 +1118,26 @@
1092	1118	/*
1093	1119	* shrink_page_list() returns the number of reclaimed pages
1094	1120	*/
1095		-static unsigned long shrink_page_list(struct list_head *page_list,
1096		- struct pglist_data *pgdat,
1097		- struct scan_control *sc,
1098		- enum ttu_flags ttu_flags,
1099		- struct reclaim_stat *stat,
1100		- bool force_reclaim)
	1121	+static unsigned int shrink_page_list(struct list_head *page_list,
	1122	+ struct pglist_data *pgdat,
	1123	+ struct scan_control *sc,
	1124	+ struct reclaim_stat *stat,
	1125	+ bool ignore_references)
1101	1126	{
1102	1127	LIST_HEAD(ret_pages);
1103	1128	LIST_HEAD(free_pages);
1104		- int pgactivate = 0;
1105		- unsigned nr_unqueued_dirty = 0;
1106		- unsigned nr_dirty = 0;
1107		- unsigned nr_congested = 0;
1108		- unsigned nr_reclaimed = 0;
1109		- unsigned nr_writeback = 0;
1110		- unsigned nr_immediate = 0;
1111		- unsigned nr_ref_keep = 0;
1112		- unsigned nr_unmap_fail = 0;
	1129	+ unsigned int nr_reclaimed = 0;
	1130	+ unsigned int pgactivate = 0;
1113	1131
	1132	+ memset(stat, 0, sizeof(*stat));
1114	1133	cond_resched();
1115	1134
1116	1135	while (!list_empty(page_list)) {
1117	1136	struct address_space *mapping;
1118	1137	struct page *page;
1119		- int may_enter_fs;
1120		- enum page_references references = PAGEREF_RECLAIM_CLEAN;
1121		- bool dirty, writeback;
	1138	+ enum page_references references = PAGEREF_RECLAIM;
	1139	+ bool dirty, writeback, may_enter_fs;
	1140	+ unsigned int nr_pages;
1122	1141
1123	1142	cond_resched();
1124	1143
..	..	@@ -1130,18 +1149,16 @@
1130	1149
1131	1150	VM_BUG_ON_PAGE(PageActive(page), page);
1132	1151
1133		- sc->nr_scanned++;
	1152	+ nr_pages = compound_nr(page);
	1153	+
	1154	+ /* Account the number of base pages even though THP */
	1155	+ sc->nr_scanned += nr_pages;
1134	1156
1135	1157	if (unlikely(!page_evictable(page)))
1136	1158	goto activate_locked;
1137	1159
1138	1160	if (!sc->may_unmap && page_mapped(page))
1139	1161	goto keep_locked;
1140		-
1141		- /* Double the slab pressure for mapped and swapcache pages */
1142		- if ((page_mapped(page) \|\| PageSwapCache(page)) &&
1143		- !(PageAnon(page) && !PageSwapBacked(page)))
1144		- sc->nr_scanned++;
1145	1162
1146	1163	may_enter_fs = (sc->gfp_mask & __GFP_FS) \|\|
1147	1164	(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
..	..	@@ -1154,10 +1171,10 @@
1154	1171	*/
1155	1172	page_check_dirty_writeback(page, &dirty, &writeback);
1156	1173	if (dirty \|\| writeback)
1157		- nr_dirty++;
	1174	+ stat->nr_dirty++;
1158	1175
1159	1176	if (dirty && !writeback)
1160		- nr_unqueued_dirty++;
	1177	+ stat->nr_unqueued_dirty++;
1161	1178
1162	1179	/*
1163	1180	* Treat this page as congested if the underlying BDI is or if
..	..	@@ -1169,7 +1186,7 @@
1169	1186	if (((dirty \|\| writeback) && mapping &&
1170	1187	inode_write_congested(mapping->host)) \|\|
1171	1188	(writeback && PageReclaim(page)))
1172		- nr_congested++;
	1189	+ stat->nr_congested++;
1173	1190
1174	1191	/*
1175	1192	* If a page at the tail of the LRU is under writeback, there
..	..	@@ -1218,11 +1235,11 @@
1218	1235	if (current_is_kswapd() &&
1219	1236	PageReclaim(page) &&
1220	1237	test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1221		- nr_immediate++;
	1238	+ stat->nr_immediate++;
1222	1239	goto activate_locked;
1223	1240
1224	1241	/* Case 2 above */
1225		- } else if (sane_reclaim(sc) \|\|
	1242	+ } else if (writeback_throttling_sane(sc) \|\|
1226	1243	!PageReclaim(page) \|\| !may_enter_fs) {
1227	1244	/*
1228	1245	* This is slightly racy - end_page_writeback()
..	..	@@ -1236,7 +1253,7 @@
1236	1253	* and it's also appropriate in global reclaim.
1237	1254	*/
1238	1255	SetPageReclaim(page);
1239		- nr_writeback++;
	1256	+ stat->nr_writeback++;
1240	1257	goto activate_locked;
1241	1258
1242	1259	/* Case 3 above */
..	..	@@ -1249,14 +1266,14 @@
1249	1266	}
1250	1267	}
1251	1268
1252		- if (!force_reclaim)
	1269	+ if (!ignore_references)
1253	1270	references = page_check_references(page, sc);
1254	1271
1255	1272	switch (references) {
1256	1273	case PAGEREF_ACTIVATE:
1257	1274	goto activate_locked;
1258	1275	case PAGEREF_KEEP:
1259		- nr_ref_keep++;
	1276	+ stat->nr_ref_keep += nr_pages;
1260	1277	goto keep_locked;
1261	1278	case PAGEREF_RECLAIM:
1262	1279	case PAGEREF_RECLAIM_CLEAN:
..	..	@@ -1271,6 +1288,8 @@
1271	1288	if (PageAnon(page) && PageSwapBacked(page)) {
1272	1289	if (!PageSwapCache(page)) {
1273	1290	if (!(sc->gfp_mask & __GFP_IO))
	1291	+ goto keep_locked;
	1292	+ if (page_maybe_dma_pinned(page))
1274	1293	goto keep_locked;
1275	1294	if (PageTransHuge(page)) {
1276	1295	/* cannot split THP, skip it */
..	..	@@ -1288,7 +1307,7 @@
1288	1307	}
1289	1308	if (!add_to_swap(page)) {
1290	1309	if (!PageTransHuge(page))
1291		- goto activate_locked;
	1310	+ goto activate_locked_split;
1292	1311	/* Fallback to swap normal pages */
1293	1312	if (split_huge_page_to_list(page,
1294	1313	page_list))
..	..	@@ -1297,10 +1316,10 @@
1297	1316	count_vm_event(THP_SWPOUT_FALLBACK);
1298	1317	#endif
1299	1318	if (!add_to_swap(page))
1300		- goto activate_locked;
	1319	+ goto activate_locked_split;
1301	1320	}
1302	1321
1303		- may_enter_fs = 1;
	1322	+ may_enter_fs = true;
1304	1323
1305	1324	/* Adding to swap updated mapping */
1306	1325	mapping = page_mapping(page);
..	..	@@ -1312,16 +1331,33 @@
1312	1331	}
1313	1332
1314	1333	/*
	1334	+ * THP may get split above, need minus tail pages and update
	1335	+ * nr_pages to avoid accounting tail pages twice.
	1336	+ *
	1337	+ * The tail pages that are added into swap cache successfully
	1338	+ * reach here.
	1339	+ */
	1340	+ if ((nr_pages > 1) && !PageTransHuge(page)) {
	1341	+ sc->nr_scanned -= (nr_pages - 1);
	1342	+ nr_pages = 1;
	1343	+ }
	1344	+
	1345	+ /*
1315	1346	* The page is mapped into the page tables of one or more
1316	1347	* processes. Try to unmap it here.
1317	1348	*/
1318	1349	if (page_mapped(page)) {
1319		- enum ttu_flags flags = ttu_flags \| TTU_BATCH_FLUSH;
	1350	+ enum ttu_flags flags = TTU_BATCH_FLUSH;
	1351	+ bool was_swapbacked = PageSwapBacked(page);
1320	1352
1321	1353	if (unlikely(PageTransHuge(page)))
1322	1354	flags \|= TTU_SPLIT_HUGE_PMD;
	1355	+ if (!ignore_references)
	1356	+ trace_android_vh_page_trylock_set(page);
1323	1357	if (!try_to_unmap(page, flags)) {
1324		- nr_unmap_fail++;
	1358	+ stat->nr_unmap_fail += nr_pages;
	1359	+ if (!was_swapbacked && PageSwapBacked(page))
	1360	+ stat->nr_lazyfree_fail += nr_pages;
1325	1361	goto activate_locked;
1326	1362	}
1327	1363	}
..	..	@@ -1337,7 +1373,7 @@
1337	1373	* the rest of the LRU for clean pages and see
1338	1374	* the same dirty pages again (PageReclaim).
1339	1375	*/
1340		- if (page_is_file_cache(page) &&
	1376	+ if (page_is_file_lru(page) &&
1341	1377	(!current_is_kswapd() \|\| !PageReclaim(page) \|\|
1342	1378	!test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1343	1379	/*
..	..	@@ -1365,12 +1401,14 @@
1365	1401	* starts and then write it out here.
1366	1402	*/
1367	1403	try_to_unmap_flush_dirty();
1368		- switch (pageout(page, mapping, sc)) {
	1404	+ switch (pageout(page, mapping)) {
1369	1405	case PAGE_KEEP:
1370	1406	goto keep_locked;
1371	1407	case PAGE_ACTIVATE:
1372	1408	goto activate_locked;
1373	1409	case PAGE_SUCCESS:
	1410	+ stat->nr_pageout += thp_nr_pages(page);
	1411	+
1374	1412	if (PageWriteback(page))
1375	1413	goto keep;
1376	1414	if (PageDirty(page))
..	..	@@ -1426,6 +1464,7 @@
1426	1464	* increment nr_reclaimed here (and
1427	1465	* leave it off the LRU).
1428	1466	*/
	1467	+ trace_android_vh_page_trylock_clear(page);
1429	1468	nr_reclaimed++;
1430	1469	continue;
1431	1470	}
..	..	@@ -1443,30 +1482,38 @@
1443	1482
1444	1483	count_vm_event(PGLAZYFREED);
1445	1484	count_memcg_page_event(page, PGLAZYFREED);
1446		- } else if (!mapping \|\| !__remove_mapping(mapping, page, true))
	1485	+ } else if (!mapping \|\| !__remove_mapping(mapping, page, true,
	1486	+ sc->target_mem_cgroup))
1447	1487	goto keep_locked;
1448		- /*
1449		- * At this point, we have no other references and there is
1450		- * no way to pick any more up (removed from LRU, removed
1451		- * from pagecache). Can use non-atomic bitops now (and
1452		- * we obviously don't have to worry about waking up a process
1453		- * waiting on the page lock, because there are no references.
1454		- */
1455		- __ClearPageLocked(page);
	1488	+
	1489	+ unlock_page(page);
1456	1490	free_it:
1457		- nr_reclaimed++;
	1491	+ /*
	1492	+ * THP may get swapped out in a whole, need account
	1493	+ * all base pages.
	1494	+ */
	1495	+ nr_reclaimed += nr_pages;
1458	1496
1459	1497	/*
1460	1498	* Is there need to periodically free_page_list? It would
1461	1499	* appear not as the counts should be low
1462	1500	*/
1463		- if (unlikely(PageTransHuge(page))) {
1464		- mem_cgroup_uncharge(page);
1465		- (*get_compound_page_dtor(page))(page);
1466		- } else
	1501	+ trace_android_vh_page_trylock_clear(page);
	1502	+ if (unlikely(PageTransHuge(page)))
	1503	+ destroy_compound_page(page);
	1504	+ else
1467	1505	list_add(&page->lru, &free_pages);
1468	1506	continue;
1469	1507
	1508	+activate_locked_split:
	1509	+ /*
	1510	+ * The tail pages that are failed to add into swap cache
	1511	+ * reach here. Fixup nr_scanned and nr_pages.
	1512	+ */
	1513	+ if (nr_pages > 1) {
	1514	+ sc->nr_scanned -= (nr_pages - 1);
	1515	+ nr_pages = 1;
	1516	+ }
1470	1517	activate_locked:
1471	1518	/* Not a candidate for swapping, so reclaim swap space. */
1472	1519	if (PageSwapCache(page) && (mem_cgroup_swap_full(page) \|\|
..	..	@@ -1474,8 +1521,9 @@
1474	1521	try_to_free_swap(page);
1475	1522	VM_BUG_ON_PAGE(PageActive(page), page);
1476	1523	if (!PageMlocked(page)) {
	1524	+ int type = page_is_file_lru(page);
1477	1525	SetPageActive(page);
1478		- pgactivate++;
	1526	+ stat->nr_activate[type] += nr_pages;
1479	1527	count_memcg_page_event(page, PGACTIVATE);
1480	1528	}
1481	1529	keep_locked:
..	..	@@ -1485,6 +1533,8 @@
1485	1533	VM_BUG_ON_PAGE(PageLRU(page) \|\| PageUnevictable(page), page);
1486	1534	}
1487	1535
	1536	+ pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
	1537	+
1488	1538	mem_cgroup_uncharge_list(&free_pages);
1489	1539	try_to_unmap_flush();
1490	1540	free_unref_page_list(&free_pages);
..	..	@@ -1492,20 +1542,10 @@
1492	1542	list_splice(&ret_pages, page_list);
1493	1543	count_vm_events(PGACTIVATE, pgactivate);
1494	1544
1495		- if (stat) {
1496		- stat->nr_dirty = nr_dirty;
1497		- stat->nr_congested = nr_congested;
1498		- stat->nr_unqueued_dirty = nr_unqueued_dirty;
1499		- stat->nr_writeback = nr_writeback;
1500		- stat->nr_immediate = nr_immediate;
1501		- stat->nr_activate = pgactivate;
1502		- stat->nr_ref_keep = nr_ref_keep;
1503		- stat->nr_unmap_fail = nr_unmap_fail;
1504		- }
1505	1545	return nr_reclaimed;
1506	1546	}
1507	1547
1508		-unsigned long reclaim_clean_pages_from_list(struct zone *zone,
	1548	+unsigned int reclaim_clean_pages_from_list(struct zone *zone,
1509	1549	struct list_head *page_list)
1510	1550	{
1511	1551	struct scan_control sc = {
..	..	@@ -1513,23 +1553,35 @@
1513	1553	.priority = DEF_PRIORITY,
1514	1554	.may_unmap = 1,
1515	1555	};
1516		- unsigned long ret;
	1556	+ struct reclaim_stat stat;
	1557	+ unsigned int nr_reclaimed;
1517	1558	struct page page, next;
1518	1559	LIST_HEAD(clean_pages);
1519	1560
1520	1561	list_for_each_entry_safe(page, next, page_list, lru) {
1521		- if (page_is_file_cache(page) && !PageDirty(page) &&
	1562	+ if (page_is_file_lru(page) && !PageDirty(page) &&
1522	1563	!__PageMovable(page) && !PageUnevictable(page)) {
1523	1564	ClearPageActive(page);
1524	1565	list_move(&page->lru, &clean_pages);
1525	1566	}
1526	1567	}
1527	1568
1528		- ret = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
1529		- TTU_IGNORE_ACCESS, NULL, true);
	1569	+ nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
	1570	+ &stat, true);
1530	1571	list_splice(&clean_pages, page_list);
1531		- mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, -ret);
1532		- return ret;
	1572	+ mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
	1573	+ -(long)nr_reclaimed);
	1574	+ /*
	1575	+ * Since lazyfree pages are isolated from file LRU from the beginning,
	1576	+ * they will rotate back to anonymous LRU in the end if it failed to
	1577	+ * discard so isolated count will be mismatched.
	1578	+ * Compensate the isolated count for both LRU lists.
	1579	+ */
	1580	+ mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON,
	1581	+ stat.nr_lazyfree_fail);
	1582	+ mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
	1583	+ -(long)stat.nr_lazyfree_fail);
	1584	+ return nr_reclaimed;
1533	1585	}
1534	1586
1535	1587	/*
..	..	@@ -1612,7 +1664,7 @@
1612	1664
1613	1665	/*
1614	1666	* Update LRU sizes after isolating pages. The LRU size updates must
1615		- * be complete before mem_cgroup_update_lru_size due to a santity check.
	1667	+ * be complete before mem_cgroup_update_lru_size due to a sanity check.
1616	1668	*/
1617	1669	static __always_inline void update_lru_sizes(struct lruvec *lruvec,
1618	1670	enum lru_list lru, unsigned long *nr_zone_taken)
..	..	@@ -1623,16 +1675,13 @@
1623	1675	if (!nr_zone_taken[zid])
1624	1676	continue;
1625	1677
1626		- __update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1627		-#ifdef CONFIG_MEMCG
1628		- mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1629		-#endif
	1678	+ update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1630	1679	}
1631	1680
1632	1681	}
1633	1682
1634		-/*
1635		- * zone_lru_lock is heavily contended. Some of the functions that
	1683	+/**
	1684	+ * pgdat->lru_lock is heavily contended. Some of the functions that
1636	1685	* shrink the lists perform better by taking out a batch of pages
1637	1686	* and working on them outside the LRU lock.
1638	1687	*
..	..	@@ -1646,7 +1695,6 @@
1646	1695	* @dst: The temp list to put pages on to.
1647	1696	* @nr_scanned: The number of pages that were scanned.
1648	1697	* @sc: The scan_control struct for this reclaim session
1649		- * @mode: One of the LRU isolation modes
1650	1698	* @lru: LRU list id for isolating
1651	1699	*
1652	1700	* returns how many pages were moved onto *@dst.
..	..	@@ -1654,7 +1702,7 @@
1654	1702	static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1655	1703	struct lruvec lruvec, struct list_head dst,
1656	1704	unsigned long nr_scanned, struct scan_control sc,
1657		- isolate_mode_t mode, enum lru_list lru)
	1705	+ enum lru_list lru)
1658	1706	{
1659	1707	struct list_head *src = &lruvec->lists[lru];
1660	1708	unsigned long nr_taken = 0;
..	..	@@ -1663,11 +1711,11 @@
1663	1711	unsigned long skipped = 0;
1664	1712	unsigned long scan, total_scan, nr_pages;
1665	1713	LIST_HEAD(pages_skipped);
	1714	+ isolate_mode_t mode = (sc->may_unmap ? 0 : ISOLATE_UNMAPPED);
1666	1715
	1716	+ total_scan = 0;
1667	1717	scan = 0;
1668		- for (total_scan = 0;
1669		- scan < nr_to_scan && nr_taken < nr_to_scan && !list_empty(src);
1670		- total_scan++) {
	1718	+ while (scan < nr_to_scan && !list_empty(src)) {
1671	1719	struct page *page;
1672	1720
1673	1721	page = lru_to_page(src);
..	..	@@ -1675,9 +1723,12 @@
1675	1723
1676	1724	VM_BUG_ON_PAGE(!PageLRU(page), page);
1677	1725
	1726	+ nr_pages = compound_nr(page);
	1727	+ total_scan += nr_pages;
	1728	+
1678	1729	if (page_zonenum(page) > sc->reclaim_idx) {
1679	1730	list_move(&page->lru, &pages_skipped);
1680		- nr_skipped[page_zonenum(page)]++;
	1731	+ nr_skipped[page_zonenum(page)] += nr_pages;
1681	1732	continue;
1682	1733	}
1683	1734
..	..	@@ -1686,13 +1737,17 @@
1686	1737	* return with no isolated pages if the LRU mostly contains
1687	1738	* ineligible pages. This causes the VM to not reclaim any
1688	1739	* pages, triggering a premature OOM.
	1740	+ *
	1741	+ * Account all tail pages of THP. This would not cause
	1742	+ * premature OOM since __isolate_lru_page() returns -EBUSY
	1743	+ * only when the page is being freed somewhere else.
1689	1744	*/
1690		- scan++;
	1745	+ scan += nr_pages;
1691	1746	switch (__isolate_lru_page(page, mode)) {
1692	1747	case 0:
1693		- nr_pages = hpage_nr_pages(page);
1694	1748	nr_taken += nr_pages;
1695	1749	nr_zone_taken[page_zonenum(page)] += nr_pages;
	1750	+ trace_android_vh_del_page_from_lrulist(page, false, lru);
1696	1751	list_move(&page->lru, dst);
1697	1752	break;
1698	1753
..	..	@@ -1753,7 +1808,7 @@
1753	1808	* Restrictions:
1754	1809	*
1755	1810	* (1) Must be called with an elevated refcount on the page. This is a
1756		- * fundamentnal difference from isolate_lru_pages (which is called
	1811	+ * fundamental difference from isolate_lru_pages (which is called
1757	1812	* without a stable reference).
1758	1813	* (2) the lru_lock must not be held.
1759	1814	* (3) interrupts must be enabled.
..	..	@@ -1766,11 +1821,11 @@
1766	1821	WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
1767	1822
1768	1823	if (PageLRU(page)) {
1769		- struct zone *zone = page_zone(page);
	1824	+ pg_data_t *pgdat = page_pgdat(page);
1770	1825	struct lruvec *lruvec;
1771	1826
1772		- spin_lock_irq(zone_lru_lock(zone));
1773		- lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
	1827	+ spin_lock_irq(&pgdat->lru_lock);
	1828	+ lruvec = mem_cgroup_page_lruvec(page, pgdat);
1774	1829	if (PageLRU(page)) {
1775	1830	int lru = page_lru(page);
1776	1831	get_page(page);
..	..	@@ -1778,14 +1833,14 @@
1778	1833	del_page_from_lru_list(page, lruvec, lru);
1779	1834	ret = 0;
1780	1835	}
1781		- spin_unlock_irq(zone_lru_lock(zone));
	1836	+ spin_unlock_irq(&pgdat->lru_lock);
1782	1837	}
1783	1838	return ret;
1784	1839	}
1785	1840
1786	1841	/*
1787	1842	* A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1788		- * then get resheduled. When there are massive number of tasks doing page
	1843	+ * then get rescheduled. When there are massive number of tasks doing page
1789	1844	* allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1790	1845	* the LRU list will go small and be scanned faster than necessary, leading to
1791	1846	* unnecessary swapping, thrashing and OOM.
..	..	@@ -1798,7 +1853,7 @@
1798	1853	if (current_is_kswapd())
1799	1854	return 0;
1800	1855
1801		- if (!sane_reclaim(sc))
	1856	+ if (!writeback_throttling_sane(sc))
1802	1857	return 0;
1803	1858
1804	1859	if (file) {
..	..	@@ -1820,40 +1875,55 @@
1820	1875	return isolated > inactive;
1821	1876	}
1822	1877
1823		-static noinline_for_stack void
1824		-putback_inactive_pages(struct lruvec lruvec, struct list_head page_list)
	1878	+/*
	1879	+ * This moves pages from @list to corresponding LRU list.
	1880	+ *
	1881	+ * We move them the other way if the page is referenced by one or more
	1882	+ * processes, from rmap.
	1883	+ *
	1884	+ * If the pages are mostly unmapped, the processing is fast and it is
	1885	+ * appropriate to hold zone_lru_lock across the whole operation. But if
	1886	+ * the pages are mapped, the processing is slow (page_referenced()) so we
	1887	+ * should drop zone_lru_lock around each page. It's impossible to balance
	1888	+ * this, so instead we remove the pages from the LRU while processing them.
	1889	+ * It is safe to rely on PG_active against the non-LRU pages in here because
	1890	+ * nobody will play with that bit on a non-LRU page.
	1891	+ *
	1892	+ * The downside is that we have to touch page->_refcount against each page.
	1893	+ * But we had to alter page->flags anyway.
	1894	+ *
	1895	+ * Returns the number of pages moved to the given lruvec.
	1896	+ */
	1897	+
	1898	+static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec,
	1899	+ struct list_head *list)
1825	1900	{
1826		- struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1827	1901	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
	1902	+ int nr_pages, nr_moved = 0;
1828	1903	LIST_HEAD(pages_to_free);
	1904	+ struct page *page;
	1905	+ enum lru_list lru;
1829	1906
1830		- /*
1831		- * Put back any unfreeable pages.
1832		- */
1833		- while (!list_empty(page_list)) {
1834		- struct page *page = lru_to_page(page_list);
1835		- int lru;
1836		-
	1907	+ while (!list_empty(list)) {
	1908	+ page = lru_to_page(list);
1837	1909	VM_BUG_ON_PAGE(PageLRU(page), page);
1838		- list_del(&page->lru);
1839	1910	if (unlikely(!page_evictable(page))) {
	1911	+ list_del(&page->lru);
1840	1912	spin_unlock_irq(&pgdat->lru_lock);
1841	1913	putback_lru_page(page);
1842	1914	spin_lock_irq(&pgdat->lru_lock);
1843	1915	continue;
1844	1916	}
1845		-
1846	1917	lruvec = mem_cgroup_page_lruvec(page, pgdat);
1847	1918
1848	1919	SetPageLRU(page);
1849	1920	lru = page_lru(page);
1850		- add_page_to_lru_list(page, lruvec, lru);
1851	1921
1852		- if (is_active_lru(lru)) {
1853		- int file = is_file_lru(lru);
1854		- int numpages = hpage_nr_pages(page);
1855		- reclaim_stat->recent_rotated[file] += numpages;
1856		- }
	1922	+ nr_pages = thp_nr_pages(page);
	1923	+ update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);
	1924	+ list_move(&page->lru, &lruvec->lists[lru]);
	1925	+ trace_android_vh_add_page_to_lrulist(page, false, lru);
	1926	+
1857	1927	if (put_page_testzero(page)) {
1858	1928	__ClearPageLRU(page);
1859	1929	__ClearPageActive(page);
..	..	@@ -1861,29 +1931,34 @@
1861	1931
1862	1932	if (unlikely(PageCompound(page))) {
1863	1933	spin_unlock_irq(&pgdat->lru_lock);
1864		- mem_cgroup_uncharge(page);
1865		- (*get_compound_page_dtor(page))(page);
	1934	+ destroy_compound_page(page);
1866	1935	spin_lock_irq(&pgdat->lru_lock);
1867	1936	} else
1868	1937	list_add(&page->lru, &pages_to_free);
	1938	+ } else {
	1939	+ nr_moved += nr_pages;
	1940	+ if (PageActive(page))
	1941	+ workingset_age_nonresident(lruvec, nr_pages);
1869	1942	}
1870	1943	}
1871	1944
1872	1945	/*
1873	1946	* To save our caller's stack, now use input list for pages to free.
1874	1947	*/
1875		- list_splice(&pages_to_free, page_list);
	1948	+ list_splice(&pages_to_free, list);
	1949	+
	1950	+ return nr_moved;
1876	1951	}
1877	1952
1878	1953	/*
1879	1954	* If a kernel thread (such as nfsd for loop-back mounts) services
1880		- * a backing device by writing to the page cache it sets PF_LESS_THROTTLE.
	1955	+ * a backing device by writing to the page cache it sets PF_LOCAL_THROTTLE.
1881	1956	* In that case we should only throttle if the backing device it is
1882	1957	* writing to is congested. In other cases it is safe to throttle.
1883	1958	*/
1884	1959	static int current_may_throttle(void)
1885	1960	{
1886		- return !(current->flags & PF_LESS_THROTTLE) \|\|
	1961	+ return !(current->flags & PF_LOCAL_THROTTLE) \|\|
1887	1962	current->backing_dev_info == NULL \|\|
1888	1963	bdi_write_congested(current->backing_dev_info);
1889	1964	}
..	..	@@ -1898,13 +1973,12 @@
1898	1973	{
1899	1974	LIST_HEAD(page_list);
1900	1975	unsigned long nr_scanned;
1901		- unsigned long nr_reclaimed = 0;
	1976	+ unsigned int nr_reclaimed = 0;
1902	1977	unsigned long nr_taken;
1903		- struct reclaim_stat stat = {};
1904		- isolate_mode_t isolate_mode = 0;
1905		- int file = is_file_lru(lru);
	1978	+ struct reclaim_stat stat;
	1979	+ bool file = is_file_lru(lru);
	1980	+ enum vm_event_item item;
1906	1981	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1907		- struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1908	1982	bool stalled = false;
1909	1983
1910	1984	while (unlikely(too_many_isolated(pgdat, file, sc))) {
..	..	@@ -1922,54 +1996,37 @@
1922	1996
1923	1997	lru_add_drain();
1924	1998
1925		- if (!sc->may_unmap)
1926		- isolate_mode \|= ISOLATE_UNMAPPED;
1927		-
1928	1999	spin_lock_irq(&pgdat->lru_lock);
1929	2000
1930	2001	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
1931		- &nr_scanned, sc, isolate_mode, lru);
	2002	+ &nr_scanned, sc, lru);
1932	2003
1933	2004	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
1934		- reclaim_stat->recent_scanned[file] += nr_taken;
	2005	+ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
	2006	+ if (!cgroup_reclaim(sc))
	2007	+ __count_vm_events(item, nr_scanned);
	2008	+ __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
	2009	+ __count_vm_events(PGSCAN_ANON + file, nr_scanned);
1935	2010
1936		- if (current_is_kswapd()) {
1937		- if (global_reclaim(sc))
1938		- __count_vm_events(PGSCAN_KSWAPD, nr_scanned);
1939		- count_memcg_events(lruvec_memcg(lruvec), PGSCAN_KSWAPD,
1940		- nr_scanned);
1941		- } else {
1942		- if (global_reclaim(sc))
1943		- __count_vm_events(PGSCAN_DIRECT, nr_scanned);
1944		- count_memcg_events(lruvec_memcg(lruvec), PGSCAN_DIRECT,
1945		- nr_scanned);
1946		- }
1947	2011	spin_unlock_irq(&pgdat->lru_lock);
1948	2012
1949	2013	if (nr_taken == 0)
1950	2014	return 0;
1951	2015
1952		- nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, 0,
1953		- &stat, false);
	2016	+ nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false);
	2017	+ trace_android_vh_handle_failed_page_trylock(&page_list);
1954	2018
1955	2019	spin_lock_irq(&pgdat->lru_lock);
1956	2020
1957		- if (current_is_kswapd()) {
1958		- if (global_reclaim(sc))
1959		- __count_vm_events(PGSTEAL_KSWAPD, nr_reclaimed);
1960		- count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_KSWAPD,
1961		- nr_reclaimed);
1962		- } else {
1963		- if (global_reclaim(sc))
1964		- __count_vm_events(PGSTEAL_DIRECT, nr_reclaimed);
1965		- count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_DIRECT,
1966		- nr_reclaimed);
1967		- }
1968		-
1969		- putback_inactive_pages(lruvec, &page_list);
	2021	+ move_pages_to_lru(lruvec, &page_list);
1970	2022
1971	2023	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
1972		-
	2024	+ lru_note_cost(lruvec, file, stat.nr_pageout);
	2025	+ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
	2026	+ if (!cgroup_reclaim(sc))
	2027	+ __count_vm_events(item, nr_reclaimed);
	2028	+ __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
	2029	+ __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed);
1973	2030	spin_unlock_irq(&pgdat->lru_lock);
1974	2031
1975	2032	mem_cgroup_uncharge_list(&page_list);
..	..	@@ -2003,73 +2060,6 @@
2003	2060	return nr_reclaimed;
2004	2061	}
2005	2062
2006		-/*
2007		- * This moves pages from the active list to the inactive list.
2008		- *
2009		- * We move them the other way if the page is referenced by one or more
2010		- * processes, from rmap.
2011		- *
2012		- * If the pages are mostly unmapped, the processing is fast and it is
2013		- * appropriate to hold zone_lru_lock across the whole operation. But if
2014		- * the pages are mapped, the processing is slow (page_referenced()) so we
2015		- * should drop zone_lru_lock around each page. It's impossible to balance
2016		- * this, so instead we remove the pages from the LRU while processing them.
2017		- * It is safe to rely on PG_active against the non-LRU pages in here because
2018		- * nobody will play with that bit on a non-LRU page.
2019		- *
2020		- * The downside is that we have to touch page->_refcount against each page.
2021		- * But we had to alter page->flags anyway.
2022		- *
2023		- * Returns the number of pages moved to the given lru.
2024		- */
2025		-
2026		-static unsigned move_active_pages_to_lru(struct lruvec *lruvec,
2027		- struct list_head *list,
2028		- struct list_head *pages_to_free,
2029		- enum lru_list lru)
2030		-{
2031		- struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2032		- struct page *page;
2033		- int nr_pages;
2034		- int nr_moved = 0;
2035		-
2036		- while (!list_empty(list)) {
2037		- page = lru_to_page(list);
2038		- lruvec = mem_cgroup_page_lruvec(page, pgdat);
2039		-
2040		- VM_BUG_ON_PAGE(PageLRU(page), page);
2041		- SetPageLRU(page);
2042		-
2043		- nr_pages = hpage_nr_pages(page);
2044		- update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);
2045		- list_move(&page->lru, &lruvec->lists[lru]);
2046		-
2047		- if (put_page_testzero(page)) {
2048		- __ClearPageLRU(page);
2049		- __ClearPageActive(page);
2050		- del_page_from_lru_list(page, lruvec, lru);
2051		-
2052		- if (unlikely(PageCompound(page))) {
2053		- spin_unlock_irq(&pgdat->lru_lock);
2054		- mem_cgroup_uncharge(page);
2055		- (*get_compound_page_dtor(page))(page);
2056		- spin_lock_irq(&pgdat->lru_lock);
2057		- } else
2058		- list_add(&page->lru, pages_to_free);
2059		- } else {
2060		- nr_moved += nr_pages;
2061		- }
2062		- }
2063		-
2064		- if (!is_active_lru(lru)) {
2065		- __count_vm_events(PGDEACTIVATE, nr_moved);
2066		- count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
2067		- nr_moved);
2068		- }
2069		-
2070		- return nr_moved;
2071		-}
2072		-
2073	2063	static void shrink_active_list(unsigned long nr_to_scan,
2074	2064	struct lruvec *lruvec,
2075	2065	struct scan_control *sc,
..	..	@@ -2082,28 +2072,25 @@
2082	2072	LIST_HEAD(l_active);
2083	2073	LIST_HEAD(l_inactive);
2084	2074	struct page *page;
2085		- struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2086	2075	unsigned nr_deactivate, nr_activate;
2087	2076	unsigned nr_rotated = 0;
2088		- isolate_mode_t isolate_mode = 0;
2089	2077	int file = is_file_lru(lru);
2090	2078	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
	2079	+ bool bypass = false;
	2080	+ bool should_protect = false;
2091	2081
2092	2082	lru_add_drain();
2093		-
2094		- if (!sc->may_unmap)
2095		- isolate_mode \|= ISOLATE_UNMAPPED;
2096	2083
2097	2084	spin_lock_irq(&pgdat->lru_lock);
2098	2085
2099	2086	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
2100		- &nr_scanned, sc, isolate_mode, lru);
	2087	+ &nr_scanned, sc, lru);
2101	2088
2102	2089	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2103		- reclaim_stat->recent_scanned[file] += nr_taken;
2104	2090
2105		- __count_vm_events(PGREFILL, nr_scanned);
2106		- count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
	2091	+ if (!cgroup_reclaim(sc))
	2092	+ __count_vm_events(PGREFILL, nr_scanned);
	2093	+ __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2107	2094
2108	2095	spin_unlock_irq(&pgdat->lru_lock);
2109	2096
..	..	@@ -2125,9 +2112,20 @@
2125	2112	}
2126	2113	}
2127	2114
	2115	+ trace_android_vh_page_should_be_protected(page, &should_protect);
	2116	+ if (unlikely(should_protect)) {
	2117	+ nr_rotated += thp_nr_pages(page);
	2118	+ list_add(&page->lru, &l_active);
	2119	+ continue;
	2120	+ }
	2121	+
	2122	+ trace_android_vh_page_referenced_check_bypass(page, nr_to_scan, lru, &bypass);
	2123	+ if (bypass)
	2124	+ goto skip_page_referenced;
	2125	+ trace_android_vh_page_trylock_set(page);
	2126	+ /* Referenced or rmap lock contention: rotate */
2128	2127	if (page_referenced(page, 0, sc->target_mem_cgroup,
2129		- &vm_flags)) {
2130		- nr_rotated += hpage_nr_pages(page);
	2128	+ &vm_flags) != 0) {
2131	2129	/*
2132	2130	* Identify referenced, file-backed active pages and
2133	2131	* give them one more trip around the active list. So
..	..	@@ -2137,12 +2135,15 @@
2137	2135	* IO, plus JVM can create lots of anon VM_EXEC pages,
2138	2136	* so we ignore them here.
2139	2137	*/
2140		- if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
	2138	+ if ((vm_flags & VM_EXEC) && page_is_file_lru(page)) {
	2139	+ trace_android_vh_page_trylock_clear(page);
	2140	+ nr_rotated += thp_nr_pages(page);
2141	2141	list_add(&page->lru, &l_active);
2142	2142	continue;
2143	2143	}
2144	2144	}
2145		-
	2145	+ trace_android_vh_page_trylock_clear(page);
	2146	+skip_page_referenced:
2146	2147	ClearPageActive(page); /* we are de-activating */
2147	2148	SetPageWorkingset(page);
2148	2149	list_add(&page->lru, &l_inactive);
..	..	@@ -2152,23 +2153,91 @@
2152	2153	* Move pages back to the lru list.
2153	2154	*/
2154	2155	spin_lock_irq(&pgdat->lru_lock);
2155		- /*
2156		- * Count referenced pages from currently used mappings as rotated,
2157		- * even though only some of them are actually re-activated. This
2158		- * helps balance scan pressure between file and anonymous pages in
2159		- * get_scan_count.
2160		- */
2161		- reclaim_stat->recent_rotated[file] += nr_rotated;
2162	2156
2163		- nr_activate = move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
2164		- nr_deactivate = move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
	2157	+ nr_activate = move_pages_to_lru(lruvec, &l_active);
	2158	+ nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);
	2159	+ /* Keep all free pages in l_active list */
	2160	+ list_splice(&l_inactive, &l_active);
	2161	+
	2162	+ __count_vm_events(PGDEACTIVATE, nr_deactivate);
	2163	+ __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
	2164	+
2165	2165	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2166	2166	spin_unlock_irq(&pgdat->lru_lock);
2167	2167
2168		- mem_cgroup_uncharge_list(&l_hold);
2169		- free_unref_page_list(&l_hold);
	2168	+ mem_cgroup_uncharge_list(&l_active);
	2169	+ free_unref_page_list(&l_active);
2170	2170	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2171	2171	nr_deactivate, nr_rotated, sc->priority, file);
	2172	+}
	2173	+
	2174	+unsigned long reclaim_pages(struct list_head *page_list)
	2175	+{
	2176	+ int nid = NUMA_NO_NODE;
	2177	+ unsigned int nr_reclaimed = 0;
	2178	+ LIST_HEAD(node_page_list);
	2179	+ struct reclaim_stat dummy_stat;
	2180	+ struct page *page;
	2181	+ struct scan_control sc = {
	2182	+ .gfp_mask = GFP_KERNEL,
	2183	+ .priority = DEF_PRIORITY,
	2184	+ .may_writepage = 1,
	2185	+ .may_unmap = 1,
	2186	+ .may_swap = 1,
	2187	+ };
	2188	+
	2189	+ while (!list_empty(page_list)) {
	2190	+ page = lru_to_page(page_list);
	2191	+ if (nid == NUMA_NO_NODE) {
	2192	+ nid = page_to_nid(page);
	2193	+ INIT_LIST_HEAD(&node_page_list);
	2194	+ }
	2195	+
	2196	+ if (nid == page_to_nid(page)) {
	2197	+ ClearPageActive(page);
	2198	+ list_move(&page->lru, &node_page_list);
	2199	+ continue;
	2200	+ }
	2201	+
	2202	+ nr_reclaimed += shrink_page_list(&node_page_list,
	2203	+ NODE_DATA(nid),
	2204	+ &sc, &dummy_stat, false);
	2205	+ while (!list_empty(&node_page_list)) {
	2206	+ page = lru_to_page(&node_page_list);
	2207	+ list_del(&page->lru);
	2208	+ putback_lru_page(page);
	2209	+ }
	2210	+
	2211	+ nid = NUMA_NO_NODE;
	2212	+ }
	2213	+
	2214	+ if (!list_empty(&node_page_list)) {
	2215	+ nr_reclaimed += shrink_page_list(&node_page_list,
	2216	+ NODE_DATA(nid),
	2217	+ &sc, &dummy_stat, false);
	2218	+ while (!list_empty(&node_page_list)) {
	2219	+ page = lru_to_page(&node_page_list);
	2220	+ list_del(&page->lru);
	2221	+ putback_lru_page(page);
	2222	+ }
	2223	+ }
	2224	+
	2225	+ return nr_reclaimed;
	2226	+}
	2227	+EXPORT_SYMBOL_GPL(reclaim_pages);
	2228	+
	2229	+static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
	2230	+ struct lruvec lruvec, struct scan_control sc)
	2231	+{
	2232	+ if (is_active_lru(lru)) {
	2233	+ if (sc->may_deactivate & (1 << is_file_lru(lru)))
	2234	+ shrink_active_list(nr_to_scan, lruvec, sc, lru);
	2235	+ else
	2236	+ sc->skipped_deactivate = 1;
	2237	+ return 0;
	2238	+ }
	2239	+
	2240	+ return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2172	2241	}
2173	2242
2174	2243	/*
..	..	@@ -2199,62 +2268,31 @@
2199	2268	* 1TB 101 10GB
2200	2269	* 10TB 320 32GB
2201	2270	*/
2202		-static bool inactive_list_is_low(struct lruvec *lruvec, bool file,
2203		- struct scan_control *sc, bool trace)
	2271	+static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2204	2272	{
2205		- enum lru_list active_lru = file * LRU_FILE + LRU_ACTIVE;
2206		- struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2207		- enum lru_list inactive_lru = file * LRU_FILE;
	2273	+ enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2208	2274	unsigned long inactive, active;
2209	2275	unsigned long inactive_ratio;
2210		- unsigned long refaults;
2211	2276	unsigned long gb;
	2277	+ bool skip = false;
2212	2278
2213		- /*
2214		- * If we don't have swap space, anonymous page deactivation
2215		- * is pointless.
2216		- */
2217		- if (!file && !total_swap_pages)
2218		- return false;
	2279	+ inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
	2280	+ active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2219	2281
2220		- inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);
2221		- active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx);
	2282	+ gb = (inactive + active) >> (30 - PAGE_SHIFT);
	2283	+ trace_android_vh_inactive_is_low(gb, &inactive_ratio, inactive_lru, &skip);
	2284	+ if (skip)
	2285	+ goto out;
2222	2286
2223		- /*
2224		- * When refaults are being observed, it means a new workingset
2225		- * is being established. Disable active list protection to get
2226		- * rid of the stale workingset quickly.
2227		- */
2228		- refaults = lruvec_page_state(lruvec, WORKINGSET_ACTIVATE);
2229		- if (file && lruvec->refaults != refaults) {
2230		- inactive_ratio = 0;
2231		- } else {
2232		- gb = (inactive + active) >> (30 - PAGE_SHIFT);
2233		- if (gb)
2234		- inactive_ratio = int_sqrt(10 * gb);
2235		- else
2236		- inactive_ratio = 1;
2237		- }
	2287	+ if (gb)
	2288	+ inactive_ratio = int_sqrt(10 * gb);
	2289	+ else
	2290	+ inactive_ratio = 1;
2238	2291
2239		- if (trace)
2240		- trace_mm_vmscan_inactive_list_is_low(pgdat->node_id, sc->reclaim_idx,
2241		- lruvec_lru_size(lruvec, inactive_lru, MAX_NR_ZONES), inactive,
2242		- lruvec_lru_size(lruvec, active_lru, MAX_NR_ZONES), active,
2243		- inactive_ratio, file);
	2292	+ trace_android_vh_tune_inactive_ratio(&inactive_ratio, is_file_lru(inactive_lru));
2244	2293
	2294	+out:
2245	2295	return inactive * inactive_ratio < active;
2246		-}
2247		-
2248		-static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2249		- struct lruvec lruvec, struct scan_control sc)
2250		-{
2251		- if (is_active_lru(lru)) {
2252		- if (inactive_list_is_low(lruvec, is_file_lru(lru), sc, true))
2253		- shrink_active_list(nr_to_scan, lruvec, sc, lru);
2254		- return 0;
2255		- }
2256		-
2257		- return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2258	2296	}
2259	2297
2260	2298	enum scan_balance {
..	..	@@ -2273,20 +2311,18 @@
2273	2311	* nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2274	2312	* nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2275	2313	*/
2276		-static void get_scan_count(struct lruvec lruvec, struct mem_cgroup memcg,
2277		- struct scan_control sc, unsigned long nr,
2278		- unsigned long *lru_pages)
	2314	+static void get_scan_count(struct lruvec lruvec, struct scan_control sc,
	2315	+ unsigned long *nr)
2279	2316	{
	2317	+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
	2318	+ unsigned long anon_cost, file_cost, total_cost;
2280	2319	int swappiness = mem_cgroup_swappiness(memcg);
2281		- struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2282		- u64 fraction[2];
	2320	+ u64 fraction[ANON_AND_FILE];
2283	2321	u64 denominator = 0; /* gcc */
2284		- struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2285		- unsigned long anon_prio, file_prio;
2286	2322	enum scan_balance scan_balance;
2287		- unsigned long anon, file;
2288	2323	unsigned long ap, fp;
2289	2324	enum lru_list lru;
	2325	+ bool balance_anon_file_reclaim = false;
2290	2326
2291	2327	/* If we have no swap space, do not bother scanning anon pages. */
2292	2328	if (!sc->may_swap \|\| mem_cgroup_get_nr_swap_pages(memcg) <= 0) {
..	..	@@ -2294,6 +2330,7 @@
2294	2330	goto out;
2295	2331	}
2296	2332
	2333	+ trace_android_vh_tune_swappiness(&swappiness);
2297	2334	/*
2298	2335	* Global reclaim will swap to prevent OOM even with no
2299	2336	* swappiness, but memcg users want to use this knob to
..	..	@@ -2301,7 +2338,7 @@
2301	2338	* using the memory controller's swap limit feature would be
2302	2339	* too expensive.
2303	2340	*/
2304		- if (!global_reclaim(sc) && !swappiness) {
	2341	+ if (cgroup_reclaim(sc) && !swappiness) {
2305	2342	scan_balance = SCAN_FILE;
2306	2343	goto out;
2307	2344	}
..	..	@@ -2317,129 +2354,133 @@
2317	2354	}
2318	2355
2319	2356	/*
2320		- * Prevent the reclaimer from falling into the cache trap: as
2321		- * cache pages start out inactive, every cache fault will tip
2322		- * the scan balance towards the file LRU. And as the file LRU
2323		- * shrinks, so does the window for rotation from references.
2324		- * This means we have a runaway feedback loop where a tiny
2325		- * thrashing file LRU becomes infinitely more attractive than
2326		- * anon pages. Try to detect this based on file LRU size.
	2357	+ * If the system is almost out of file pages, force-scan anon.
2327	2358	*/
2328		- if (global_reclaim(sc)) {
2329		- unsigned long pgdatfile;
2330		- unsigned long pgdatfree;
2331		- int z;
2332		- unsigned long total_high_wmark = 0;
2333		-
2334		- pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2335		- pgdatfile = node_page_state(pgdat, NR_ACTIVE_FILE) +
2336		- node_page_state(pgdat, NR_INACTIVE_FILE);
2337		-
2338		- for (z = 0; z < MAX_NR_ZONES; z++) {
2339		- struct zone *zone = &pgdat->node_zones[z];
2340		- if (!managed_zone(zone))
2341		- continue;
2342		-
2343		- total_high_wmark += high_wmark_pages(zone);
2344		- }
2345		-
2346		- if (unlikely(pgdatfile + pgdatfree <= total_high_wmark)) {
2347		- /*
2348		- * Force SCAN_ANON if there are enough inactive
2349		- * anonymous pages on the LRU in eligible zones.
2350		- * Otherwise, the small LRU gets thrashed.
2351		- */
2352		- if (!inactive_list_is_low(lruvec, false, sc, false) &&
2353		- lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, sc->reclaim_idx)
2354		- >> sc->priority) {
2355		- scan_balance = SCAN_ANON;
2356		- goto out;
2357		- }
2358		- }
	2359	+ if (sc->file_is_tiny) {
	2360	+ scan_balance = SCAN_ANON;
	2361	+ goto out;
2359	2362	}
2360	2363
	2364	+ trace_android_rvh_set_balance_anon_file_reclaim(&balance_anon_file_reclaim);
	2365	+
2361	2366	/*
2362		- * If there is enough inactive page cache, i.e. if the size of the
2363		- * inactive list is greater than that of the active list and the
2364		- * inactive list actually has some pages to scan on this priority, we
2365		- * do not reclaim anything from the anonymous working set right now.
2366		- * Without the second condition we could end up never scanning an
2367		- * lruvec even if it has plenty of old anonymous pages unless the
2368		- * system is under heavy pressure.
	2367	+ * If there is enough inactive page cache, we do not reclaim
	2368	+ * anything from the anonymous working right now. But when balancing
	2369	+ * anon and page cache files for reclaim, allow swapping of anon pages
	2370	+ * even if there are a number of inactive file cache pages.
2369	2371	*/
2370		- if (!inactive_list_is_low(lruvec, true, sc, false) &&
2371		- lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) {
	2372	+ if (!balance_anon_file_reclaim && sc->cache_trim_mode) {
2372	2373	scan_balance = SCAN_FILE;
2373	2374	goto out;
2374	2375	}
2375	2376
2376	2377	scan_balance = SCAN_FRACT;
2377		-
2378	2378	/*
2379		- * With swappiness at 100, anonymous and file have the same priority.
2380		- * This scanning priority is essentially the inverse of IO cost.
2381		- */
2382		- anon_prio = swappiness;
2383		- file_prio = 200 - anon_prio;
2384		-
2385		- /*
2386		- * OK, so we have swap space and a fair amount of page cache
2387		- * pages. We use the recently rotated / recently scanned
2388		- * ratios to determine how valuable each cache is.
	2379	+ * Calculate the pressure balance between anon and file pages.
2389	2380	*
2390		- * Because workloads change over time (and to avoid overflow)
2391		- * we keep these statistics as a floating average, which ends
2392		- * up weighing recent references more than old ones.
	2381	+ * The amount of pressure we put on each LRU is inversely
	2382	+ * proportional to the cost of reclaiming each list, as
	2383	+ * determined by the share of pages that are refaulting, times
	2384	+ * the relative IO cost of bringing back a swapped out
	2385	+ * anonymous page vs reloading a filesystem page (swappiness).
2393	2386	*
2394		- * anon in [0], file in [1]
	2387	+ * Although we limit that influence to ensure no list gets
	2388	+ * left behind completely: at least a third of the pressure is
	2389	+ * applied, before swappiness.
	2390	+ *
	2391	+ * With swappiness at 100, anon and file have equal IO cost.
2395	2392	*/
	2393	+ total_cost = sc->anon_cost + sc->file_cost;
	2394	+ anon_cost = total_cost + sc->anon_cost;
	2395	+ file_cost = total_cost + sc->file_cost;
	2396	+ total_cost = anon_cost + file_cost;
2396	2397
2397		- anon = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +
2398		- lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, MAX_NR_ZONES);
2399		- file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) +
2400		- lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);
	2398	+ ap = swappiness * (total_cost + 1);
	2399	+ ap /= anon_cost + 1;
2401	2400
2402		- spin_lock_irq(&pgdat->lru_lock);
2403		- if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
2404		- reclaim_stat->recent_scanned[0] /= 2;
2405		- reclaim_stat->recent_rotated[0] /= 2;
2406		- }
2407		-
2408		- if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
2409		- reclaim_stat->recent_scanned[1] /= 2;
2410		- reclaim_stat->recent_rotated[1] /= 2;
2411		- }
2412		-
2413		- /*
2414		- * The amount of pressure on anon vs file pages is inversely
2415		- * proportional to the fraction of recently scanned pages on
2416		- * each list that were recently referenced and in active use.
2417		- */
2418		- ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
2419		- ap /= reclaim_stat->recent_rotated[0] + 1;
2420		-
2421		- fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
2422		- fp /= reclaim_stat->recent_rotated[1] + 1;
2423		- spin_unlock_irq(&pgdat->lru_lock);
	2401	+ fp = (200 - swappiness) * (total_cost + 1);
	2402	+ fp /= file_cost + 1;
2424	2403
2425	2404	fraction[0] = ap;
2426	2405	fraction[1] = fp;
2427		- denominator = ap + fp + 1;
	2406	+ denominator = ap + fp;
2428	2407	out:
2429		- *lru_pages = 0;
	2408	+ trace_android_vh_tune_scan_type((char *)(&scan_balance));
2430	2409	for_each_evictable_lru(lru) {
2431	2410	int file = is_file_lru(lru);
2432		- unsigned long size;
	2411	+ unsigned long lruvec_size;
	2412	+ unsigned long low, min;
2433	2413	unsigned long scan;
2434	2414
2435		- size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2436		- scan = size >> sc->priority;
	2415	+ lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
	2416	+ mem_cgroup_protection(sc->target_mem_cgroup, memcg,
	2417	+ &min, &low);
	2418	+
	2419	+ if (min \|\| low) {
	2420	+ /*
	2421	+ * Scale a cgroup's reclaim pressure by proportioning
	2422	+ * its current usage to its memory.low or memory.min
	2423	+ * setting.
	2424	+ *
	2425	+ * This is important, as otherwise scanning aggression
	2426	+ * becomes extremely binary -- from nothing as we
	2427	+ * approach the memory protection threshold, to totally
	2428	+ * nominal as we exceed it. This results in requiring
	2429	+ * setting extremely liberal protection thresholds. It
	2430	+ * also means we simply get no protection at all if we
	2431	+ * set it too low, which is not ideal.
	2432	+ *
	2433	+ * If there is any protection in place, we reduce scan
	2434	+ * pressure by how much of the total memory used is
	2435	+ * within protection thresholds.
	2436	+ *
	2437	+ * There is one special case: in the first reclaim pass,
	2438	+ * we skip over all groups that are within their low
	2439	+ * protection. If that fails to reclaim enough pages to
	2440	+ * satisfy the reclaim goal, we come back and override
	2441	+ * the best-effort low protection. However, we still
	2442	+ * ideally want to honor how well-behaved groups are in
	2443	+ * that case instead of simply punishing them all
	2444	+ * equally. As such, we reclaim them based on how much
	2445	+ * memory they are using, reducing the scan pressure
	2446	+ * again by how much of the total memory used is under
	2447	+ * hard protection.
	2448	+ */
	2449	+ unsigned long cgroup_size = mem_cgroup_size(memcg);
	2450	+ unsigned long protection;
	2451	+
	2452	+ /* memory.low scaling, make sure we retry before OOM */
	2453	+ if (!sc->memcg_low_reclaim && low > min) {
	2454	+ protection = low;
	2455	+ sc->memcg_low_skipped = 1;
	2456	+ } else {
	2457	+ protection = min;
	2458	+ }
	2459	+
	2460	+ /* Avoid TOCTOU with earlier protection check */
	2461	+ cgroup_size = max(cgroup_size, protection);
	2462	+
	2463	+ scan = lruvec_size - lruvec_size * protection /
	2464	+ (cgroup_size + 1);
	2465	+
	2466	+ /*
	2467	+ * Minimally target SWAP_CLUSTER_MAX pages to keep
	2468	+ * reclaim moving forwards, avoiding decrementing
	2469	+ * sc->priority further than desirable.
	2470	+ */
	2471	+ scan = max(scan, SWAP_CLUSTER_MAX);
	2472	+ } else {
	2473	+ scan = lruvec_size;
	2474	+ }
	2475	+
	2476	+ scan >>= sc->priority;
	2477	+
2437	2478	/*
2438	2479	* If the cgroup's already been deleted, make sure to
2439	2480	* scrape out the remaining cache.
2440	2481	*/
2441	2482	if (!scan && !mem_cgroup_online(memcg))
2442		- scan = min(size, SWAP_CLUSTER_MAX);
	2483	+ scan = min(lruvec_size, SWAP_CLUSTER_MAX);
2443	2484
2444	2485	switch (scan_balance) {
2445	2486	case SCAN_EQUAL:
..	..	@@ -2461,38 +2502,30 @@
2461	2502	case SCAN_FILE:
2462	2503	case SCAN_ANON:
2463	2504	/* Scan one type exclusively */
2464		- if ((scan_balance == SCAN_FILE) != file) {
2465		- size = 0;
	2505	+ if ((scan_balance == SCAN_FILE) != file)
2466	2506	scan = 0;
2467		- }
2468	2507	break;
2469	2508	default:
2470	2509	/* Look ma, no brain */
2471	2510	BUG();
2472	2511	}
2473	2512
2474		- *lru_pages += size;
2475	2513	nr[lru] = scan;
2476	2514	}
2477	2515	}
2478	2516
2479		-/*
2480		- * This is a basic per-node page freer. Used by both kswapd and direct reclaim.
2481		- */
2482		-static void shrink_node_memcg(struct pglist_data pgdat, struct mem_cgroup memcg,
2483		- struct scan_control sc, unsigned long lru_pages)
	2517	+static void shrink_lruvec(struct lruvec lruvec, struct scan_control sc)
2484	2518	{
2485		- struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
2486	2519	unsigned long nr[NR_LRU_LISTS];
2487	2520	unsigned long targets[NR_LRU_LISTS];
2488	2521	unsigned long nr_to_scan;
2489	2522	enum lru_list lru;
2490	2523	unsigned long nr_reclaimed = 0;
2491	2524	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
	2525	+ bool proportional_reclaim;
2492	2526	struct blk_plug plug;
2493		- bool scan_adjusted;
2494	2527
2495		- get_scan_count(lruvec, memcg, sc, nr, lru_pages);
	2528	+ get_scan_count(lruvec, sc, nr);
2496	2529
2497	2530	/* Record the original scan target for proportional adjustments later */
2498	2531	memcpy(targets, nr, sizeof(nr));
..	..	@@ -2508,8 +2541,8 @@
2508	2541	* abort proportional reclaim if either the file or anon lru has already
2509	2542	* dropped to zero at the first pass.
2510	2543	*/
2511		- scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
2512		- sc->priority == DEF_PRIORITY);
	2544	+ proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
	2545	+ sc->priority == DEF_PRIORITY);
2513	2546
2514	2547	blk_start_plug(&plug);
2515	2548	while (nr[LRU_INACTIVE_ANON] \|\| nr[LRU_ACTIVE_FILE] \|\|
..	..	@@ -2529,7 +2562,7 @@
2529	2562
2530	2563	cond_resched();
2531	2564
2532		- if (nr_reclaimed < nr_to_reclaim \|\| scan_adjusted)
	2565	+ if (nr_reclaimed < nr_to_reclaim \|\| proportional_reclaim)
2533	2566	continue;
2534	2567
2535	2568	/*
..	..	@@ -2580,8 +2613,6 @@
2580	2613	nr_scanned = targets[lru] - nr[lru];
2581	2614	nr[lru] = targets[lru] * (100 - percentage) / 100;
2582	2615	nr[lru] -= min(nr[lru], nr_scanned);
2583		-
2584		- scan_adjusted = true;
2585	2616	}
2586	2617	blk_finish_plug(&plug);
2587	2618	sc->nr_reclaimed += nr_reclaimed;
..	..	@@ -2590,7 +2621,7 @@
2590	2621	* Even if we did not try to evict anon pages at all, we want to
2591	2622	* rebalance the anon lru active/inactive ratio.
2592	2623	*/
2593		- if (inactive_list_is_low(lruvec, false, sc, true))
	2624	+ if (total_swap_pages && inactive_is_low(lruvec, LRU_INACTIVE_ANON))
2594	2625	shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2595	2626	sc, LRU_ACTIVE_ANON);
2596	2627	}
..	..	@@ -2610,12 +2641,11 @@
2610	2641	* Reclaim/compaction is used for high-order allocation requests. It reclaims
2611	2642	* order-0 pages before compacting the zone. should_continue_reclaim() returns
2612	2643	* true if more pages should be reclaimed such that when the page allocator
2613		- * calls try_to_compact_zone() that it will have enough free pages to succeed.
	2644	+ * calls try_to_compact_pages() that it will have enough free pages to succeed.
2614	2645	* It will give up earlier than that if there is difficulty reclaiming pages.
2615	2646	*/
2616	2647	static inline bool should_continue_reclaim(struct pglist_data *pgdat,
2617	2648	unsigned long nr_reclaimed,
2618		- unsigned long nr_scanned,
2619	2649	struct scan_control *sc)
2620	2650	{
2621	2651	unsigned long pages_for_compaction;
..	..	@@ -2626,40 +2656,18 @@
2626	2656	if (!in_reclaim_compaction(sc))
2627	2657	return false;
2628	2658
2629		- /* Consider stopping depending on scan and reclaim activity */
2630		- if (sc->gfp_mask & __GFP_RETRY_MAYFAIL) {
2631		- /*
2632		- * For __GFP_RETRY_MAYFAIL allocations, stop reclaiming if the
2633		- * full LRU list has been scanned and we are still failing
2634		- * to reclaim pages. This full LRU scan is potentially
2635		- * expensive but a __GFP_RETRY_MAYFAIL caller really wants to succeed
2636		- */
2637		- if (!nr_reclaimed && !nr_scanned)
2638		- return false;
2639		- } else {
2640		- /*
2641		- * For non-__GFP_RETRY_MAYFAIL allocations which can presumably
2642		- * fail without consequence, stop if we failed to reclaim
2643		- * any pages from the last SWAP_CLUSTER_MAX number of
2644		- * pages that were scanned. This will return to the
2645		- * caller faster at the risk reclaim/compaction and
2646		- * the resulting allocation attempt fails
2647		- */
2648		- if (!nr_reclaimed)
2649		- return false;
2650		- }
2651		-
2652	2659	/*
2653		- * If we have not reclaimed enough pages for compaction and the
2654		- * inactive lists are large enough, continue reclaiming
	2660	+ * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
	2661	+ * number of pages that were scanned. This will return to the caller
	2662	+ * with the risk reclaim/compaction and the resulting allocation attempt
	2663	+ * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
	2664	+ * allocations through requiring that the full LRU list has been scanned
	2665	+ * first, by assuming that zero delta of sc->nr_scanned means full LRU
	2666	+ * scan, but that approximation was wrong, and there were corner cases
	2667	+ * where always a non-zero amount of pages were scanned.
2655	2668	*/
2656		- pages_for_compaction = compact_gap(sc->order);
2657		- inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
2658		- if (get_nr_swap_pages() > 0)
2659		- inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
2660		- if (sc->nr_reclaimed < pages_for_compaction &&
2661		- inactive_lru_pages > pages_for_compaction)
2662		- return true;
	2669	+ if (!nr_reclaimed)
	2670	+ return false;
2663	2671
2664	2672	/* If compaction would go ahead or the allocation would succeed, stop */
2665	2673	for (z = 0; z <= sc->reclaim_idx; z++) {
..	..	@@ -2676,179 +2684,262 @@
2676	2684	;
2677	2685	}
2678	2686	}
2679		- return true;
	2687	+
	2688	+ /*
	2689	+ * If we have not reclaimed enough pages for compaction and the
	2690	+ * inactive lists are large enough, continue reclaiming
	2691	+ */
	2692	+ pages_for_compaction = compact_gap(sc->order);
	2693	+ inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
	2694	+ if (get_nr_swap_pages() > 0)
	2695	+ inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
	2696	+
	2697	+ return inactive_lru_pages > pages_for_compaction;
2680	2698	}
2681	2699
2682		-static bool pgdat_memcg_congested(pg_data_t pgdat, struct mem_cgroup memcg)
	2700	+static void shrink_node_memcgs(pg_data_t pgdat, struct scan_control sc)
2683	2701	{
2684		- return test_bit(PGDAT_CONGESTED, &pgdat->flags) \|\|
2685		- (memcg && memcg_congested(pgdat, memcg));
	2702	+ struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
	2703	+ struct mem_cgroup *memcg;
	2704	+
	2705	+ memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
	2706	+ do {
	2707	+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
	2708	+ unsigned long reclaimed;
	2709	+ unsigned long scanned;
	2710	+ bool skip = false;
	2711	+
	2712	+ /*
	2713	+ * This loop can become CPU-bound when target memcgs
	2714	+ * aren't eligible for reclaim - either because they
	2715	+ * don't have any reclaimable pages, or because their
	2716	+ * memory is explicitly protected. Avoid soft lockups.
	2717	+ */
	2718	+ cond_resched();
	2719	+
	2720	+ trace_android_vh_shrink_node_memcgs(memcg, &skip);
	2721	+ if (skip)
	2722	+ continue;
	2723	+
	2724	+ mem_cgroup_calculate_protection(target_memcg, memcg);
	2725	+
	2726	+ if (mem_cgroup_below_min(memcg)) {
	2727	+ /*
	2728	+ * Hard protection.
	2729	+ * If there is no reclaimable memory, OOM.
	2730	+ */
	2731	+ continue;
	2732	+ } else if (mem_cgroup_below_low(memcg)) {
	2733	+ /*
	2734	+ * Soft protection.
	2735	+ * Respect the protection only as long as
	2736	+ * there is an unprotected supply
	2737	+ * of reclaimable memory from other cgroups.
	2738	+ */
	2739	+ if (!sc->memcg_low_reclaim) {
	2740	+ sc->memcg_low_skipped = 1;
	2741	+ continue;
	2742	+ }
	2743	+ memcg_memory_event(memcg, MEMCG_LOW);
	2744	+ }
	2745	+
	2746	+ reclaimed = sc->nr_reclaimed;
	2747	+ scanned = sc->nr_scanned;
	2748	+
	2749	+ shrink_lruvec(lruvec, sc);
	2750	+
	2751	+ shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
	2752	+ sc->priority);
	2753	+
	2754	+ /* Record the group's reclaim efficiency */
	2755	+ vmpressure(sc->gfp_mask, memcg, false,
	2756	+ sc->nr_scanned - scanned,
	2757	+ sc->nr_reclaimed - reclaimed);
	2758	+
	2759	+ } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
2686	2760	}
2687	2761
2688		-static bool shrink_node(pg_data_t pgdat, struct scan_control sc)
	2762	+static void shrink_node(pg_data_t pgdat, struct scan_control sc)
2689	2763	{
2690	2764	struct reclaim_state *reclaim_state = current->reclaim_state;
2691	2765	unsigned long nr_reclaimed, nr_scanned;
	2766	+ struct lruvec *target_lruvec;
2692	2767	bool reclaimable = false;
	2768	+ unsigned long file;
2693	2769
2694		- do {
2695		- struct mem_cgroup *root = sc->target_mem_cgroup;
2696		- struct mem_cgroup_reclaim_cookie reclaim = {
2697		- .pgdat = pgdat,
2698		- .priority = sc->priority,
2699		- };
2700		- unsigned long node_lru_pages = 0;
2701		- struct mem_cgroup *memcg;
	2770	+ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
2702	2771
2703		- memset(&sc->nr, 0, sizeof(sc->nr));
	2772	+again:
	2773	+ memset(&sc->nr, 0, sizeof(sc->nr));
2704	2774
2705		- nr_reclaimed = sc->nr_reclaimed;
2706		- nr_scanned = sc->nr_scanned;
	2775	+ nr_reclaimed = sc->nr_reclaimed;
	2776	+ nr_scanned = sc->nr_scanned;
2707	2777
2708		- memcg = mem_cgroup_iter(root, NULL, &reclaim);
2709		- do {
2710		- unsigned long lru_pages;
2711		- unsigned long reclaimed;
2712		- unsigned long scanned;
	2778	+ /*
	2779	+ * Determine the scan balance between anon and file LRUs.
	2780	+ */
	2781	+ spin_lock_irq(&pgdat->lru_lock);
	2782	+ sc->anon_cost = target_lruvec->anon_cost;
	2783	+ sc->file_cost = target_lruvec->file_cost;
	2784	+ spin_unlock_irq(&pgdat->lru_lock);
2713	2785
2714		- /*
2715		- * This loop can become CPU-bound when target memcgs
2716		- * aren't eligible for reclaim - either because they
2717		- * don't have any reclaimable pages, or because their
2718		- * memory is explicitly protected. Avoid soft lockups.
2719		- */
2720		- cond_resched();
	2786	+ /*
	2787	+ * Target desirable inactive:active list ratios for the anon
	2788	+ * and file LRU lists.
	2789	+ */
	2790	+ if (!sc->force_deactivate) {
	2791	+ unsigned long refaults;
2721	2792
2722		- switch (mem_cgroup_protected(root, memcg)) {
2723		- case MEMCG_PROT_MIN:
2724		- /*
2725		- * Hard protection.
2726		- * If there is no reclaimable memory, OOM.
2727		- */
	2793	+ refaults = lruvec_page_state(target_lruvec,
	2794	+ WORKINGSET_ACTIVATE_ANON);
	2795	+ if (refaults != target_lruvec->refaults[0] \|\|
	2796	+ inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
	2797	+ sc->may_deactivate \|= DEACTIVATE_ANON;
	2798	+ else
	2799	+ sc->may_deactivate &= ~DEACTIVATE_ANON;
	2800	+
	2801	+ /*
	2802	+ * When refaults are being observed, it means a new
	2803	+ * workingset is being established. Deactivate to get
	2804	+ * rid of any stale active pages quickly.
	2805	+ */
	2806	+ refaults = lruvec_page_state(target_lruvec,
	2807	+ WORKINGSET_ACTIVATE_FILE);
	2808	+ if (refaults != target_lruvec->refaults[1] \|\|
	2809	+ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
	2810	+ sc->may_deactivate \|= DEACTIVATE_FILE;
	2811	+ else
	2812	+ sc->may_deactivate &= ~DEACTIVATE_FILE;
	2813	+ } else
	2814	+ sc->may_deactivate = DEACTIVATE_ANON \| DEACTIVATE_FILE;
	2815	+
	2816	+ /*
	2817	+ * If we have plenty of inactive file pages that aren't
	2818	+ * thrashing, try to reclaim those first before touching
	2819	+ * anonymous pages.
	2820	+ */
	2821	+ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
	2822	+ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
	2823	+ sc->cache_trim_mode = 1;
	2824	+ else
	2825	+ sc->cache_trim_mode = 0;
	2826	+
	2827	+ /*
	2828	+ * Prevent the reclaimer from falling into the cache trap: as
	2829	+ * cache pages start out inactive, every cache fault will tip
	2830	+ * the scan balance towards the file LRU. And as the file LRU
	2831	+ * shrinks, so does the window for rotation from references.
	2832	+ * This means we have a runaway feedback loop where a tiny
	2833	+ * thrashing file LRU becomes infinitely more attractive than
	2834	+ * anon pages. Try to detect this based on file LRU size.
	2835	+ */
	2836	+ if (!cgroup_reclaim(sc)) {
	2837	+ unsigned long total_high_wmark = 0;
	2838	+ unsigned long free, anon;
	2839	+ int z;
	2840	+
	2841	+ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
	2842	+ file = node_page_state(pgdat, NR_ACTIVE_FILE) +
	2843	+ node_page_state(pgdat, NR_INACTIVE_FILE);
	2844	+
	2845	+ for (z = 0; z < MAX_NR_ZONES; z++) {
	2846	+ struct zone *zone = &pgdat->node_zones[z];
	2847	+ if (!managed_zone(zone))
2728	2848	continue;
2729		- case MEMCG_PROT_LOW:
2730		- /*
2731		- * Soft protection.
2732		- * Respect the protection only as long as
2733		- * there is an unprotected supply
2734		- * of reclaimable memory from other cgroups.
2735		- */
2736		- if (!sc->memcg_low_reclaim) {
2737		- sc->memcg_low_skipped = 1;
2738		- continue;
2739		- }
2740		- memcg_memory_event(memcg, MEMCG_LOW);
2741		- break;
2742		- case MEMCG_PROT_NONE:
2743		- break;
2744		- }
2745	2849
2746		- reclaimed = sc->nr_reclaimed;
2747		- scanned = sc->nr_scanned;
2748		- shrink_node_memcg(pgdat, memcg, sc, &lru_pages);
2749		- node_lru_pages += lru_pages;
2750		-
2751		- shrink_slab(sc->gfp_mask, pgdat->node_id,
2752		- memcg, sc->priority);
2753		-
2754		- /* Record the group's reclaim efficiency */
2755		- vmpressure(sc->gfp_mask, memcg, false,
2756		- sc->nr_scanned - scanned,
2757		- sc->nr_reclaimed - reclaimed);
2758		-
2759		- /*
2760		- * Direct reclaim and kswapd have to scan all memory
2761		- * cgroups to fulfill the overall scan target for the
2762		- * node.
2763		- *
2764		- * Limit reclaim, on the other hand, only cares about
2765		- * nr_to_reclaim pages to be reclaimed and it will
2766		- * retry with decreasing priority if one round over the
2767		- * whole hierarchy is not sufficient.
2768		- */
2769		- if (!global_reclaim(sc) &&
2770		- sc->nr_reclaimed >= sc->nr_to_reclaim) {
2771		- mem_cgroup_iter_break(root, memcg);
2772		- break;
2773		- }
2774		- } while ((memcg = mem_cgroup_iter(root, memcg, &reclaim)));
2775		-
2776		- if (reclaim_state) {
2777		- sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2778		- reclaim_state->reclaimed_slab = 0;
2779		- }
2780		-
2781		- /* Record the subtree's reclaim efficiency */
2782		- vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2783		- sc->nr_scanned - nr_scanned,
2784		- sc->nr_reclaimed - nr_reclaimed);
2785		-
2786		- if (sc->nr_reclaimed - nr_reclaimed)
2787		- reclaimable = true;
2788		-
2789		- if (current_is_kswapd()) {
2790		- /*
2791		- * If reclaim is isolating dirty pages under writeback,
2792		- * it implies that the long-lived page allocation rate
2793		- * is exceeding the page laundering rate. Either the
2794		- * global limits are not being effective at throttling
2795		- * processes due to the page distribution throughout
2796		- * zones or there is heavy usage of a slow backing
2797		- * device. The only option is to throttle from reclaim
2798		- * context which is not ideal as there is no guarantee
2799		- * the dirtying process is throttled in the same way
2800		- * balance_dirty_pages() manages.
2801		- *
2802		- * Once a node is flagged PGDAT_WRITEBACK, kswapd will
2803		- * count the number of pages under pages flagged for
2804		- * immediate reclaim and stall if any are encountered
2805		- * in the nr_immediate check below.
2806		- */
2807		- if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
2808		- set_bit(PGDAT_WRITEBACK, &pgdat->flags);
2809		-
2810		- /*
2811		- * Tag a node as congested if all the dirty pages
2812		- * scanned were backed by a congested BDI and
2813		- * wait_iff_congested will stall.
2814		- */
2815		- if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2816		- set_bit(PGDAT_CONGESTED, &pgdat->flags);
2817		-
2818		- /* Allow kswapd to start writing pages during reclaim.*/
2819		- if (sc->nr.unqueued_dirty == sc->nr.file_taken)
2820		- set_bit(PGDAT_DIRTY, &pgdat->flags);
2821		-
2822		- /*
2823		- * If kswapd scans pages marked marked for immediate
2824		- * reclaim and under writeback (nr_immediate), it
2825		- * implies that pages are cycling through the LRU
2826		- * faster than they are written so also forcibly stall.
2827		- */
2828		- if (sc->nr.immediate)
2829		- congestion_wait(BLK_RW_ASYNC, HZ/10);
	2850	+ total_high_wmark += high_wmark_pages(zone);
2830	2851	}
2831	2852
2832	2853	/*
2833		- * Legacy memcg will stall in page writeback so avoid forcibly
2834		- * stalling in wait_iff_congested().
	2854	+ * Consider anon: if that's low too, this isn't a
	2855	+ * runaway file reclaim problem, but rather just
	2856	+ * extreme pressure. Reclaim as per usual then.
2835	2857	*/
2836		- if (!global_reclaim(sc) && sane_reclaim(sc) &&
2837		- sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2838		- set_memcg_congestion(pgdat, root, true);
	2858	+ anon = node_page_state(pgdat, NR_INACTIVE_ANON);
	2859	+
	2860	+ sc->file_is_tiny =
	2861	+ file + free <= total_high_wmark &&
	2862	+ !(sc->may_deactivate & DEACTIVATE_ANON) &&
	2863	+ anon >> sc->priority;
	2864	+ }
	2865	+
	2866	+ shrink_node_memcgs(pgdat, sc);
	2867	+
	2868	+ if (reclaim_state) {
	2869	+ sc->nr_reclaimed += reclaim_state->reclaimed_slab;
	2870	+ reclaim_state->reclaimed_slab = 0;
	2871	+ }
	2872	+
	2873	+ /* Record the subtree's reclaim efficiency */
	2874	+ vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
	2875	+ sc->nr_scanned - nr_scanned,
	2876	+ sc->nr_reclaimed - nr_reclaimed);
	2877	+
	2878	+ if (sc->nr_reclaimed - nr_reclaimed)
	2879	+ reclaimable = true;
	2880	+
	2881	+ if (current_is_kswapd()) {
	2882	+ /*
	2883	+ * If reclaim is isolating dirty pages under writeback,
	2884	+ * it implies that the long-lived page allocation rate
	2885	+ * is exceeding the page laundering rate. Either the
	2886	+ * global limits are not being effective at throttling
	2887	+ * processes due to the page distribution throughout
	2888	+ * zones or there is heavy usage of a slow backing
	2889	+ * device. The only option is to throttle from reclaim
	2890	+ * context which is not ideal as there is no guarantee
	2891	+ * the dirtying process is throttled in the same way
	2892	+ * balance_dirty_pages() manages.
	2893	+ *
	2894	+ * Once a node is flagged PGDAT_WRITEBACK, kswapd will
	2895	+ * count the number of pages under pages flagged for
	2896	+ * immediate reclaim and stall if any are encountered
	2897	+ * in the nr_immediate check below.
	2898	+ */
	2899	+ if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
	2900	+ set_bit(PGDAT_WRITEBACK, &pgdat->flags);
	2901	+
	2902	+ /* Allow kswapd to start writing pages during reclaim.*/
	2903	+ if (sc->nr.unqueued_dirty == sc->nr.file_taken)
	2904	+ set_bit(PGDAT_DIRTY, &pgdat->flags);
2839	2905
2840	2906	/*
2841		- * Stall direct reclaim for IO completions if underlying BDIs
2842		- * and node is congested. Allow kswapd to continue until it
2843		- * starts encountering unqueued dirty pages or cycling through
2844		- * the LRU too quickly.
	2907	+ * If kswapd scans pages marked for immediate
	2908	+ * reclaim and under writeback (nr_immediate), it
	2909	+ * implies that pages are cycling through the LRU
	2910	+ * faster than they are written so also forcibly stall.
2845	2911	*/
2846		- if (!sc->hibernation_mode && !current_is_kswapd() &&
2847		- current_may_throttle() && pgdat_memcg_congested(pgdat, root))
2848		- wait_iff_congested(BLK_RW_ASYNC, HZ/10);
	2912	+ if (sc->nr.immediate)
	2913	+ congestion_wait(BLK_RW_ASYNC, HZ/10);
	2914	+ }
2849	2915
2850		- } while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
2851		- sc->nr_scanned - nr_scanned, sc));
	2916	+ /*
	2917	+ * Tag a node/memcg as congested if all the dirty pages
	2918	+ * scanned were backed by a congested BDI and
	2919	+ * wait_iff_congested will stall.
	2920	+ *
	2921	+ * Legacy memcg will stall in page writeback so avoid forcibly
	2922	+ * stalling in wait_iff_congested().
	2923	+ */
	2924	+ if ((current_is_kswapd() \|\|
	2925	+ (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
	2926	+ sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
	2927	+ set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
	2928	+
	2929	+ /*
	2930	+ * Stall direct reclaim for IO completions if underlying BDIs
	2931	+ * and node is congested. Allow kswapd to continue until it
	2932	+ * starts encountering unqueued dirty pages or cycling through
	2933	+ * the LRU too quickly.
	2934	+ */
	2935	+ if (!current_is_kswapd() && current_may_throttle() &&
	2936	+ !sc->hibernation_mode &&
	2937	+ test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
	2938	+ wait_iff_congested(BLK_RW_ASYNC, HZ/10);
	2939	+
	2940	+ if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
	2941	+ sc))
	2942	+ goto again;
2852	2943
2853	2944	/*
2854	2945	* Kswapd gives up on balancing particular nodes after too
..	..	@@ -2858,8 +2949,6 @@
2858	2949	*/
2859	2950	if (reclaimable)
2860	2951	pgdat->kswapd_failures = 0;
2861		-
2862		- return reclaimable;
2863	2952	}
2864	2953
2865	2954	/*
..	..	@@ -2928,7 +3017,7 @@
2928	3017	* Take care memory controller reclaiming has small influence
2929	3018	* to global LRU.
2930	3019	*/
2931		- if (global_reclaim(sc)) {
	3020	+ if (!cgroup_reclaim(sc)) {
2932	3021	if (!cpuset_zone_allowed(zone,
2933	3022	GFP_KERNEL \| __GFP_HARDWALL))
2934	3023	continue;
..	..	@@ -2987,19 +3076,17 @@
2987	3076	sc->gfp_mask = orig_mask;
2988	3077	}
2989	3078
2990		-static void snapshot_refaults(struct mem_cgroup root_memcg, pg_data_t pgdat)
	3079	+static void snapshot_refaults(struct mem_cgroup target_memcg, pg_data_t pgdat)
2991	3080	{
2992		- struct mem_cgroup *memcg;
	3081	+ struct lruvec *target_lruvec;
	3082	+ unsigned long refaults;
2993	3083
2994		- memcg = mem_cgroup_iter(root_memcg, NULL, NULL);
2995		- do {
2996		- unsigned long refaults;
2997		- struct lruvec *lruvec;
2998		-
2999		- lruvec = mem_cgroup_lruvec(pgdat, memcg);
3000		- refaults = lruvec_page_state(lruvec, WORKINGSET_ACTIVATE);
3001		- lruvec->refaults = refaults;
3002		- } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
	3084	+ target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
	3085	+ refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
	3086	+ target_lruvec->refaults[0] = refaults;
	3087	+ refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
	3088	+ target_lruvec->refaults[1] = refaults;
	3089	+ trace_android_vh_snapshot_refaults(target_lruvec);
3003	3090	}
3004	3091
3005	3092	/*
..	..	@@ -3028,7 +3115,7 @@
3028	3115	retry:
3029	3116	delayacct_freepages_start();
3030	3117
3031		- if (global_reclaim(sc))
	3118	+ if (!cgroup_reclaim(sc))
3032	3119	__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
3033	3120
3034	3121	do {
..	..	@@ -3057,8 +3144,16 @@
3057	3144	if (zone->zone_pgdat == last_pgdat)
3058	3145	continue;
3059	3146	last_pgdat = zone->zone_pgdat;
	3147	+
3060	3148	snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3061		- set_memcg_congestion(last_pgdat, sc->target_mem_cgroup, false);
	3149	+
	3150	+ if (cgroup_reclaim(sc)) {
	3151	+ struct lruvec *lruvec;
	3152	+
	3153	+ lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
	3154	+ zone->zone_pgdat);
	3155	+ clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
	3156	+ }
3062	3157	}
3063	3158
3064	3159	delayacct_freepages_end();
..	..	@@ -3070,9 +3165,26 @@
3070	3165	if (sc->compaction_ready)
3071	3166	return 1;
3072	3167
	3168	+ /*
	3169	+ * We make inactive:active ratio decisions based on the node's
	3170	+ * composition of memory, but a restrictive reclaim_idx or a
	3171	+ * memory.low cgroup setting can exempt large amounts of
	3172	+ * memory from reclaim. Neither of which are very common, so
	3173	+ * instead of doing costly eligibility calculations of the
	3174	+ * entire cgroup subtree up front, we assume the estimates are
	3175	+ * good, and retry with forcible deactivation if that fails.
	3176	+ */
	3177	+ if (sc->skipped_deactivate) {
	3178	+ sc->priority = initial_priority;
	3179	+ sc->force_deactivate = 1;
	3180	+ sc->skipped_deactivate = 0;
	3181	+ goto retry;
	3182	+ }
	3183	+
3073	3184	/* Untapped cgroup reserves? Don't OOM, retry. */
3074	3185	if (sc->memcg_low_skipped) {
3075	3186	sc->priority = initial_priority;
	3187	+ sc->force_deactivate = 0;
3076	3188	sc->memcg_low_reclaim = 1;
3077	3189	sc->memcg_low_skipped = 0;
3078	3190	goto retry;
..	..	@@ -3112,8 +3224,8 @@
3112	3224
3113	3225	/* kswapd must be awake if processes are being throttled */
3114	3226	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
3115		- if (READ_ONCE(pgdat->kswapd_classzone_idx) > ZONE_NORMAL)
3116		- WRITE_ONCE(pgdat->kswapd_classzone_idx, ZONE_NORMAL);
	3227	+ if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
	3228	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);
3117	3229
3118	3230	wake_up_interruptible(&pgdat->kswapd_wait);
3119	3231	}
..	..	@@ -3246,25 +3358,26 @@
3246	3358	if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3247	3359	return 1;
3248	3360
3249		- trace_mm_vmscan_direct_reclaim_begin(order,
3250		- sc.may_writepage,
3251		- sc.gfp_mask,
3252		- sc.reclaim_idx);
	3361	+ set_task_reclaim_state(current, &sc.reclaim_state);
	3362	+ trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask);
3253	3363
3254	3364	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3255	3365
3256	3366	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
	3367	+ set_task_reclaim_state(current, NULL);
3257	3368
3258	3369	return nr_reclaimed;
3259	3370	}
3260	3371
3261	3372	#ifdef CONFIG_MEMCG
3262	3373
	3374	+/* Only used by soft limit reclaim. Do not reuse for anything else. */
3263	3375	unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3264	3376	gfp_t gfp_mask, bool noswap,
3265	3377	pg_data_t *pgdat,
3266	3378	unsigned long *nr_scanned)
3267	3379	{
	3380	+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
3268	3381	struct scan_control sc = {
3269	3382	.nr_to_reclaim = SWAP_CLUSTER_MAX,
3270	3383	.target_mem_cgroup = memcg,
..	..	@@ -3273,15 +3386,14 @@
3273	3386	.reclaim_idx = MAX_NR_ZONES - 1,
3274	3387	.may_swap = !noswap,
3275	3388	};
3276		- unsigned long lru_pages;
	3389	+
	3390	+ WARN_ON_ONCE(!current->reclaim_state);
3277	3391
3278	3392	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) \|
3279	3393	(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3280	3394
3281	3395	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
3282		- sc.may_writepage,
3283		- sc.gfp_mask,
3284		- sc.reclaim_idx);
	3396	+ sc.gfp_mask);
3285	3397
3286	3398	/*
3287	3399	* NOTE: Although we can get the priority field, using it
..	..	@@ -3290,11 +3402,12 @@
3290	3402	* will pick up pages from other mem cgroup's as well. We hack
3291	3403	* the priority and make it zero.
3292	3404	*/
3293		- shrink_node_memcg(pgdat, memcg, &sc, &lru_pages);
	3405	+ shrink_lruvec(lruvec, &sc);
3294	3406
3295	3407	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3296	3408
3297	3409	*nr_scanned = sc.nr_scanned;
	3410	+
3298	3411	return sc.nr_reclaimed;
3299	3412	}
3300	3413
..	..	@@ -3303,10 +3416,7 @@
3303	3416	gfp_t gfp_mask,
3304	3417	bool may_swap)
3305	3418	{
3306		- struct zonelist *zonelist;
3307	3419	unsigned long nr_reclaimed;
3308		- unsigned long pflags;
3309		- int nid;
3310	3420	unsigned int noreclaim_flag;
3311	3421	struct scan_control sc = {
3312	3422	.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
..	..	@@ -3319,78 +3429,101 @@
3319	3429	.may_unmap = 1,
3320	3430	.may_swap = may_swap,
3321	3431	};
3322		-
3323	3432	/*
3324		- * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
3325		- * take care of from where we get pages. So the node where we start the
3326		- * scan does not need to be the current node.
	3433	+ * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
	3434	+ * equal pressure on all the nodes. This is based on the assumption that
	3435	+ * the reclaim does not bail out early.
3327	3436	*/
3328		- nid = mem_cgroup_select_victim_node(memcg);
	3437	+ struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3329	3438
3330		- zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
3331		-
3332		- trace_mm_vmscan_memcg_reclaim_begin(0,
3333		- sc.may_writepage,
3334		- sc.gfp_mask,
3335		- sc.reclaim_idx);
3336		-
3337		- psi_memstall_enter(&pflags);
	3439	+ set_task_reclaim_state(current, &sc.reclaim_state);
	3440	+ trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
3338	3441	noreclaim_flag = memalloc_noreclaim_save();
3339	3442
3340	3443	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3341	3444
3342	3445	memalloc_noreclaim_restore(noreclaim_flag);
3343		- psi_memstall_leave(&pflags);
3344		-
3345	3446	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
	3447	+ set_task_reclaim_state(current, NULL);
3346	3448
3347	3449	return nr_reclaimed;
3348	3450	}
	3451	+EXPORT_SYMBOL_GPL(try_to_free_mem_cgroup_pages);
3349	3452	#endif
3350	3453
3351	3454	static void age_active_anon(struct pglist_data *pgdat,
3352	3455	struct scan_control *sc)
3353	3456	{
3354	3457	struct mem_cgroup *memcg;
	3458	+ struct lruvec *lruvec;
3355	3459
3356	3460	if (!total_swap_pages)
3357	3461	return;
3358	3462
	3463	+ lruvec = mem_cgroup_lruvec(NULL, pgdat);
	3464	+ if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
	3465	+ return;
	3466	+
3359	3467	memcg = mem_cgroup_iter(NULL, NULL, NULL);
3360	3468	do {
3361		- struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
3362		-
3363		- if (inactive_list_is_low(lruvec, false, sc, true))
3364		- shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3365		- sc, LRU_ACTIVE_ANON);
3366		-
	3469	+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
	3470	+ shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
	3471	+ sc, LRU_ACTIVE_ANON);
3367	3472	memcg = mem_cgroup_iter(NULL, memcg, NULL);
3368	3473	} while (memcg);
3369	3474	}
3370	3475
	3476	+static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
	3477	+{
	3478	+ int i;
	3479	+ struct zone *zone;
	3480	+
	3481	+ /*
	3482	+ * Check for watermark boosts top-down as the higher zones
	3483	+ * are more likely to be boosted. Both watermarks and boosts
	3484	+ * should not be checked at the same time as reclaim would
	3485	+ * start prematurely when there is no boosting and a lower
	3486	+ * zone is balanced.
	3487	+ */
	3488	+ for (i = highest_zoneidx; i >= 0; i--) {
	3489	+ zone = pgdat->node_zones + i;
	3490	+ if (!managed_zone(zone))
	3491	+ continue;
	3492	+
	3493	+ if (zone->watermark_boost)
	3494	+ return true;
	3495	+ }
	3496	+
	3497	+ return false;
	3498	+}
	3499	+
3371	3500	/*
3372	3501	* Returns true if there is an eligible zone balanced for the request order
3373		- * and classzone_idx
	3502	+ * and highest_zoneidx
3374	3503	*/
3375		-static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx)
	3504	+static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
3376	3505	{
3377	3506	int i;
3378	3507	unsigned long mark = -1;
3379	3508	struct zone *zone;
3380	3509
3381		- for (i = 0; i <= classzone_idx; i++) {
	3510	+ /*
	3511	+ * Check watermarks bottom-up as lower zones are more likely to
	3512	+ * meet watermarks.
	3513	+ */
	3514	+ for (i = 0; i <= highest_zoneidx; i++) {
3382	3515	zone = pgdat->node_zones + i;
3383	3516
3384	3517	if (!managed_zone(zone))
3385	3518	continue;
3386	3519
3387	3520	mark = high_wmark_pages(zone);
3388		- if (zone_watermark_ok_safe(zone, order, mark, classzone_idx))
	3521	+ if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx))
3389	3522	return true;
3390	3523	}
3391	3524
3392	3525	/*
3393		- * If a node has no populated zone within classzone_idx, it does not
	3526	+ * If a node has no populated zone within highest_zoneidx, it does not
3394	3527	* need balancing by definition. This can happen if a zone-restricted
3395	3528	* allocation tries to wake a remote kswapd.
3396	3529	*/
..	..	@@ -3403,7 +3536,9 @@
3403	3536	/* Clear pgdat state for congested, dirty or under writeback. */
3404	3537	static void clear_pgdat_congested(pg_data_t *pgdat)
3405	3538	{
3406		- clear_bit(PGDAT_CONGESTED, &pgdat->flags);
	3539	+ struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
	3540	+
	3541	+ clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3407	3542	clear_bit(PGDAT_DIRTY, &pgdat->flags);
3408	3543	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
3409	3544	}
..	..	@@ -3414,7 +3549,8 @@
3414	3549	*
3415	3550	* Returns true if kswapd is ready to sleep
3416	3551	*/
3417		-static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, int classzone_idx)
	3552	+static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
	3553	+ int highest_zoneidx)
3418	3554	{
3419	3555	/*
3420	3556	* The throttled processes are normally woken up in balance_pgdat() as
..	..	@@ -3436,7 +3572,7 @@
3436	3572	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3437	3573	return true;
3438	3574
3439		- if (pgdat_balanced(pgdat, order, classzone_idx)) {
	3575	+ if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
3440	3576	clear_pgdat_congested(pgdat);
3441	3577	return true;
3442	3578	}
..	..	@@ -3496,37 +3632,57 @@
3496	3632	*
3497	3633	* kswapd scans the zones in the highmem->normal->dma direction. It skips
3498	3634	* zones which have free_pages > high_wmark_pages(zone), but once a zone is
3499		- * found to have free_pages <= high_wmark_pages(zone), any page is that zone
	3635	+ * found to have free_pages <= high_wmark_pages(zone), any page in that zone
3500	3636	* or lower is eligible for reclaim until at least one usable zone is
3501	3637	* balanced.
3502	3638	*/
3503		-static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
	3639	+static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
3504	3640	{
3505	3641	int i;
3506	3642	unsigned long nr_soft_reclaimed;
3507	3643	unsigned long nr_soft_scanned;
3508	3644	unsigned long pflags;
	3645	+ unsigned long nr_boost_reclaim;
	3646	+ unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
	3647	+ bool boosted;
3509	3648	struct zone *zone;
3510	3649	struct scan_control sc = {
3511	3650	.gfp_mask = GFP_KERNEL,
3512	3651	.order = order,
3513		- .priority = DEF_PRIORITY,
3514		- .may_writepage = !laptop_mode,
3515	3652	.may_unmap = 1,
3516		- .may_swap = 1,
3517	3653	};
3518	3654
	3655	+ set_task_reclaim_state(current, &sc.reclaim_state);
3519	3656	psi_memstall_enter(&pflags);
3520	3657	__fs_reclaim_acquire();
3521	3658
3522	3659	count_vm_event(PAGEOUTRUN);
3523	3660
	3661	+ /*
	3662	+ * Account for the reclaim boost. Note that the zone boost is left in
	3663	+ * place so that parallel allocations that are near the watermark will
	3664	+ * stall or direct reclaim until kswapd is finished.
	3665	+ */
	3666	+ nr_boost_reclaim = 0;
	3667	+ for (i = 0; i <= highest_zoneidx; i++) {
	3668	+ zone = pgdat->node_zones + i;
	3669	+ if (!managed_zone(zone))
	3670	+ continue;
	3671	+
	3672	+ nr_boost_reclaim += zone->watermark_boost;
	3673	+ zone_boosts[i] = zone->watermark_boost;
	3674	+ }
	3675	+ boosted = nr_boost_reclaim;
	3676	+
	3677	+restart:
	3678	+ sc.priority = DEF_PRIORITY;
3524	3679	do {
3525	3680	unsigned long nr_reclaimed = sc.nr_reclaimed;
3526	3681	bool raise_priority = true;
	3682	+ bool balanced;
3527	3683	bool ret;
3528	3684
3529		- sc.reclaim_idx = classzone_idx;
	3685	+ sc.reclaim_idx = highest_zoneidx;
3530	3686
3531	3687	/*
3532	3688	* If the number of buffer_heads exceeds the maximum allowed
..	..	@@ -3550,12 +3706,38 @@
3550	3706	}
3551	3707
3552	3708	/*
3553		- * Only reclaim if there are no eligible zones. Note that
3554		- * sc.reclaim_idx is not used as buffer_heads_over_limit may
3555		- * have adjusted it.
	3709	+ * If the pgdat is imbalanced then ignore boosting and preserve
	3710	+ * the watermarks for a later time and restart. Note that the
	3711	+ * zone watermarks will be still reset at the end of balancing
	3712	+ * on the grounds that the normal reclaim should be enough to
	3713	+ * re-evaluate if boosting is required when kswapd next wakes.
3556	3714	*/
3557		- if (pgdat_balanced(pgdat, sc.order, classzone_idx))
	3715	+ balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);
	3716	+ if (!balanced && nr_boost_reclaim) {
	3717	+ nr_boost_reclaim = 0;
	3718	+ goto restart;
	3719	+ }
	3720	+
	3721	+ /*
	3722	+ * If boosting is not active then only reclaim if there are no
	3723	+ * eligible zones. Note that sc.reclaim_idx is not used as
	3724	+ * buffer_heads_over_limit may have adjusted it.
	3725	+ */
	3726	+ if (!nr_boost_reclaim && balanced)
3558	3727	goto out;
	3728	+
	3729	+ /* Limit the priority of boosting to avoid reclaim writeback */
	3730	+ if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
	3731	+ raise_priority = false;
	3732	+
	3733	+ /*
	3734	+ * Do not writeback or swap pages for boosted reclaim. The
	3735	+ * intent is to relieve pressure not issue sub-optimal IO
	3736	+ * from reclaim context. If no pages are reclaimed, the
	3737	+ * reclaim will be aborted.
	3738	+ */
	3739	+ sc.may_writepage = !laptop_mode && !nr_boost_reclaim;
	3740	+ sc.may_swap = !nr_boost_reclaim;
3559	3741
3560	3742	/*
3561	3743	* Do some background aging of the anon list, to give
..	..	@@ -3608,6 +3790,16 @@
3608	3790	* progress in reclaiming pages
3609	3791	*/
3610	3792	nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
	3793	+ nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
	3794	+
	3795	+ /*
	3796	+ * If reclaim made no progress for a boost, stop reclaim as
	3797	+ * IO cannot be queued and it could be an infinite loop in
	3798	+ * extreme circumstances.
	3799	+ */
	3800	+ if (nr_boost_reclaim && !nr_reclaimed)
	3801	+ break;
	3802	+
3611	3803	if (raise_priority \|\| !nr_reclaimed)
3612	3804	sc.priority--;
3613	3805	} while (sc.priority >= 1);
..	..	@@ -3616,9 +3808,33 @@
3616	3808	pgdat->kswapd_failures++;
3617	3809
3618	3810	out:
	3811	+ /* If reclaim was boosted, account for the reclaim done in this pass */
	3812	+ if (boosted) {
	3813	+ unsigned long flags;
	3814	+
	3815	+ for (i = 0; i <= highest_zoneidx; i++) {
	3816	+ if (!zone_boosts[i])
	3817	+ continue;
	3818	+
	3819	+ /* Increments are under the zone lock */
	3820	+ zone = pgdat->node_zones + i;
	3821	+ spin_lock_irqsave(&zone->lock, flags);
	3822	+ zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
	3823	+ spin_unlock_irqrestore(&zone->lock, flags);
	3824	+ }
	3825	+
	3826	+ /*
	3827	+ * As there is now likely space, wakeup kcompact to defragment
	3828	+ * pageblocks.
	3829	+ */
	3830	+ wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);
	3831	+ }
	3832	+
3619	3833	snapshot_refaults(NULL, pgdat);
3620	3834	__fs_reclaim_release();
3621	3835	psi_memstall_leave(&pflags);
	3836	+ set_task_reclaim_state(current, NULL);
	3837	+
3622	3838	/*
3623	3839	* Return the order kswapd stopped reclaiming at as
3624	3840	* prepare_kswapd_sleep() takes it into account. If another caller
..	..	@@ -3629,22 +3845,22 @@
3629	3845	}
3630	3846
3631	3847	/*
3632		- * The pgdat->kswapd_classzone_idx is used to pass the highest zone index to be
3633		- * reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is not
3634		- * a valid index then either kswapd runs for first time or kswapd couldn't sleep
3635		- * after previous reclaim attempt (node is still unbalanced). In that case
3636		- * return the zone index of the previous kswapd reclaim cycle.
	3848	+ * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
	3849	+ * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
	3850	+ * not a valid index then either kswapd runs for first time or kswapd couldn't
	3851	+ * sleep after previous reclaim attempt (node is still unbalanced). In that
	3852	+ * case return the zone index of the previous kswapd reclaim cycle.
3637	3853	*/
3638		-static enum zone_type kswapd_classzone_idx(pg_data_t *pgdat,
3639		- enum zone_type prev_classzone_idx)
	3854	+static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,
	3855	+ enum zone_type prev_highest_zoneidx)
3640	3856	{
3641		- enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_classzone_idx);
	3857	+ enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
3642	3858
3643		- return curr_idx == MAX_NR_ZONES ? prev_classzone_idx : curr_idx;
	3859	+ return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx;
3644	3860	}
3645	3861
3646	3862	static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
3647		- unsigned int classzone_idx)
	3863	+ unsigned int highest_zoneidx)
3648	3864	{
3649	3865	long remaining = 0;
3650	3866	DEFINE_WAIT(wait);
..	..	@@ -3661,7 +3877,7 @@
3661	3877	* eligible zone balanced that it's also unlikely that compaction will
3662	3878	* succeed.
3663	3879	*/
3664		- if (prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
	3880	+ if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
3665	3881	/*
3666	3882	* Compaction records what page blocks it recently failed to
3667	3883	* isolate pages from and skips them in the future scanning.
..	..	@@ -3674,18 +3890,19 @@
3674	3890	* We have freed the memory, now we should compact it to make
3675	3891	* allocation of the requested order possible.
3676	3892	*/
3677		- wakeup_kcompactd(pgdat, alloc_order, classzone_idx);
	3893	+ wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);
3678	3894
3679	3895	remaining = schedule_timeout(HZ/10);
3680	3896
3681	3897	/*
3682		- * If woken prematurely then reset kswapd_classzone_idx and
	3898	+ * If woken prematurely then reset kswapd_highest_zoneidx and
3683	3899	* order. The values will either be from a wakeup request or
3684	3900	* the previous request that slept prematurely.
3685	3901	*/
3686	3902	if (remaining) {
3687		- WRITE_ONCE(pgdat->kswapd_classzone_idx,
3688		- kswapd_classzone_idx(pgdat, classzone_idx));
	3903	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx,
	3904	+ kswapd_highest_zoneidx(pgdat,
	3905	+ highest_zoneidx));
3689	3906
3690	3907	if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)
3691	3908	WRITE_ONCE(pgdat->kswapd_order, reclaim_order);
..	..	@@ -3700,7 +3917,7 @@
3700	3917	* go fully to sleep until explicitly woken up.
3701	3918	*/
3702	3919	if (!remaining &&
3703		- prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
	3920	+ prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
3704	3921	trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
3705	3922
3706	3923	/*
..	..	@@ -3742,18 +3959,13 @@
3742	3959	static int kswapd(void *p)
3743	3960	{
3744	3961	unsigned int alloc_order, reclaim_order;
3745		- unsigned int classzone_idx = MAX_NR_ZONES - 1;
	3962	+ unsigned int highest_zoneidx = MAX_NR_ZONES - 1;
3746	3963	pg_data_t pgdat = (pg_data_t)p;
3747	3964	struct task_struct *tsk = current;
3748		-
3749		- struct reclaim_state reclaim_state = {
3750		- .reclaimed_slab = 0,
3751		- };
3752	3965	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3753	3966
3754	3967	if (!cpumask_empty(cpumask))
3755	3968	set_cpus_allowed_ptr(tsk, cpumask);
3756		- current->reclaim_state = &reclaim_state;
3757	3969
3758	3970	/*
3759	3971	* Tell the memory management that we're a "memory allocator",
..	..	@@ -3771,22 +3983,24 @@
3771	3983	set_freezable();
3772	3984
3773	3985	WRITE_ONCE(pgdat->kswapd_order, 0);
3774		- WRITE_ONCE(pgdat->kswapd_classzone_idx, MAX_NR_ZONES);
	3986	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
3775	3987	for ( ; ; ) {
3776	3988	bool ret;
3777	3989
3778	3990	alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
3779		- classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx);
	3991	+ highest_zoneidx = kswapd_highest_zoneidx(pgdat,
	3992	+ highest_zoneidx);
3780	3993
3781	3994	kswapd_try_sleep:
3782	3995	kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
3783		- classzone_idx);
	3996	+ highest_zoneidx);
3784	3997
3785		- /* Read the new order and classzone_idx */
	3998	+ /* Read the new order and highest_zoneidx */
3786	3999	alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
3787		- classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx);
	4000	+ highest_zoneidx = kswapd_highest_zoneidx(pgdat,
	4001	+ highest_zoneidx);
3788	4002	WRITE_ONCE(pgdat->kswapd_order, 0);
3789		- WRITE_ONCE(pgdat->kswapd_classzone_idx, MAX_NR_ZONES);
	4003	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
3790	4004
3791	4005	ret = try_to_freeze();
3792	4006	if (kthread_should_stop())
..	..	@@ -3807,17 +4021,57 @@
3807	4021	* but kcompactd is woken to compact for the original
3808	4022	* request (alloc_order).
3809	4023	*/
3810		- trace_mm_vmscan_kswapd_wake(pgdat->node_id, classzone_idx,
	4024	+ trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,
3811	4025	alloc_order);
3812		- reclaim_order = balance_pgdat(pgdat, alloc_order, classzone_idx);
	4026	+ reclaim_order = balance_pgdat(pgdat, alloc_order,
	4027	+ highest_zoneidx);
3813	4028	if (reclaim_order < alloc_order)
3814	4029	goto kswapd_try_sleep;
3815	4030	}
3816	4031
3817	4032	tsk->flags &= ~(PF_MEMALLOC \| PF_SWAPWRITE \| PF_KSWAPD);
3818		- current->reclaim_state = NULL;
3819	4033
3820	4034	return 0;
	4035	+}
	4036	+
	4037	+static int kswapd_per_node_run(int nid)
	4038	+{
	4039	+ pg_data_t *pgdat = NODE_DATA(nid);
	4040	+ int hid;
	4041	+ int ret = 0;
	4042	+
	4043	+ for (hid = 0; hid < kswapd_threads; ++hid) {
	4044	+ pgdat->mkswapd[hid] = kthread_run(kswapd, pgdat, "kswapd%d:%d",
	4045	+ nid, hid);
	4046	+ if (IS_ERR(pgdat->mkswapd[hid])) {
	4047	+ /* failure at boot is fatal */
	4048	+ WARN_ON(system_state < SYSTEM_RUNNING);
	4049	+ pr_err("Failed to start kswapd%d on node %d\n",
	4050	+ hid, nid);
	4051	+ ret = PTR_ERR(pgdat->mkswapd[hid]);
	4052	+ pgdat->mkswapd[hid] = NULL;
	4053	+ continue;
	4054	+ }
	4055	+ if (!pgdat->kswapd)
	4056	+ pgdat->kswapd = pgdat->mkswapd[hid];
	4057	+ }
	4058	+
	4059	+ return ret;
	4060	+}
	4061	+
	4062	+static void kswapd_per_node_stop(int nid)
	4063	+{
	4064	+ int hid = 0;
	4065	+ struct task_struct *kswapd;
	4066	+
	4067	+ for (hid = 0; hid < kswapd_threads; hid++) {
	4068	+ kswapd = NODE_DATA(nid)->mkswapd[hid];
	4069	+ if (kswapd) {
	4070	+ kthread_stop(kswapd);
	4071	+ NODE_DATA(nid)->mkswapd[hid] = NULL;
	4072	+ }
	4073	+ }
	4074	+ NODE_DATA(nid)->kswapd = NULL;
3821	4075	}
3822	4076
3823	4077	/*
..	..	@@ -3828,7 +4082,7 @@
3828	4082	* needed.
3829	4083	*/
3830	4084	void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
3831		- enum zone_type classzone_idx)
	4085	+ enum zone_type highest_zoneidx)
3832	4086	{
3833	4087	pg_data_t *pgdat;
3834	4088	enum zone_type curr_idx;
..	..	@@ -3840,10 +4094,10 @@
3840	4094	return;
3841	4095
3842	4096	pgdat = zone->zone_pgdat;
3843		- curr_idx = READ_ONCE(pgdat->kswapd_classzone_idx);
	4097	+ curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
3844	4098
3845		- if (curr_idx == MAX_NR_ZONES \|\| curr_idx < classzone_idx)
3846		- WRITE_ONCE(pgdat->kswapd_classzone_idx, classzone_idx);
	4099	+ if (curr_idx == MAX_NR_ZONES \|\| curr_idx < highest_zoneidx)
	4100	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);
3847	4101
3848	4102	if (READ_ONCE(pgdat->kswapd_order) < order)
3849	4103	WRITE_ONCE(pgdat->kswapd_order, order);
..	..	@@ -3853,7 +4107,8 @@
3853	4107
3854	4108	/* Hopeless node, leave it to direct reclaim if possible */
3855	4109	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES \|\|
3856		- pgdat_balanced(pgdat, order, classzone_idx)) {
	4110	+ (pgdat_balanced(pgdat, order, highest_zoneidx) &&
	4111	+ !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
3857	4112	/*
3858	4113	* There may be plenty of free memory available, but it's too
3859	4114	* fragmented for high-order allocations. Wake up kcompactd
..	..	@@ -3862,11 +4117,11 @@
3862	4117	* ratelimit its work.
3863	4118	*/
3864	4119	if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
3865		- wakeup_kcompactd(pgdat, order, classzone_idx);
	4120	+ wakeup_kcompactd(pgdat, order, highest_zoneidx);
3866	4121	return;
3867	4122	}
3868	4123
3869		- trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, classzone_idx, order,
	4124	+ trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,
3870	4125	gfp_flags);
3871	4126	wake_up_interruptible(&pgdat->kswapd_wait);
3872	4127	}
..	..	@@ -3882,7 +4137,6 @@
3882	4137	*/
3883	4138	unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3884	4139	{
3885		- struct reclaim_state reclaim_state;
3886	4140	struct scan_control sc = {
3887	4141	.nr_to_reclaim = nr_to_reclaim,
3888	4142	.gfp_mask = GFP_HIGHUSER_MOVABLE,
..	..	@@ -3894,45 +4148,22 @@
3894	4148	.hibernation_mode = 1,
3895	4149	};
3896	4150	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3897		- struct task_struct *p = current;
3898	4151	unsigned long nr_reclaimed;
3899	4152	unsigned int noreclaim_flag;
3900	4153
3901	4154	fs_reclaim_acquire(sc.gfp_mask);
3902	4155	noreclaim_flag = memalloc_noreclaim_save();
3903		- reclaim_state.reclaimed_slab = 0;
3904		- p->reclaim_state = &reclaim_state;
	4156	+ set_task_reclaim_state(current, &sc.reclaim_state);
3905	4157
3906	4158	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3907	4159
3908		- p->reclaim_state = NULL;
	4160	+ set_task_reclaim_state(current, NULL);
3909	4161	memalloc_noreclaim_restore(noreclaim_flag);
3910	4162	fs_reclaim_release(sc.gfp_mask);
3911	4163
3912	4164	return nr_reclaimed;
3913	4165	}
3914	4166	#endif /* CONFIG_HIBERNATION */
3915		-
3916		-/* It's optimal to keep kswapds on the same CPUs as their memory, but
3917		- not required for correctness. So if the last cpu in a node goes
3918		- away, we get changed to run anywhere: as the first one comes back,
3919		- restore their cpu bindings. */
3920		-static int kswapd_cpu_online(unsigned int cpu)
3921		-{
3922		- int nid;
3923		-
3924		- for_each_node_state(nid, N_MEMORY) {
3925		- pg_data_t *pgdat = NODE_DATA(nid);
3926		- const struct cpumask *mask;
3927		-
3928		- mask = cpumask_of_node(pgdat->node_id);
3929		-
3930		- if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3931		- /* One of our CPUs online: restore mask */
3932		- set_cpus_allowed_ptr(pgdat->kswapd, mask);
3933		- }
3934		- return 0;
3935		-}
3936	4167
3937	4168	/*
3938	4169	* This kswapd start function will be called by init and node-hot-add.
..	..	@@ -3945,6 +4176,9 @@
3945	4176
3946	4177	if (pgdat->kswapd)
3947	4178	return 0;
	4179	+
	4180	+ if (kswapd_threads > 1)
	4181	+ return kswapd_per_node_run(nid);
3948	4182
3949	4183	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
3950	4184	if (IS_ERR(pgdat->kswapd)) {
..	..	@@ -3965,6 +4199,11 @@
3965	4199	{
3966	4200	struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
3967	4201
	4202	+ if (kswapd_threads > 1) {
	4203	+ kswapd_per_node_stop(nid);
	4204	+ return;
	4205	+ }
	4206	+
3968	4207	if (kswapd) {
3969	4208	kthread_stop(kswapd);
3970	4209	NODE_DATA(nid)->kswapd = NULL;
..	..	@@ -3973,15 +4212,11 @@
3973	4212
3974	4213	static int __init kswapd_init(void)
3975	4214	{
3976		- int nid, ret;
	4215	+ int nid;
3977	4216
3978	4217	swap_setup();
3979	4218	for_each_node_state(nid, N_MEMORY)
3980	4219	kswapd_run(nid);
3981		- ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
3982		- "mm/vmscan:online", kswapd_cpu_online,
3983		- NULL);
3984		- WARN_ON(ret < 0);
3985	4220	return 0;
3986	4221	}
3987	4222
..	..	@@ -3996,10 +4231,13 @@
3996	4231	*/
3997	4232	int node_reclaim_mode __read_mostly;
3998	4233
3999		-#define RECLAIM_OFF 0
4000		-#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */
4001		-#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
4002		-#define RECLAIM_UNMAP (1<<2) /* Unmap pages during reclaim */
	4234	+/*
	4235	+ * These bit locations are exposed in the vm.zone_reclaim_mode sysctl
	4236	+ * ABI. New bits are OK, but existing bits can never change.
	4237	+ */
	4238	+#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */
	4239	+#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
	4240	+#define RECLAIM_UNMAP (1<<2) /* Unmap pages during reclaim */
4003	4241
4004	4242	/*
4005	4243	* Priority for NODE_RECLAIM. This determines the fraction of pages
..	..	@@ -4070,7 +4308,6 @@
4070	4308	/* Minimum pages needed in order to stay on node */
4071	4309	const unsigned long nr_pages = 1 << order;
4072	4310	struct task_struct *p = current;
4073		- struct reclaim_state reclaim_state;
4074	4311	unsigned int noreclaim_flag;
4075	4312	struct scan_control sc = {
4076	4313	.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
..	..	@@ -4083,6 +4320,9 @@
4083	4320	.reclaim_idx = gfp_zone(gfp_mask),
4084	4321	};
4085	4322
	4323	+ trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,
	4324	+ sc.gfp_mask);
	4325	+
4086	4326	cond_resched();
4087	4327	fs_reclaim_acquire(sc.gfp_mask);
4088	4328	/*
..	..	@@ -4092,8 +4332,7 @@
4092	4332	*/
4093	4333	noreclaim_flag = memalloc_noreclaim_save();
4094	4334	p->flags \|= PF_SWAPWRITE;
4095		- reclaim_state.reclaimed_slab = 0;
4096		- p->reclaim_state = &reclaim_state;
	4335	+ set_task_reclaim_state(p, &sc.reclaim_state);
4097	4336
4098	4337	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
4099	4338	/*
..	..	@@ -4105,10 +4344,13 @@
4105	4344	} while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
4106	4345	}
4107	4346
4108		- p->reclaim_state = NULL;
	4347	+ set_task_reclaim_state(p, NULL);
4109	4348	current->flags &= ~PF_SWAPWRITE;
4110	4349	memalloc_noreclaim_restore(noreclaim_flag);
4111	4350	fs_reclaim_release(sc.gfp_mask);
	4351	+
	4352	+ trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed);
	4353	+
4112	4354	return sc.nr_reclaimed >= nr_pages;
4113	4355	}
4114	4356
..	..	@@ -4127,7 +4369,8 @@
4127	4369	* unmapped file backed pages.
4128	4370	*/
4129	4371	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
4130		- node_page_state(pgdat, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
	4372	+ node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
	4373	+ pgdat->min_slab_pages)
4131	4374	return NODE_RECLAIM_FULL;
4132	4375
4133	4376	/*
..	..	@@ -4158,29 +4401,6 @@
4158	4401	}
4159	4402	#endif
4160	4403
4161		-/*
4162		- * page_evictable - test whether a page is evictable
4163		- * @page: the page to test
4164		- *
4165		- * Test whether page is evictable--i.e., should be placed on active/inactive
4166		- * lists vs unevictable list.
4167		- *
4168		- * Reasons page might not be evictable:
4169		- * (1) page's mapping marked unevictable
4170		- * (2) page is part of an mlocked VMA
4171		- *
4172		- */
4173		-int page_evictable(struct page *page)
4174		-{
4175		- int ret;
4176		-
4177		- /* Prevent address_space of inode and swap cache from being freed */
4178		- rcu_read_lock();
4179		- ret = !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
4180		- rcu_read_unlock();
4181		- return ret;
4182		-}
4183		-
4184	4404	/**
4185	4405	* check_move_unevictable_pages - check pages for evictability and move to
4186	4406	* appropriate zone lru list
..	..	@@ -4201,8 +4421,14 @@
4201	4421	for (i = 0; i < pvec->nr; i++) {
4202	4422	struct page *page = pvec->pages[i];
4203	4423	struct pglist_data *pagepgdat = page_pgdat(page);
	4424	+ int nr_pages;
4204	4425
4205		- pgscanned++;
	4426	+ if (PageTransTail(page))
	4427	+ continue;
	4428	+
	4429	+ nr_pages = thp_nr_pages(page);
	4430	+ pgscanned += nr_pages;
	4431	+
4206	4432	if (pagepgdat != pgdat) {
4207	4433	if (pgdat)
4208	4434	spin_unlock_irq(&pgdat->lru_lock);
..	..	@@ -4221,7 +4447,7 @@
4221	4447	ClearPageUnevictable(page);
4222	4448	del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
4223	4449	add_page_to_lru_list(page, lruvec, lru);
4224		- pgrescued++;
	4450	+ pgrescued += nr_pages;
4225	4451	}
4226	4452	}
4227	4453