~hc/RK356X_SDK_RELEASE.git

..	..	@@ -63,6 +63,12 @@
63	63	#define CREATE_TRACE_POINTS
64	64	#include <trace/events/vmscan.h>
65	65
	66	+#undef CREATE_TRACE_POINTS
	67	+#include <trace/hooks/vmscan.h>
	68	+
	69	+EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin);
	70	+EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end);
	71	+
66	72	struct scan_control {
67	73	/* How many pages shrink_list() should reclaim */
68	74	unsigned long nr_to_reclaim;
..	..	@@ -79,6 +85,19 @@
79	85	*/
80	86	struct mem_cgroup *target_mem_cgroup;
81	87
	88	+ /*
	89	+ * Scan pressure balancing between anon and file LRUs
	90	+ */
	91	+ unsigned long anon_cost;
	92	+ unsigned long file_cost;
	93	+
	94	+ /* Can active pages be deactivated as part of reclaim? */
	95	+#define DEACTIVATE_ANON 1
	96	+#define DEACTIVATE_FILE 2
	97	+ unsigned int may_deactivate:2;
	98	+ unsigned int force_deactivate:1;
	99	+ unsigned int skipped_deactivate:1;
	100	+
82	101	/* Writepage batching in laptop mode; RECLAIM_WRITE */
83	102	unsigned int may_writepage:1;
84	103
..	..	@@ -89,9 +108,12 @@
89	108	unsigned int may_swap:1;
90	109
91	110	/*
92		- * Cgroups are not reclaimed below their configured memory.low,
93		- * unless we threaten to OOM. If any cgroups are skipped due to
94		- * memory.low and nothing was reclaimed, go back for memory.low.
	111	+ * Cgroup memory below memory.low is protected as long as we
	112	+ * don't threaten to OOM. If any cgroup is reclaimed at
	113	+ * reduced force or passed over entirely due to its memory.low
	114	+ * setting (memcg_low_skipped), and nothing is reclaimed as a
	115	+ * result, then go back for one more cycle that reclaims the protected
	116	+ * memory (memcg_low_reclaim) to avert OOM.
95	117	*/
96	118	unsigned int memcg_low_reclaim:1;
97	119	unsigned int memcg_low_skipped:1;
..	..	@@ -100,6 +122,12 @@
100	122
101	123	/* One of the zones is ready for compaction */
102	124	unsigned int compaction_ready:1;
	125	+
	126	+ /* There is easily reclaimable cold cache in the current node */
	127	+ unsigned int cache_trim_mode:1;
	128	+
	129	+ /* The file pages on the current node are dangerously low */
	130	+ unsigned int file_is_tiny:1;
103	131
104	132	/* Allocation order */
105	133	s8 order;
..	..	@@ -128,21 +156,10 @@
128	156	unsigned int file_taken;
129	157	unsigned int taken;
130	158	} nr;
131		-};
132	159
133		-#ifdef ARCH_HAS_PREFETCH
134		-#define prefetch_prev_lru_page(_page, _base, _field) \
135		- do { \
136		- if ((_page)->lru.prev != _base) { \
137		- struct page *prev; \
138		- \
139		- prev = lru_to_page(&(_page->lru)); \
140		- prefetch(&prev->_field); \
141		- } \
142		- } while (0)
143		-#else
144		-#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
145		-#endif
	160	+ /* for recording the reclaimed slab by now */
	161	+ struct reclaim_state reclaim_state;
	162	+};
146	163
147	164	#ifdef ARCH_HAS_PREFETCHW
148	165	#define prefetchw_prev_lru_page(_page, _base, _field) \
..	..	@@ -159,20 +176,43 @@
159	176	#endif
160	177
161	178	/*
162		- * From 0 .. 100. Higher means more swappy.
	179	+ * From 0 .. 200. Higher means more swappy.
163	180	*/
164	181	int vm_swappiness = 60;
165		-/*
166		- * The total number of pages which are beyond the high watermark within all
167		- * zones.
168		- */
169		-unsigned long vm_total_pages;
	182	+
	183	+#define DEF_KSWAPD_THREADS_PER_NODE 1
	184	+static int kswapd_threads = DEF_KSWAPD_THREADS_PER_NODE;
	185	+static int __init kswapd_per_node_setup(char *str)
	186	+{
	187	+ int tmp;
	188	+
	189	+ if (kstrtoint(str, 0, &tmp) < 0)
	190	+ return 0;
	191	+
	192	+ if (tmp > MAX_KSWAPD_THREADS \|\| tmp <= 0)
	193	+ return 0;
	194	+
	195	+ kswapd_threads = tmp;
	196	+ return 1;
	197	+}
	198	+__setup("kswapd_per_node=", kswapd_per_node_setup);
	199	+
	200	+static void set_task_reclaim_state(struct task_struct *task,
	201	+ struct reclaim_state *rs)
	202	+{
	203	+ /* Check for an overwrite */
	204	+ WARN_ON_ONCE(rs && task->reclaim_state);
	205	+
	206	+ /* Check for the nulling of an already-nulled member */
	207	+ WARN_ON_ONCE(!rs && !task->reclaim_state);
	208	+
	209	+ task->reclaim_state = rs;
	210	+}
170	211
171	212	static LIST_HEAD(shrinker_list);
172	213	static DECLARE_RWSEM(shrinker_rwsem);
173	214
174		-#ifdef CONFIG_MEMCG_KMEM
175		-
	215	+#ifdef CONFIG_MEMCG
176	216	/*
177	217	* We allow subsystems to populate their shrinker-related
178	218	* LRU lists before register_shrinker_prepared() is called
..	..	@@ -224,25 +264,14 @@
224	264	idr_remove(&shrinker_idr, id);
225	265	up_write(&shrinker_rwsem);
226	266	}
227		-#else /* CONFIG_MEMCG_KMEM */
228		-static int prealloc_memcg_shrinker(struct shrinker *shrinker)
229		-{
230		- return 0;
231		-}
232	267
233		-static void unregister_memcg_shrinker(struct shrinker *shrinker)
	268	+static bool cgroup_reclaim(struct scan_control *sc)
234	269	{
235		-}
236		-#endif /* CONFIG_MEMCG_KMEM */
237		-
238		-#ifdef CONFIG_MEMCG
239		-static bool global_reclaim(struct scan_control *sc)
240		-{
241		- return !sc->target_mem_cgroup;
	270	+ return sc->target_mem_cgroup;
242	271	}
243	272
244	273	/**
245		- * sane_reclaim - is the usual dirty throttling mechanism operational?
	274	+ * writeback_throttling_sane - is the usual dirty throttling mechanism available?
246	275	* @sc: scan_control in question
247	276	*
248	277	* The normal page dirty throttling mechanism in balance_dirty_pages() is
..	..	@@ -254,11 +283,9 @@
254	283	* This function tests whether the vmscan currently in progress can assume
255	284	* that the normal dirty throttling mechanism is operational.
256	285	*/
257		-static bool sane_reclaim(struct scan_control *sc)
	286	+static bool writeback_throttling_sane(struct scan_control *sc)
258	287	{
259		- struct mem_cgroup *memcg = sc->target_mem_cgroup;
260		-
261		- if (!memcg)
	288	+ if (!cgroup_reclaim(sc))
262	289	return true;
263	290	#ifdef CONFIG_CGROUP_WRITEBACK
264	291	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
..	..	@@ -266,50 +293,24 @@
266	293	#endif
267	294	return false;
268	295	}
269		-
270		-static void set_memcg_congestion(pg_data_t *pgdat,
271		- struct mem_cgroup *memcg,
272		- bool congested)
273		-{
274		- struct mem_cgroup_per_node *mn;
275		-
276		- if (!memcg)
277		- return;
278		-
279		- mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
280		- WRITE_ONCE(mn->congested, congested);
281		-}
282		-
283		-static bool memcg_congested(pg_data_t *pgdat,
284		- struct mem_cgroup *memcg)
285		-{
286		- struct mem_cgroup_per_node *mn;
287		-
288		- mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
289		- return READ_ONCE(mn->congested);
290		-
291		-}
292	296	#else
293		-static bool global_reclaim(struct scan_control *sc)
	297	+static int prealloc_memcg_shrinker(struct shrinker *shrinker)
294	298	{
295		- return true;
	299	+ return 0;
296	300	}
297	301
298		-static bool sane_reclaim(struct scan_control *sc)
299		-{
300		- return true;
301		-}
302		-
303		-static inline void set_memcg_congestion(struct pglist_data *pgdat,
304		- struct mem_cgroup *memcg, bool congested)
	302	+static void unregister_memcg_shrinker(struct shrinker *shrinker)
305	303	{
306	304	}
307	305
308		-static inline bool memcg_congested(struct pglist_data *pgdat,
309		- struct mem_cgroup *memcg)
	306	+static bool cgroup_reclaim(struct scan_control *sc)
310	307	{
311	308	return false;
	309	+}
312	310
	311	+static bool writeback_throttling_sane(struct scan_control *sc)
	312	+{
	313	+ return true;
313	314	}
314	315	#endif
315	316
..	..	@@ -339,31 +340,21 @@
339	340	*/
340	341	unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
341	342	{
342		- unsigned long lru_size;
	343	+ unsigned long size = 0;
343	344	int zid;
344	345
345		- if (!mem_cgroup_disabled())
346		- lru_size = mem_cgroup_get_lru_size(lruvec, lru);
347		- else
348		- lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
349		-
350		- for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
	346	+ for (zid = 0; zid <= zone_idx && zid < MAX_NR_ZONES; zid++) {
351	347	struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
352		- unsigned long size;
353	348
354	349	if (!managed_zone(zone))
355	350	continue;
356	351
357	352	if (!mem_cgroup_disabled())
358		- size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
	353	+ size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
359	354	else
360		- size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],
361		- NR_ZONE_LRU_BASE + lru);
362		- lru_size -= min(size, lru_size);
	355	+ size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
363	356	}
364		-
365		- return lru_size;
366		-
	357	+ return size;
367	358	}
368	359
369	360	/*
..	..	@@ -371,7 +362,7 @@
371	362	*/
372	363	int prealloc_shrinker(struct shrinker *shrinker)
373	364	{
374		- size_t size = sizeof(*shrinker->nr_deferred);
	365	+ unsigned int size = sizeof(*shrinker->nr_deferred);
375	366
376	367	if (shrinker->flags & SHRINKER_NUMA_AWARE)
377	368	size *= nr_node_ids;
..	..	@@ -409,7 +400,7 @@
409	400	{
410	401	down_write(&shrinker_rwsem);
411	402	list_add_tail(&shrinker->list, &shrinker_list);
412		-#ifdef CONFIG_MEMCG_KMEM
	403	+#ifdef CONFIG_MEMCG
413	404	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
414	405	idr_replace(&shrinker_idr, shrinker, shrinker->id);
415	406	#endif
..	..	@@ -475,13 +466,22 @@
475	466	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
476	467
477	468	total_scan = nr;
478		- delta = freeable >> priority;
479		- delta *= 4;
480		- do_div(delta, shrinker->seeks);
	469	+ if (shrinker->seeks) {
	470	+ delta = freeable >> priority;
	471	+ delta *= 4;
	472	+ do_div(delta, shrinker->seeks);
	473	+ } else {
	474	+ /*
	475	+ * These objects don't require any IO to create. Trim
	476	+ * them aggressively under memory pressure to keep
	477	+ * them from causing refetches in the IO caches.
	478	+ */
	479	+ delta = freeable / 2;
	480	+ }
481	481
482	482	total_scan += delta;
483	483	if (total_scan < 0) {
484		- pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n",
	484	+ pr_err("shrink_slab: %pS negative objects to delete nr=%ld\n",
485	485	shrinker->scan_objects, total_scan);
486	486	total_scan = freeable;
487	487	next_deferred = nr;
..	..	@@ -567,7 +567,7 @@
567	567	return freed;
568	568	}
569	569
570		-#ifdef CONFIG_MEMCG_KMEM
	570	+#ifdef CONFIG_MEMCG
571	571	static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
572	572	struct mem_cgroup *memcg, int priority)
573	573	{
..	..	@@ -575,7 +575,7 @@
575	575	unsigned long ret, freed = 0;
576	576	int i;
577	577
578		- if (!memcg_kmem_enabled() \|\| !mem_cgroup_online(memcg))
	578	+ if (!mem_cgroup_online(memcg))
579	579	return 0;
580	580
581	581	if (!down_read_trylock(&shrinker_rwsem))
..	..	@@ -600,6 +600,11 @@
600	600	clear_bit(i, map->map);
601	601	continue;
602	602	}
	603	+
	604	+ /* Call non-slab shrinkers even though kmem is disabled */
	605	+ if (!memcg_kmem_enabled() &&
	606	+ !(shrinker->flags & SHRINKER_NONSLAB))
	607	+ continue;
603	608
604	609	ret = do_shrink_slab(&sc, shrinker, priority);
605	610	if (ret == SHRINK_EMPTY) {
..	..	@@ -637,13 +642,13 @@
637	642	up_read(&shrinker_rwsem);
638	643	return freed;
639	644	}
640		-#else /* CONFIG_MEMCG_KMEM */
	645	+#else /* CONFIG_MEMCG */
641	646	static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
642	647	struct mem_cgroup *memcg, int priority)
643	648	{
644	649	return 0;
645	650	}
646		-#endif /* CONFIG_MEMCG_KMEM */
	651	+#endif /* CONFIG_MEMCG */
647	652
648	653	/**
649	654	* shrink_slab - shrink slab caches
..	..	@@ -665,12 +670,17 @@
665	670	*
666	671	* Returns the number of reclaimed slab objects.
667	672	*/
668		-static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
	673	+unsigned long shrink_slab(gfp_t gfp_mask, int nid,
669	674	struct mem_cgroup *memcg,
670	675	int priority)
671	676	{
672	677	unsigned long ret, freed = 0;
673	678	struct shrinker *shrinker;
	679	+ bool bypass = false;
	680	+
	681	+ trace_android_vh_shrink_slab_bypass(gfp_mask, nid, memcg, priority, &bypass);
	682	+ if (bypass)
	683	+ return 0;
674	684
675	685	/*
676	686	* The root memcg might be allocated even though memcg is disabled
..	..	@@ -698,7 +708,7 @@
698	708	freed += ret;
699	709	/*
700	710	* Bail out if someone want to register a new shrinker to
701		- * prevent the regsitration from being stalled for long periods
	711	+ * prevent the registration from being stalled for long periods
702	712	* by parallel ongoing shrinking.
703	713	*/
704	714	if (rwsem_is_contended(&shrinker_rwsem)) {
..	..	@@ -712,6 +722,7 @@
712	722	cond_resched();
713	723	return freed;
714	724	}
	725	+EXPORT_SYMBOL_GPL(shrink_slab);
715	726
716	727	void drop_slab_node(int nid)
717	728	{
..	..	@@ -719,6 +730,9 @@
719	730
720	731	do {
721	732	struct mem_cgroup *memcg = NULL;
	733	+
	734	+ if (fatal_signal_pending(current))
	735	+ return;
722	736
723	737	freed = 0;
724	738	memcg = mem_cgroup_iter(NULL, NULL, NULL);
..	..	@@ -740,15 +754,14 @@
740	754	{
741	755	/*
742	756	* A freeable page cache page is referenced only by the caller
743		- * that isolated the page, the page cache radix tree and
744		- * optional buffer heads at page->private.
	757	+ * that isolated the page, the page cache and optional buffer
	758	+ * heads at page->private.
745	759	*/
746		- int radix_pins = PageTransHuge(page) && PageSwapCache(page) ?
747		- HPAGE_PMD_NR : 1;
748		- return page_count(page) - page_has_private(page) == 1 + radix_pins;
	760	+ int page_cache_pins = thp_nr_pages(page);
	761	+ return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
749	762	}
750	763
751		-static int may_write_to_inode(struct inode inode, struct scan_control sc)
	764	+static int may_write_to_inode(struct inode *inode)
752	765	{
753	766	if (current->flags & PF_SWAPWRITE)
754	767	return 1;
..	..	@@ -796,8 +809,7 @@
796	809	* pageout is called by shrink_page_list() for each dirty page.
797	810	* Calls ->writepage().
798	811	*/
799		-static pageout_t pageout(struct page page, struct address_space mapping,
800		- struct scan_control *sc)
	812	+static pageout_t pageout(struct page page, struct address_space mapping)
801	813	{
802	814	/*
803	815	* If the page is dirty, only perform writeback if that write
..	..	@@ -833,7 +845,7 @@
833	845	}
834	846	if (mapping->a_ops->writepage == NULL)
835	847	return PAGE_ACTIVATE;
836		- if (!may_write_to_inode(mapping->host, sc))
	848	+ if (!may_write_to_inode(mapping->host))
837	849	return PAGE_KEEP;
838	850
839	851	if (clear_page_dirty_for_io(page)) {
..	..	@@ -872,10 +884,11 @@
872	884	* gets returned with a refcount of 0.
873	885	*/
874	886	static int __remove_mapping(struct address_space mapping, struct page page,
875		- bool reclaimed)
	887	+ bool reclaimed, struct mem_cgroup *target_memcg)
876	888	{
877	889	unsigned long flags;
878	890	int refcount;
	891	+ void *shadow = NULL;
879	892
880	893	BUG_ON(!PageLocked(page));
881	894	BUG_ON(mapping != page_mapping(page));
..	..	@@ -906,10 +919,7 @@
906	919	* Note that if SetPageDirty is always performed via set_page_dirty,
907	920	* and thus under the i_pages lock, then this ordering is not required.
908	921	*/
909		- if (unlikely(PageTransHuge(page)) && PageSwapCache(page))
910		- refcount = 1 + HPAGE_PMD_NR;
911		- else
912		- refcount = 2;
	922	+ refcount = 1 + compound_nr(page);
913	923	if (!page_ref_freeze(page, refcount))
914	924	goto cannot_free;
915	925	/* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
..	..	@@ -921,12 +931,13 @@
921	931	if (PageSwapCache(page)) {
922	932	swp_entry_t swap = { .val = page_private(page) };
923	933	mem_cgroup_swapout(page, swap);
924		- __delete_from_swap_cache(page);
	934	+ if (reclaimed && !mapping_exiting(mapping))
	935	+ shadow = workingset_eviction(page, target_memcg);
	936	+ __delete_from_swap_cache(page, swap, shadow);
925	937	xa_unlock_irqrestore(&mapping->i_pages, flags);
926	938	put_swap_page(page, swap);
927	939	} else {
928	940	void (freepage)(struct page );
929		- void *shadow = NULL;
930	941
931	942	freepage = mapping->a_ops->freepage;
932	943	/*
..	..	@@ -934,7 +945,7 @@
934	945	* order to detect refaults, thus thrashing, later on.
935	946	*
936	947	* But don't store shadows in an address space that is
937		- * already exiting. This is not just an optizimation,
	948	+ * already exiting. This is not just an optimization,
938	949	* inode reclaim needs to empty out the radix tree or
939	950	* the nodes are lost. Don't plant shadows behind its
940	951	* back.
..	..	@@ -945,9 +956,9 @@
945	956	* exceptional entries and shadow exceptional entries in the
946	957	* same address_space.
947	958	*/
948		- if (reclaimed && page_is_file_cache(page) &&
	959	+ if (reclaimed && page_is_file_lru(page) &&
949	960	!mapping_exiting(mapping) && !dax_mapping(mapping))
950		- shadow = workingset_eviction(mapping, page);
	961	+ shadow = workingset_eviction(page, target_memcg);
951	962	__delete_from_page_cache(page, shadow);
952	963	xa_unlock_irqrestore(&mapping->i_pages, flags);
953	964
..	..	@@ -970,7 +981,7 @@
970	981	*/
971	982	int remove_mapping(struct address_space mapping, struct page page)
972	983	{
973		- if (__remove_mapping(mapping, page, false)) {
	984	+ if (__remove_mapping(mapping, page, false, NULL)) {
974	985	/*
975	986	* Unfreezing the refcount with 1 rather than 2 effectively
976	987	* drops the pagecache ref for us without requiring another
..	..	@@ -1009,11 +1020,24 @@
1009	1020	{
1010	1021	int referenced_ptes, referenced_page;
1011	1022	unsigned long vm_flags;
	1023	+ bool should_protect = false;
	1024	+ bool trylock_fail = false;
	1025	+ int ret = 0;
1012	1026
	1027	+ trace_android_vh_page_should_be_protected(page, &should_protect);
	1028	+ if (unlikely(should_protect))
	1029	+ return PAGEREF_ACTIVATE;
	1030	+
	1031	+ trace_android_vh_page_trylock_set(page);
	1032	+ trace_android_vh_check_page_look_around_ref(page, &ret);
	1033	+ if (ret)
	1034	+ return ret;
1013	1035	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
1014	1036	&vm_flags);
1015	1037	referenced_page = TestClearPageReferenced(page);
1016		-
	1038	+ trace_android_vh_page_trylock_get_result(page, &trylock_fail);
	1039	+ if (trylock_fail)
	1040	+ return PAGEREF_KEEP;
1017	1041	/*
1018	1042	* Mlock lost the isolation race with us. Let try_to_unmap()
1019	1043	* move the page to the unevictable list.
..	..	@@ -1021,9 +1045,11 @@
1021	1045	if (vm_flags & VM_LOCKED)
1022	1046	return PAGEREF_RECLAIM;
1023	1047
	1048	+ /* rmap lock contention: rotate */
	1049	+ if (referenced_ptes == -1)
	1050	+ return PAGEREF_KEEP;
	1051	+
1024	1052	if (referenced_ptes) {
1025		- if (PageSwapBacked(page))
1026		- return PAGEREF_ACTIVATE;
1027	1053	/*
1028	1054	* All mapped pages start out with page table
1029	1055	* references from the instantiating fault, so we need
..	..	@@ -1046,7 +1072,7 @@
1046	1072	/*
1047	1073	* Activate file-backed executable pages after first usage.
1048	1074	*/
1049		- if (vm_flags & VM_EXEC)
	1075	+ if ((vm_flags & VM_EXEC) && !PageSwapBacked(page))
1050	1076	return PAGEREF_ACTIVATE;
1051	1077
1052	1078	return PAGEREF_KEEP;
..	..	@@ -1069,7 +1095,7 @@
1069	1095	* Anonymous pages are not handled by flushers and must be written
1070	1096	* from reclaim context. Do not stall reclaim based on them
1071	1097	*/
1072		- if (!page_is_file_cache(page) \|\|
	1098	+ if (!page_is_file_lru(page) \|\|
1073	1099	(PageAnon(page) && !PageSwapBacked(page))) {
1074	1100	*dirty = false;
1075	1101	*writeback = false;
..	..	@@ -1092,33 +1118,27 @@
1092	1118	/*
1093	1119	* shrink_page_list() returns the number of reclaimed pages
1094	1120	*/
1095		-static unsigned long shrink_page_list(struct list_head *page_list,
1096		- struct pglist_data *pgdat,
1097		- struct scan_control *sc,
1098		- enum ttu_flags ttu_flags,
1099		- struct reclaim_stat *stat,
1100		- bool force_reclaim)
	1121	+static unsigned int shrink_page_list(struct list_head *page_list,
	1122	+ struct pglist_data *pgdat,
	1123	+ struct scan_control *sc,
	1124	+ struct reclaim_stat *stat,
	1125	+ bool ignore_references)
1101	1126	{
1102	1127	LIST_HEAD(ret_pages);
1103	1128	LIST_HEAD(free_pages);
1104		- int pgactivate = 0;
1105		- unsigned nr_unqueued_dirty = 0;
1106		- unsigned nr_dirty = 0;
1107		- unsigned nr_congested = 0;
1108		- unsigned nr_reclaimed = 0;
1109		- unsigned nr_writeback = 0;
1110		- unsigned nr_immediate = 0;
1111		- unsigned nr_ref_keep = 0;
1112		- unsigned nr_unmap_fail = 0;
	1129	+ unsigned int nr_reclaimed = 0;
	1130	+ unsigned int pgactivate = 0;
	1131	+ bool page_trylock_result;
1113	1132
	1133	+ memset(stat, 0, sizeof(*stat));
1114	1134	cond_resched();
1115	1135
1116	1136	while (!list_empty(page_list)) {
1117	1137	struct address_space *mapping;
1118	1138	struct page *page;
1119		- int may_enter_fs;
1120		- enum page_references references = PAGEREF_RECLAIM_CLEAN;
1121		- bool dirty, writeback;
	1139	+ enum page_references references = PAGEREF_RECLAIM;
	1140	+ bool dirty, writeback, may_enter_fs;
	1141	+ unsigned int nr_pages;
1122	1142
1123	1143	cond_resched();
1124	1144
..	..	@@ -1130,18 +1150,16 @@
1130	1150
1131	1151	VM_BUG_ON_PAGE(PageActive(page), page);
1132	1152
1133		- sc->nr_scanned++;
	1153	+ nr_pages = compound_nr(page);
	1154	+
	1155	+ /* Account the number of base pages even though THP */
	1156	+ sc->nr_scanned += nr_pages;
1134	1157
1135	1158	if (unlikely(!page_evictable(page)))
1136	1159	goto activate_locked;
1137	1160
1138	1161	if (!sc->may_unmap && page_mapped(page))
1139	1162	goto keep_locked;
1140		-
1141		- /* Double the slab pressure for mapped and swapcache pages */
1142		- if ((page_mapped(page) \|\| PageSwapCache(page)) &&
1143		- !(PageAnon(page) && !PageSwapBacked(page)))
1144		- sc->nr_scanned++;
1145	1163
1146	1164	may_enter_fs = (sc->gfp_mask & __GFP_FS) \|\|
1147	1165	(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
..	..	@@ -1154,10 +1172,10 @@
1154	1172	*/
1155	1173	page_check_dirty_writeback(page, &dirty, &writeback);
1156	1174	if (dirty \|\| writeback)
1157		- nr_dirty++;
	1175	+ stat->nr_dirty++;
1158	1176
1159	1177	if (dirty && !writeback)
1160		- nr_unqueued_dirty++;
	1178	+ stat->nr_unqueued_dirty++;
1161	1179
1162	1180	/*
1163	1181	* Treat this page as congested if the underlying BDI is or if
..	..	@@ -1169,7 +1187,7 @@
1169	1187	if (((dirty \|\| writeback) && mapping &&
1170	1188	inode_write_congested(mapping->host)) \|\|
1171	1189	(writeback && PageReclaim(page)))
1172		- nr_congested++;
	1190	+ stat->nr_congested++;
1173	1191
1174	1192	/*
1175	1193	* If a page at the tail of the LRU is under writeback, there
..	..	@@ -1218,11 +1236,11 @@
1218	1236	if (current_is_kswapd() &&
1219	1237	PageReclaim(page) &&
1220	1238	test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1221		- nr_immediate++;
	1239	+ stat->nr_immediate++;
1222	1240	goto activate_locked;
1223	1241
1224	1242	/* Case 2 above */
1225		- } else if (sane_reclaim(sc) \|\|
	1243	+ } else if (writeback_throttling_sane(sc) \|\|
1226	1244	!PageReclaim(page) \|\| !may_enter_fs) {
1227	1245	/*
1228	1246	* This is slightly racy - end_page_writeback()
..	..	@@ -1236,7 +1254,7 @@
1236	1254	* and it's also appropriate in global reclaim.
1237	1255	*/
1238	1256	SetPageReclaim(page);
1239		- nr_writeback++;
	1257	+ stat->nr_writeback++;
1240	1258	goto activate_locked;
1241	1259
1242	1260	/* Case 3 above */
..	..	@@ -1249,14 +1267,14 @@
1249	1267	}
1250	1268	}
1251	1269
1252		- if (!force_reclaim)
	1270	+ if (!ignore_references)
1253	1271	references = page_check_references(page, sc);
1254	1272
1255	1273	switch (references) {
1256	1274	case PAGEREF_ACTIVATE:
1257	1275	goto activate_locked;
1258	1276	case PAGEREF_KEEP:
1259		- nr_ref_keep++;
	1277	+ stat->nr_ref_keep += nr_pages;
1260	1278	goto keep_locked;
1261	1279	case PAGEREF_RECLAIM:
1262	1280	case PAGEREF_RECLAIM_CLEAN:
..	..	@@ -1271,6 +1289,8 @@
1271	1289	if (PageAnon(page) && PageSwapBacked(page)) {
1272	1290	if (!PageSwapCache(page)) {
1273	1291	if (!(sc->gfp_mask & __GFP_IO))
	1292	+ goto keep_locked;
	1293	+ if (page_maybe_dma_pinned(page))
1274	1294	goto keep_locked;
1275	1295	if (PageTransHuge(page)) {
1276	1296	/* cannot split THP, skip it */
..	..	@@ -1288,7 +1308,7 @@
1288	1308	}
1289	1309	if (!add_to_swap(page)) {
1290	1310	if (!PageTransHuge(page))
1291		- goto activate_locked;
	1311	+ goto activate_locked_split;
1292	1312	/* Fallback to swap normal pages */
1293	1313	if (split_huge_page_to_list(page,
1294	1314	page_list))
..	..	@@ -1297,10 +1317,10 @@
1297	1317	count_vm_event(THP_SWPOUT_FALLBACK);
1298	1318	#endif
1299	1319	if (!add_to_swap(page))
1300		- goto activate_locked;
	1320	+ goto activate_locked_split;
1301	1321	}
1302	1322
1303		- may_enter_fs = 1;
	1323	+ may_enter_fs = true;
1304	1324
1305	1325	/* Adding to swap updated mapping */
1306	1326	mapping = page_mapping(page);
..	..	@@ -1312,16 +1332,33 @@
1312	1332	}
1313	1333
1314	1334	/*
	1335	+ * THP may get split above, need minus tail pages and update
	1336	+ * nr_pages to avoid accounting tail pages twice.
	1337	+ *
	1338	+ * The tail pages that are added into swap cache successfully
	1339	+ * reach here.
	1340	+ */
	1341	+ if ((nr_pages > 1) && !PageTransHuge(page)) {
	1342	+ sc->nr_scanned -= (nr_pages - 1);
	1343	+ nr_pages = 1;
	1344	+ }
	1345	+
	1346	+ /*
1315	1347	* The page is mapped into the page tables of one or more
1316	1348	* processes. Try to unmap it here.
1317	1349	*/
1318	1350	if (page_mapped(page)) {
1319		- enum ttu_flags flags = ttu_flags \| TTU_BATCH_FLUSH;
	1351	+ enum ttu_flags flags = TTU_BATCH_FLUSH;
	1352	+ bool was_swapbacked = PageSwapBacked(page);
1320	1353
1321	1354	if (unlikely(PageTransHuge(page)))
1322	1355	flags \|= TTU_SPLIT_HUGE_PMD;
	1356	+ if (!ignore_references)
	1357	+ trace_android_vh_page_trylock_set(page);
1323	1358	if (!try_to_unmap(page, flags)) {
1324		- nr_unmap_fail++;
	1359	+ stat->nr_unmap_fail += nr_pages;
	1360	+ if (!was_swapbacked && PageSwapBacked(page))
	1361	+ stat->nr_lazyfree_fail += nr_pages;
1325	1362	goto activate_locked;
1326	1363	}
1327	1364	}
..	..	@@ -1337,7 +1374,7 @@
1337	1374	* the rest of the LRU for clean pages and see
1338	1375	* the same dirty pages again (PageReclaim).
1339	1376	*/
1340		- if (page_is_file_cache(page) &&
	1377	+ if (page_is_file_lru(page) &&
1341	1378	(!current_is_kswapd() \|\| !PageReclaim(page) \|\|
1342	1379	!test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1343	1380	/*
..	..	@@ -1365,12 +1402,14 @@
1365	1402	* starts and then write it out here.
1366	1403	*/
1367	1404	try_to_unmap_flush_dirty();
1368		- switch (pageout(page, mapping, sc)) {
	1405	+ switch (pageout(page, mapping)) {
1369	1406	case PAGE_KEEP:
1370	1407	goto keep_locked;
1371	1408	case PAGE_ACTIVATE:
1372	1409	goto activate_locked;
1373	1410	case PAGE_SUCCESS:
	1411	+ stat->nr_pageout += thp_nr_pages(page);
	1412	+
1374	1413	if (PageWriteback(page))
1375	1414	goto keep;
1376	1415	if (PageDirty(page))
..	..	@@ -1426,6 +1465,7 @@
1426	1465	* increment nr_reclaimed here (and
1427	1466	* leave it off the LRU).
1428	1467	*/
	1468	+ trace_android_vh_page_trylock_clear(page);
1429	1469	nr_reclaimed++;
1430	1470	continue;
1431	1471	}
..	..	@@ -1443,30 +1483,38 @@
1443	1483
1444	1484	count_vm_event(PGLAZYFREED);
1445	1485	count_memcg_page_event(page, PGLAZYFREED);
1446		- } else if (!mapping \|\| !__remove_mapping(mapping, page, true))
	1486	+ } else if (!mapping \|\| !__remove_mapping(mapping, page, true,
	1487	+ sc->target_mem_cgroup))
1447	1488	goto keep_locked;
1448		- /*
1449		- * At this point, we have no other references and there is
1450		- * no way to pick any more up (removed from LRU, removed
1451		- * from pagecache). Can use non-atomic bitops now (and
1452		- * we obviously don't have to worry about waking up a process
1453		- * waiting on the page lock, because there are no references.
1454		- */
1455		- __ClearPageLocked(page);
	1489	+
	1490	+ unlock_page(page);
1456	1491	free_it:
1457		- nr_reclaimed++;
	1492	+ /*
	1493	+ * THP may get swapped out in a whole, need account
	1494	+ * all base pages.
	1495	+ */
	1496	+ nr_reclaimed += nr_pages;
1458	1497
1459	1498	/*
1460	1499	* Is there need to periodically free_page_list? It would
1461	1500	* appear not as the counts should be low
1462	1501	*/
1463		- if (unlikely(PageTransHuge(page))) {
1464		- mem_cgroup_uncharge(page);
1465		- (*get_compound_page_dtor(page))(page);
1466		- } else
	1502	+ trace_android_vh_page_trylock_clear(page);
	1503	+ if (unlikely(PageTransHuge(page)))
	1504	+ destroy_compound_page(page);
	1505	+ else
1467	1506	list_add(&page->lru, &free_pages);
1468	1507	continue;
1469	1508
	1509	+activate_locked_split:
	1510	+ /*
	1511	+ * The tail pages that are failed to add into swap cache
	1512	+ * reach here. Fixup nr_scanned and nr_pages.
	1513	+ */
	1514	+ if (nr_pages > 1) {
	1515	+ sc->nr_scanned -= (nr_pages - 1);
	1516	+ nr_pages = 1;
	1517	+ }
1470	1518	activate_locked:
1471	1519	/* Not a candidate for swapping, so reclaim swap space. */
1472	1520	if (PageSwapCache(page) && (mem_cgroup_swap_full(page) \|\|
..	..	@@ -1474,16 +1522,31 @@
1474	1522	try_to_free_swap(page);
1475	1523	VM_BUG_ON_PAGE(PageActive(page), page);
1476	1524	if (!PageMlocked(page)) {
	1525	+ int type = page_is_file_lru(page);
1477	1526	SetPageActive(page);
1478		- pgactivate++;
	1527	+ stat->nr_activate[type] += nr_pages;
1479	1528	count_memcg_page_event(page, PGACTIVATE);
1480	1529	}
1481	1530	keep_locked:
	1531	+ /*
	1532	+ * The page with trylock-bit will be added ret_pages and
	1533	+ * handled in trace_android_vh_handle_failed_page_trylock.
	1534	+ * In the progress[unlock_page, handled], the page carried
	1535	+ * with trylock-bit will cause some error-issues in other
	1536	+ * scene, so clear trylock-bit here.
	1537	+ * trace_android_vh_page_trylock_get_result will clear
	1538	+ * trylock-bit and return if page tyrlock failed in
	1539	+ * reclaim-process. Here we just want to clear trylock-bit
	1540	+ * so that ignore page_trylock_result.
	1541	+ */
	1542	+ trace_android_vh_page_trylock_get_result(page, &page_trylock_result);
1482	1543	unlock_page(page);
1483	1544	keep:
1484	1545	list_add(&page->lru, &ret_pages);
1485	1546	VM_BUG_ON_PAGE(PageLRU(page) \|\| PageUnevictable(page), page);
1486	1547	}
	1548	+
	1549	+ pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
1487	1550
1488	1551	mem_cgroup_uncharge_list(&free_pages);
1489	1552	try_to_unmap_flush();
..	..	@@ -1492,20 +1555,10 @@
1492	1555	list_splice(&ret_pages, page_list);
1493	1556	count_vm_events(PGACTIVATE, pgactivate);
1494	1557
1495		- if (stat) {
1496		- stat->nr_dirty = nr_dirty;
1497		- stat->nr_congested = nr_congested;
1498		- stat->nr_unqueued_dirty = nr_unqueued_dirty;
1499		- stat->nr_writeback = nr_writeback;
1500		- stat->nr_immediate = nr_immediate;
1501		- stat->nr_activate = pgactivate;
1502		- stat->nr_ref_keep = nr_ref_keep;
1503		- stat->nr_unmap_fail = nr_unmap_fail;
1504		- }
1505	1558	return nr_reclaimed;
1506	1559	}
1507	1560
1508		-unsigned long reclaim_clean_pages_from_list(struct zone *zone,
	1561	+unsigned int reclaim_clean_pages_from_list(struct zone *zone,
1509	1562	struct list_head *page_list)
1510	1563	{
1511	1564	struct scan_control sc = {
..	..	@@ -1513,23 +1566,35 @@
1513	1566	.priority = DEF_PRIORITY,
1514	1567	.may_unmap = 1,
1515	1568	};
1516		- unsigned long ret;
	1569	+ struct reclaim_stat stat;
	1570	+ unsigned int nr_reclaimed;
1517	1571	struct page page, next;
1518	1572	LIST_HEAD(clean_pages);
1519	1573
1520	1574	list_for_each_entry_safe(page, next, page_list, lru) {
1521		- if (page_is_file_cache(page) && !PageDirty(page) &&
	1575	+ if (page_is_file_lru(page) && !PageDirty(page) &&
1522	1576	!__PageMovable(page) && !PageUnevictable(page)) {
1523	1577	ClearPageActive(page);
1524	1578	list_move(&page->lru, &clean_pages);
1525	1579	}
1526	1580	}
1527	1581
1528		- ret = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
1529		- TTU_IGNORE_ACCESS, NULL, true);
	1582	+ nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
	1583	+ &stat, true);
1530	1584	list_splice(&clean_pages, page_list);
1531		- mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, -ret);
1532		- return ret;
	1585	+ mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
	1586	+ -(long)nr_reclaimed);
	1587	+ /*
	1588	+ * Since lazyfree pages are isolated from file LRU from the beginning,
	1589	+ * they will rotate back to anonymous LRU in the end if it failed to
	1590	+ * discard so isolated count will be mismatched.
	1591	+ * Compensate the isolated count for both LRU lists.
	1592	+ */
	1593	+ mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON,
	1594	+ stat.nr_lazyfree_fail);
	1595	+ mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
	1596	+ -(long)stat.nr_lazyfree_fail);
	1597	+ return nr_reclaimed;
1533	1598	}
1534	1599
1535	1600	/*
..	..	@@ -1612,7 +1677,7 @@
1612	1677
1613	1678	/*
1614	1679	* Update LRU sizes after isolating pages. The LRU size updates must
1615		- * be complete before mem_cgroup_update_lru_size due to a santity check.
	1680	+ * be complete before mem_cgroup_update_lru_size due to a sanity check.
1616	1681	*/
1617	1682	static __always_inline void update_lru_sizes(struct lruvec *lruvec,
1618	1683	enum lru_list lru, unsigned long *nr_zone_taken)
..	..	@@ -1623,16 +1688,13 @@
1623	1688	if (!nr_zone_taken[zid])
1624	1689	continue;
1625	1690
1626		- __update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1627		-#ifdef CONFIG_MEMCG
1628		- mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1629		-#endif
	1691	+ update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1630	1692	}
1631	1693
1632	1694	}
1633	1695
1634		-/*
1635		- * zone_lru_lock is heavily contended. Some of the functions that
	1696	+/**
	1697	+ * pgdat->lru_lock is heavily contended. Some of the functions that
1636	1698	* shrink the lists perform better by taking out a batch of pages
1637	1699	* and working on them outside the LRU lock.
1638	1700	*
..	..	@@ -1646,7 +1708,6 @@
1646	1708	* @dst: The temp list to put pages on to.
1647	1709	* @nr_scanned: The number of pages that were scanned.
1648	1710	* @sc: The scan_control struct for this reclaim session
1649		- * @mode: One of the LRU isolation modes
1650	1711	* @lru: LRU list id for isolating
1651	1712	*
1652	1713	* returns how many pages were moved onto *@dst.
..	..	@@ -1654,7 +1715,7 @@
1654	1715	static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1655	1716	struct lruvec lruvec, struct list_head dst,
1656	1717	unsigned long nr_scanned, struct scan_control sc,
1657		- isolate_mode_t mode, enum lru_list lru)
	1718	+ enum lru_list lru)
1658	1719	{
1659	1720	struct list_head *src = &lruvec->lists[lru];
1660	1721	unsigned long nr_taken = 0;
..	..	@@ -1663,11 +1724,11 @@
1663	1724	unsigned long skipped = 0;
1664	1725	unsigned long scan, total_scan, nr_pages;
1665	1726	LIST_HEAD(pages_skipped);
	1727	+ isolate_mode_t mode = (sc->may_unmap ? 0 : ISOLATE_UNMAPPED);
1666	1728
	1729	+ total_scan = 0;
1667	1730	scan = 0;
1668		- for (total_scan = 0;
1669		- scan < nr_to_scan && nr_taken < nr_to_scan && !list_empty(src);
1670		- total_scan++) {
	1731	+ while (scan < nr_to_scan && !list_empty(src)) {
1671	1732	struct page *page;
1672	1733
1673	1734	page = lru_to_page(src);
..	..	@@ -1675,9 +1736,12 @@
1675	1736
1676	1737	VM_BUG_ON_PAGE(!PageLRU(page), page);
1677	1738
	1739	+ nr_pages = compound_nr(page);
	1740	+ total_scan += nr_pages;
	1741	+
1678	1742	if (page_zonenum(page) > sc->reclaim_idx) {
1679	1743	list_move(&page->lru, &pages_skipped);
1680		- nr_skipped[page_zonenum(page)]++;
	1744	+ nr_skipped[page_zonenum(page)] += nr_pages;
1681	1745	continue;
1682	1746	}
1683	1747
..	..	@@ -1686,13 +1750,17 @@
1686	1750	* return with no isolated pages if the LRU mostly contains
1687	1751	* ineligible pages. This causes the VM to not reclaim any
1688	1752	* pages, triggering a premature OOM.
	1753	+ *
	1754	+ * Account all tail pages of THP. This would not cause
	1755	+ * premature OOM since __isolate_lru_page() returns -EBUSY
	1756	+ * only when the page is being freed somewhere else.
1689	1757	*/
1690		- scan++;
	1758	+ scan += nr_pages;
1691	1759	switch (__isolate_lru_page(page, mode)) {
1692	1760	case 0:
1693		- nr_pages = hpage_nr_pages(page);
1694	1761	nr_taken += nr_pages;
1695	1762	nr_zone_taken[page_zonenum(page)] += nr_pages;
	1763	+ trace_android_vh_del_page_from_lrulist(page, false, lru);
1696	1764	list_move(&page->lru, dst);
1697	1765	break;
1698	1766
..	..	@@ -1753,7 +1821,7 @@
1753	1821	* Restrictions:
1754	1822	*
1755	1823	* (1) Must be called with an elevated refcount on the page. This is a
1756		- * fundamentnal difference from isolate_lru_pages (which is called
	1824	+ * fundamental difference from isolate_lru_pages (which is called
1757	1825	* without a stable reference).
1758	1826	* (2) the lru_lock must not be held.
1759	1827	* (3) interrupts must be enabled.
..	..	@@ -1766,11 +1834,11 @@
1766	1834	WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
1767	1835
1768	1836	if (PageLRU(page)) {
1769		- struct zone *zone = page_zone(page);
	1837	+ pg_data_t *pgdat = page_pgdat(page);
1770	1838	struct lruvec *lruvec;
1771	1839
1772		- spin_lock_irq(zone_lru_lock(zone));
1773		- lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
	1840	+ spin_lock_irq(&pgdat->lru_lock);
	1841	+ lruvec = mem_cgroup_page_lruvec(page, pgdat);
1774	1842	if (PageLRU(page)) {
1775	1843	int lru = page_lru(page);
1776	1844	get_page(page);
..	..	@@ -1778,14 +1846,14 @@
1778	1846	del_page_from_lru_list(page, lruvec, lru);
1779	1847	ret = 0;
1780	1848	}
1781		- spin_unlock_irq(zone_lru_lock(zone));
	1849	+ spin_unlock_irq(&pgdat->lru_lock);
1782	1850	}
1783	1851	return ret;
1784	1852	}
1785	1853
1786	1854	/*
1787	1855	* A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1788		- * then get resheduled. When there are massive number of tasks doing page
	1856	+ * then get rescheduled. When there are massive number of tasks doing page
1789	1857	* allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1790	1858	* the LRU list will go small and be scanned faster than necessary, leading to
1791	1859	* unnecessary swapping, thrashing and OOM.
..	..	@@ -1798,7 +1866,7 @@
1798	1866	if (current_is_kswapd())
1799	1867	return 0;
1800	1868
1801		- if (!sane_reclaim(sc))
	1869	+ if (!writeback_throttling_sane(sc))
1802	1870	return 0;
1803	1871
1804	1872	if (file) {
..	..	@@ -1820,40 +1888,55 @@
1820	1888	return isolated > inactive;
1821	1889	}
1822	1890
1823		-static noinline_for_stack void
1824		-putback_inactive_pages(struct lruvec lruvec, struct list_head page_list)
	1891	+/*
	1892	+ * This moves pages from @list to corresponding LRU list.
	1893	+ *
	1894	+ * We move them the other way if the page is referenced by one or more
	1895	+ * processes, from rmap.
	1896	+ *
	1897	+ * If the pages are mostly unmapped, the processing is fast and it is
	1898	+ * appropriate to hold zone_lru_lock across the whole operation. But if
	1899	+ * the pages are mapped, the processing is slow (page_referenced()) so we
	1900	+ * should drop zone_lru_lock around each page. It's impossible to balance
	1901	+ * this, so instead we remove the pages from the LRU while processing them.
	1902	+ * It is safe to rely on PG_active against the non-LRU pages in here because
	1903	+ * nobody will play with that bit on a non-LRU page.
	1904	+ *
	1905	+ * The downside is that we have to touch page->_refcount against each page.
	1906	+ * But we had to alter page->flags anyway.
	1907	+ *
	1908	+ * Returns the number of pages moved to the given lruvec.
	1909	+ */
	1910	+
	1911	+static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec,
	1912	+ struct list_head *list)
1825	1913	{
1826		- struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1827	1914	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
	1915	+ int nr_pages, nr_moved = 0;
1828	1916	LIST_HEAD(pages_to_free);
	1917	+ struct page *page;
	1918	+ enum lru_list lru;
1829	1919
1830		- /*
1831		- * Put back any unfreeable pages.
1832		- */
1833		- while (!list_empty(page_list)) {
1834		- struct page *page = lru_to_page(page_list);
1835		- int lru;
1836		-
	1920	+ while (!list_empty(list)) {
	1921	+ page = lru_to_page(list);
1837	1922	VM_BUG_ON_PAGE(PageLRU(page), page);
1838		- list_del(&page->lru);
1839	1923	if (unlikely(!page_evictable(page))) {
	1924	+ list_del(&page->lru);
1840	1925	spin_unlock_irq(&pgdat->lru_lock);
1841	1926	putback_lru_page(page);
1842	1927	spin_lock_irq(&pgdat->lru_lock);
1843	1928	continue;
1844	1929	}
1845		-
1846	1930	lruvec = mem_cgroup_page_lruvec(page, pgdat);
1847	1931
1848	1932	SetPageLRU(page);
1849	1933	lru = page_lru(page);
1850		- add_page_to_lru_list(page, lruvec, lru);
1851	1934
1852		- if (is_active_lru(lru)) {
1853		- int file = is_file_lru(lru);
1854		- int numpages = hpage_nr_pages(page);
1855		- reclaim_stat->recent_rotated[file] += numpages;
1856		- }
	1935	+ nr_pages = thp_nr_pages(page);
	1936	+ update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);
	1937	+ list_move(&page->lru, &lruvec->lists[lru]);
	1938	+ trace_android_vh_add_page_to_lrulist(page, false, lru);
	1939	+
1857	1940	if (put_page_testzero(page)) {
1858	1941	__ClearPageLRU(page);
1859	1942	__ClearPageActive(page);
..	..	@@ -1861,29 +1944,34 @@
1861	1944
1862	1945	if (unlikely(PageCompound(page))) {
1863	1946	spin_unlock_irq(&pgdat->lru_lock);
1864		- mem_cgroup_uncharge(page);
1865		- (*get_compound_page_dtor(page))(page);
	1947	+ destroy_compound_page(page);
1866	1948	spin_lock_irq(&pgdat->lru_lock);
1867	1949	} else
1868	1950	list_add(&page->lru, &pages_to_free);
	1951	+ } else {
	1952	+ nr_moved += nr_pages;
	1953	+ if (PageActive(page))
	1954	+ workingset_age_nonresident(lruvec, nr_pages);
1869	1955	}
1870	1956	}
1871	1957
1872	1958	/*
1873	1959	* To save our caller's stack, now use input list for pages to free.
1874	1960	*/
1875		- list_splice(&pages_to_free, page_list);
	1961	+ list_splice(&pages_to_free, list);
	1962	+
	1963	+ return nr_moved;
1876	1964	}
1877	1965
1878	1966	/*
1879	1967	* If a kernel thread (such as nfsd for loop-back mounts) services
1880		- * a backing device by writing to the page cache it sets PF_LESS_THROTTLE.
	1968	+ * a backing device by writing to the page cache it sets PF_LOCAL_THROTTLE.
1881	1969	* In that case we should only throttle if the backing device it is
1882	1970	* writing to is congested. In other cases it is safe to throttle.
1883	1971	*/
1884	1972	static int current_may_throttle(void)
1885	1973	{
1886		- return !(current->flags & PF_LESS_THROTTLE) \|\|
	1974	+ return !(current->flags & PF_LOCAL_THROTTLE) \|\|
1887	1975	current->backing_dev_info == NULL \|\|
1888	1976	bdi_write_congested(current->backing_dev_info);
1889	1977	}
..	..	@@ -1898,13 +1986,12 @@
1898	1986	{
1899	1987	LIST_HEAD(page_list);
1900	1988	unsigned long nr_scanned;
1901		- unsigned long nr_reclaimed = 0;
	1989	+ unsigned int nr_reclaimed = 0;
1902	1990	unsigned long nr_taken;
1903		- struct reclaim_stat stat = {};
1904		- isolate_mode_t isolate_mode = 0;
1905		- int file = is_file_lru(lru);
	1991	+ struct reclaim_stat stat;
	1992	+ bool file = is_file_lru(lru);
	1993	+ enum vm_event_item item;
1906	1994	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1907		- struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1908	1995	bool stalled = false;
1909	1996
1910	1997	while (unlikely(too_many_isolated(pgdat, file, sc))) {
..	..	@@ -1922,54 +2009,37 @@
1922	2009
1923	2010	lru_add_drain();
1924	2011
1925		- if (!sc->may_unmap)
1926		- isolate_mode \|= ISOLATE_UNMAPPED;
1927		-
1928	2012	spin_lock_irq(&pgdat->lru_lock);
1929	2013
1930	2014	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
1931		- &nr_scanned, sc, isolate_mode, lru);
	2015	+ &nr_scanned, sc, lru);
1932	2016
1933	2017	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
1934		- reclaim_stat->recent_scanned[file] += nr_taken;
	2018	+ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
	2019	+ if (!cgroup_reclaim(sc))
	2020	+ __count_vm_events(item, nr_scanned);
	2021	+ __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
	2022	+ __count_vm_events(PGSCAN_ANON + file, nr_scanned);
1935	2023
1936		- if (current_is_kswapd()) {
1937		- if (global_reclaim(sc))
1938		- __count_vm_events(PGSCAN_KSWAPD, nr_scanned);
1939		- count_memcg_events(lruvec_memcg(lruvec), PGSCAN_KSWAPD,
1940		- nr_scanned);
1941		- } else {
1942		- if (global_reclaim(sc))
1943		- __count_vm_events(PGSCAN_DIRECT, nr_scanned);
1944		- count_memcg_events(lruvec_memcg(lruvec), PGSCAN_DIRECT,
1945		- nr_scanned);
1946		- }
1947	2024	spin_unlock_irq(&pgdat->lru_lock);
1948	2025
1949	2026	if (nr_taken == 0)
1950	2027	return 0;
1951	2028
1952		- nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, 0,
1953		- &stat, false);
	2029	+ nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false);
	2030	+ trace_android_vh_handle_failed_page_trylock(&page_list);
1954	2031
1955	2032	spin_lock_irq(&pgdat->lru_lock);
1956	2033
1957		- if (current_is_kswapd()) {
1958		- if (global_reclaim(sc))
1959		- __count_vm_events(PGSTEAL_KSWAPD, nr_reclaimed);
1960		- count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_KSWAPD,
1961		- nr_reclaimed);
1962		- } else {
1963		- if (global_reclaim(sc))
1964		- __count_vm_events(PGSTEAL_DIRECT, nr_reclaimed);
1965		- count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_DIRECT,
1966		- nr_reclaimed);
1967		- }
1968		-
1969		- putback_inactive_pages(lruvec, &page_list);
	2034	+ move_pages_to_lru(lruvec, &page_list);
1970	2035
1971	2036	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
1972		-
	2037	+ lru_note_cost(lruvec, file, stat.nr_pageout);
	2038	+ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
	2039	+ if (!cgroup_reclaim(sc))
	2040	+ __count_vm_events(item, nr_reclaimed);
	2041	+ __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
	2042	+ __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed);
1973	2043	spin_unlock_irq(&pgdat->lru_lock);
1974	2044
1975	2045	mem_cgroup_uncharge_list(&page_list);
..	..	@@ -2003,73 +2073,6 @@
2003	2073	return nr_reclaimed;
2004	2074	}
2005	2075
2006		-/*
2007		- * This moves pages from the active list to the inactive list.
2008		- *
2009		- * We move them the other way if the page is referenced by one or more
2010		- * processes, from rmap.
2011		- *
2012		- * If the pages are mostly unmapped, the processing is fast and it is
2013		- * appropriate to hold zone_lru_lock across the whole operation. But if
2014		- * the pages are mapped, the processing is slow (page_referenced()) so we
2015		- * should drop zone_lru_lock around each page. It's impossible to balance
2016		- * this, so instead we remove the pages from the LRU while processing them.
2017		- * It is safe to rely on PG_active against the non-LRU pages in here because
2018		- * nobody will play with that bit on a non-LRU page.
2019		- *
2020		- * The downside is that we have to touch page->_refcount against each page.
2021		- * But we had to alter page->flags anyway.
2022		- *
2023		- * Returns the number of pages moved to the given lru.
2024		- */
2025		-
2026		-static unsigned move_active_pages_to_lru(struct lruvec *lruvec,
2027		- struct list_head *list,
2028		- struct list_head *pages_to_free,
2029		- enum lru_list lru)
2030		-{
2031		- struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2032		- struct page *page;
2033		- int nr_pages;
2034		- int nr_moved = 0;
2035		-
2036		- while (!list_empty(list)) {
2037		- page = lru_to_page(list);
2038		- lruvec = mem_cgroup_page_lruvec(page, pgdat);
2039		-
2040		- VM_BUG_ON_PAGE(PageLRU(page), page);
2041		- SetPageLRU(page);
2042		-
2043		- nr_pages = hpage_nr_pages(page);
2044		- update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);
2045		- list_move(&page->lru, &lruvec->lists[lru]);
2046		-
2047		- if (put_page_testzero(page)) {
2048		- __ClearPageLRU(page);
2049		- __ClearPageActive(page);
2050		- del_page_from_lru_list(page, lruvec, lru);
2051		-
2052		- if (unlikely(PageCompound(page))) {
2053		- spin_unlock_irq(&pgdat->lru_lock);
2054		- mem_cgroup_uncharge(page);
2055		- (*get_compound_page_dtor(page))(page);
2056		- spin_lock_irq(&pgdat->lru_lock);
2057		- } else
2058		- list_add(&page->lru, pages_to_free);
2059		- } else {
2060		- nr_moved += nr_pages;
2061		- }
2062		- }
2063		-
2064		- if (!is_active_lru(lru)) {
2065		- __count_vm_events(PGDEACTIVATE, nr_moved);
2066		- count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
2067		- nr_moved);
2068		- }
2069		-
2070		- return nr_moved;
2071		-}
2072		-
2073	2076	static void shrink_active_list(unsigned long nr_to_scan,
2074	2077	struct lruvec *lruvec,
2075	2078	struct scan_control *sc,
..	..	@@ -2082,28 +2085,25 @@
2082	2085	LIST_HEAD(l_active);
2083	2086	LIST_HEAD(l_inactive);
2084	2087	struct page *page;
2085		- struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2086	2088	unsigned nr_deactivate, nr_activate;
2087	2089	unsigned nr_rotated = 0;
2088		- isolate_mode_t isolate_mode = 0;
2089	2090	int file = is_file_lru(lru);
2090	2091	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
	2092	+ bool bypass = false;
	2093	+ bool should_protect = false;
2091	2094
2092	2095	lru_add_drain();
2093		-
2094		- if (!sc->may_unmap)
2095		- isolate_mode \|= ISOLATE_UNMAPPED;
2096	2096
2097	2097	spin_lock_irq(&pgdat->lru_lock);
2098	2098
2099	2099	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
2100		- &nr_scanned, sc, isolate_mode, lru);
	2100	+ &nr_scanned, sc, lru);
2101	2101
2102	2102	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2103		- reclaim_stat->recent_scanned[file] += nr_taken;
2104	2103
2105		- __count_vm_events(PGREFILL, nr_scanned);
2106		- count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
	2104	+ if (!cgroup_reclaim(sc))
	2105	+ __count_vm_events(PGREFILL, nr_scanned);
	2106	+ __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2107	2107
2108	2108	spin_unlock_irq(&pgdat->lru_lock);
2109	2109
..	..	@@ -2125,9 +2125,20 @@
2125	2125	}
2126	2126	}
2127	2127
	2128	+ trace_android_vh_page_should_be_protected(page, &should_protect);
	2129	+ if (unlikely(should_protect)) {
	2130	+ nr_rotated += thp_nr_pages(page);
	2131	+ list_add(&page->lru, &l_active);
	2132	+ continue;
	2133	+ }
	2134	+
	2135	+ trace_android_vh_page_referenced_check_bypass(page, nr_to_scan, lru, &bypass);
	2136	+ if (bypass)
	2137	+ goto skip_page_referenced;
	2138	+ trace_android_vh_page_trylock_set(page);
	2139	+ /* Referenced or rmap lock contention: rotate */
2128	2140	if (page_referenced(page, 0, sc->target_mem_cgroup,
2129		- &vm_flags)) {
2130		- nr_rotated += hpage_nr_pages(page);
	2141	+ &vm_flags) != 0) {
2131	2142	/*
2132	2143	* Identify referenced, file-backed active pages and
2133	2144	* give them one more trip around the active list. So
..	..	@@ -2137,12 +2148,15 @@
2137	2148	* IO, plus JVM can create lots of anon VM_EXEC pages,
2138	2149	* so we ignore them here.
2139	2150	*/
2140		- if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
	2151	+ if ((vm_flags & VM_EXEC) && page_is_file_lru(page)) {
	2152	+ trace_android_vh_page_trylock_clear(page);
	2153	+ nr_rotated += thp_nr_pages(page);
2141	2154	list_add(&page->lru, &l_active);
2142	2155	continue;
2143	2156	}
2144	2157	}
2145		-
	2158	+ trace_android_vh_page_trylock_clear(page);
	2159	+skip_page_referenced:
2146	2160	ClearPageActive(page); /* we are de-activating */
2147	2161	SetPageWorkingset(page);
2148	2162	list_add(&page->lru, &l_inactive);
..	..	@@ -2152,23 +2166,91 @@
2152	2166	* Move pages back to the lru list.
2153	2167	*/
2154	2168	spin_lock_irq(&pgdat->lru_lock);
2155		- /*
2156		- * Count referenced pages from currently used mappings as rotated,
2157		- * even though only some of them are actually re-activated. This
2158		- * helps balance scan pressure between file and anonymous pages in
2159		- * get_scan_count.
2160		- */
2161		- reclaim_stat->recent_rotated[file] += nr_rotated;
2162	2169
2163		- nr_activate = move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
2164		- nr_deactivate = move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
	2170	+ nr_activate = move_pages_to_lru(lruvec, &l_active);
	2171	+ nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);
	2172	+ /* Keep all free pages in l_active list */
	2173	+ list_splice(&l_inactive, &l_active);
	2174	+
	2175	+ __count_vm_events(PGDEACTIVATE, nr_deactivate);
	2176	+ __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
	2177	+
2165	2178	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2166	2179	spin_unlock_irq(&pgdat->lru_lock);
2167	2180
2168		- mem_cgroup_uncharge_list(&l_hold);
2169		- free_unref_page_list(&l_hold);
	2181	+ mem_cgroup_uncharge_list(&l_active);
	2182	+ free_unref_page_list(&l_active);
2170	2183	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2171	2184	nr_deactivate, nr_rotated, sc->priority, file);
	2185	+}
	2186	+
	2187	+unsigned long reclaim_pages(struct list_head *page_list)
	2188	+{
	2189	+ int nid = NUMA_NO_NODE;
	2190	+ unsigned int nr_reclaimed = 0;
	2191	+ LIST_HEAD(node_page_list);
	2192	+ struct reclaim_stat dummy_stat;
	2193	+ struct page *page;
	2194	+ struct scan_control sc = {
	2195	+ .gfp_mask = GFP_KERNEL,
	2196	+ .priority = DEF_PRIORITY,
	2197	+ .may_writepage = 1,
	2198	+ .may_unmap = 1,
	2199	+ .may_swap = 1,
	2200	+ };
	2201	+
	2202	+ while (!list_empty(page_list)) {
	2203	+ page = lru_to_page(page_list);
	2204	+ if (nid == NUMA_NO_NODE) {
	2205	+ nid = page_to_nid(page);
	2206	+ INIT_LIST_HEAD(&node_page_list);
	2207	+ }
	2208	+
	2209	+ if (nid == page_to_nid(page)) {
	2210	+ ClearPageActive(page);
	2211	+ list_move(&page->lru, &node_page_list);
	2212	+ continue;
	2213	+ }
	2214	+
	2215	+ nr_reclaimed += shrink_page_list(&node_page_list,
	2216	+ NODE_DATA(nid),
	2217	+ &sc, &dummy_stat, false);
	2218	+ while (!list_empty(&node_page_list)) {
	2219	+ page = lru_to_page(&node_page_list);
	2220	+ list_del(&page->lru);
	2221	+ putback_lru_page(page);
	2222	+ }
	2223	+
	2224	+ nid = NUMA_NO_NODE;
	2225	+ }
	2226	+
	2227	+ if (!list_empty(&node_page_list)) {
	2228	+ nr_reclaimed += shrink_page_list(&node_page_list,
	2229	+ NODE_DATA(nid),
	2230	+ &sc, &dummy_stat, false);
	2231	+ while (!list_empty(&node_page_list)) {
	2232	+ page = lru_to_page(&node_page_list);
	2233	+ list_del(&page->lru);
	2234	+ putback_lru_page(page);
	2235	+ }
	2236	+ }
	2237	+
	2238	+ return nr_reclaimed;
	2239	+}
	2240	+EXPORT_SYMBOL_GPL(reclaim_pages);
	2241	+
	2242	+static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
	2243	+ struct lruvec lruvec, struct scan_control sc)
	2244	+{
	2245	+ if (is_active_lru(lru)) {
	2246	+ if (sc->may_deactivate & (1 << is_file_lru(lru)))
	2247	+ shrink_active_list(nr_to_scan, lruvec, sc, lru);
	2248	+ else
	2249	+ sc->skipped_deactivate = 1;
	2250	+ return 0;
	2251	+ }
	2252	+
	2253	+ return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2172	2254	}
2173	2255
2174	2256	/*
..	..	@@ -2199,62 +2281,31 @@
2199	2281	* 1TB 101 10GB
2200	2282	* 10TB 320 32GB
2201	2283	*/
2202		-static bool inactive_list_is_low(struct lruvec *lruvec, bool file,
2203		- struct scan_control *sc, bool trace)
	2284	+static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2204	2285	{
2205		- enum lru_list active_lru = file * LRU_FILE + LRU_ACTIVE;
2206		- struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2207		- enum lru_list inactive_lru = file * LRU_FILE;
	2286	+ enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2208	2287	unsigned long inactive, active;
2209	2288	unsigned long inactive_ratio;
2210		- unsigned long refaults;
2211	2289	unsigned long gb;
	2290	+ bool skip = false;
2212	2291
2213		- /*
2214		- * If we don't have swap space, anonymous page deactivation
2215		- * is pointless.
2216		- */
2217		- if (!file && !total_swap_pages)
2218		- return false;
	2292	+ inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
	2293	+ active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2219	2294
2220		- inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);
2221		- active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx);
	2295	+ gb = (inactive + active) >> (30 - PAGE_SHIFT);
	2296	+ trace_android_vh_inactive_is_low(gb, &inactive_ratio, inactive_lru, &skip);
	2297	+ if (skip)
	2298	+ goto out;
2222	2299
2223		- /*
2224		- * When refaults are being observed, it means a new workingset
2225		- * is being established. Disable active list protection to get
2226		- * rid of the stale workingset quickly.
2227		- */
2228		- refaults = lruvec_page_state(lruvec, WORKINGSET_ACTIVATE);
2229		- if (file && lruvec->refaults != refaults) {
2230		- inactive_ratio = 0;
2231		- } else {
2232		- gb = (inactive + active) >> (30 - PAGE_SHIFT);
2233		- if (gb)
2234		- inactive_ratio = int_sqrt(10 * gb);
2235		- else
2236		- inactive_ratio = 1;
2237		- }
	2300	+ if (gb)
	2301	+ inactive_ratio = int_sqrt(10 * gb);
	2302	+ else
	2303	+ inactive_ratio = 1;
2238	2304
2239		- if (trace)
2240		- trace_mm_vmscan_inactive_list_is_low(pgdat->node_id, sc->reclaim_idx,
2241		- lruvec_lru_size(lruvec, inactive_lru, MAX_NR_ZONES), inactive,
2242		- lruvec_lru_size(lruvec, active_lru, MAX_NR_ZONES), active,
2243		- inactive_ratio, file);
	2305	+ trace_android_vh_tune_inactive_ratio(&inactive_ratio, is_file_lru(inactive_lru));
2244	2306
	2307	+out:
2245	2308	return inactive * inactive_ratio < active;
2246		-}
2247		-
2248		-static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2249		- struct lruvec lruvec, struct scan_control sc)
2250		-{
2251		- if (is_active_lru(lru)) {
2252		- if (inactive_list_is_low(lruvec, is_file_lru(lru), sc, true))
2253		- shrink_active_list(nr_to_scan, lruvec, sc, lru);
2254		- return 0;
2255		- }
2256		-
2257		- return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2258	2309	}
2259	2310
2260	2311	enum scan_balance {
..	..	@@ -2273,20 +2324,18 @@
2273	2324	* nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2274	2325	* nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2275	2326	*/
2276		-static void get_scan_count(struct lruvec lruvec, struct mem_cgroup memcg,
2277		- struct scan_control sc, unsigned long nr,
2278		- unsigned long *lru_pages)
	2327	+static void get_scan_count(struct lruvec lruvec, struct scan_control sc,
	2328	+ unsigned long *nr)
2279	2329	{
	2330	+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
	2331	+ unsigned long anon_cost, file_cost, total_cost;
2280	2332	int swappiness = mem_cgroup_swappiness(memcg);
2281		- struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2282		- u64 fraction[2];
	2333	+ u64 fraction[ANON_AND_FILE];
2283	2334	u64 denominator = 0; /* gcc */
2284		- struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2285		- unsigned long anon_prio, file_prio;
2286	2335	enum scan_balance scan_balance;
2287		- unsigned long anon, file;
2288	2336	unsigned long ap, fp;
2289	2337	enum lru_list lru;
	2338	+ bool balance_anon_file_reclaim = false;
2290	2339
2291	2340	/* If we have no swap space, do not bother scanning anon pages. */
2292	2341	if (!sc->may_swap \|\| mem_cgroup_get_nr_swap_pages(memcg) <= 0) {
..	..	@@ -2294,6 +2343,7 @@
2294	2343	goto out;
2295	2344	}
2296	2345
	2346	+ trace_android_vh_tune_swappiness(&swappiness);
2297	2347	/*
2298	2348	* Global reclaim will swap to prevent OOM even with no
2299	2349	* swappiness, but memcg users want to use this knob to
..	..	@@ -2301,7 +2351,7 @@
2301	2351	* using the memory controller's swap limit feature would be
2302	2352	* too expensive.
2303	2353	*/
2304		- if (!global_reclaim(sc) && !swappiness) {
	2354	+ if (cgroup_reclaim(sc) && !swappiness) {
2305	2355	scan_balance = SCAN_FILE;
2306	2356	goto out;
2307	2357	}
..	..	@@ -2317,129 +2367,133 @@
2317	2367	}
2318	2368
2319	2369	/*
2320		- * Prevent the reclaimer from falling into the cache trap: as
2321		- * cache pages start out inactive, every cache fault will tip
2322		- * the scan balance towards the file LRU. And as the file LRU
2323		- * shrinks, so does the window for rotation from references.
2324		- * This means we have a runaway feedback loop where a tiny
2325		- * thrashing file LRU becomes infinitely more attractive than
2326		- * anon pages. Try to detect this based on file LRU size.
	2370	+ * If the system is almost out of file pages, force-scan anon.
2327	2371	*/
2328		- if (global_reclaim(sc)) {
2329		- unsigned long pgdatfile;
2330		- unsigned long pgdatfree;
2331		- int z;
2332		- unsigned long total_high_wmark = 0;
2333		-
2334		- pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2335		- pgdatfile = node_page_state(pgdat, NR_ACTIVE_FILE) +
2336		- node_page_state(pgdat, NR_INACTIVE_FILE);
2337		-
2338		- for (z = 0; z < MAX_NR_ZONES; z++) {
2339		- struct zone *zone = &pgdat->node_zones[z];
2340		- if (!managed_zone(zone))
2341		- continue;
2342		-
2343		- total_high_wmark += high_wmark_pages(zone);
2344		- }
2345		-
2346		- if (unlikely(pgdatfile + pgdatfree <= total_high_wmark)) {
2347		- /*
2348		- * Force SCAN_ANON if there are enough inactive
2349		- * anonymous pages on the LRU in eligible zones.
2350		- * Otherwise, the small LRU gets thrashed.
2351		- */
2352		- if (!inactive_list_is_low(lruvec, false, sc, false) &&
2353		- lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, sc->reclaim_idx)
2354		- >> sc->priority) {
2355		- scan_balance = SCAN_ANON;
2356		- goto out;
2357		- }
2358		- }
	2372	+ if (sc->file_is_tiny) {
	2373	+ scan_balance = SCAN_ANON;
	2374	+ goto out;
2359	2375	}
2360	2376
	2377	+ trace_android_rvh_set_balance_anon_file_reclaim(&balance_anon_file_reclaim);
	2378	+
2361	2379	/*
2362		- * If there is enough inactive page cache, i.e. if the size of the
2363		- * inactive list is greater than that of the active list and the
2364		- * inactive list actually has some pages to scan on this priority, we
2365		- * do not reclaim anything from the anonymous working set right now.
2366		- * Without the second condition we could end up never scanning an
2367		- * lruvec even if it has plenty of old anonymous pages unless the
2368		- * system is under heavy pressure.
	2380	+ * If there is enough inactive page cache, we do not reclaim
	2381	+ * anything from the anonymous working right now. But when balancing
	2382	+ * anon and page cache files for reclaim, allow swapping of anon pages
	2383	+ * even if there are a number of inactive file cache pages.
2369	2384	*/
2370		- if (!inactive_list_is_low(lruvec, true, sc, false) &&
2371		- lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) {
	2385	+ if (!balance_anon_file_reclaim && sc->cache_trim_mode) {
2372	2386	scan_balance = SCAN_FILE;
2373	2387	goto out;
2374	2388	}
2375	2389
2376	2390	scan_balance = SCAN_FRACT;
2377		-
2378	2391	/*
2379		- * With swappiness at 100, anonymous and file have the same priority.
2380		- * This scanning priority is essentially the inverse of IO cost.
2381		- */
2382		- anon_prio = swappiness;
2383		- file_prio = 200 - anon_prio;
2384		-
2385		- /*
2386		- * OK, so we have swap space and a fair amount of page cache
2387		- * pages. We use the recently rotated / recently scanned
2388		- * ratios to determine how valuable each cache is.
	2392	+ * Calculate the pressure balance between anon and file pages.
2389	2393	*
2390		- * Because workloads change over time (and to avoid overflow)
2391		- * we keep these statistics as a floating average, which ends
2392		- * up weighing recent references more than old ones.
	2394	+ * The amount of pressure we put on each LRU is inversely
	2395	+ * proportional to the cost of reclaiming each list, as
	2396	+ * determined by the share of pages that are refaulting, times
	2397	+ * the relative IO cost of bringing back a swapped out
	2398	+ * anonymous page vs reloading a filesystem page (swappiness).
2393	2399	*
2394		- * anon in [0], file in [1]
	2400	+ * Although we limit that influence to ensure no list gets
	2401	+ * left behind completely: at least a third of the pressure is
	2402	+ * applied, before swappiness.
	2403	+ *
	2404	+ * With swappiness at 100, anon and file have equal IO cost.
2395	2405	*/
	2406	+ total_cost = sc->anon_cost + sc->file_cost;
	2407	+ anon_cost = total_cost + sc->anon_cost;
	2408	+ file_cost = total_cost + sc->file_cost;
	2409	+ total_cost = anon_cost + file_cost;
2396	2410
2397		- anon = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +
2398		- lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, MAX_NR_ZONES);
2399		- file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) +
2400		- lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);
	2411	+ ap = swappiness * (total_cost + 1);
	2412	+ ap /= anon_cost + 1;
2401	2413
2402		- spin_lock_irq(&pgdat->lru_lock);
2403		- if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
2404		- reclaim_stat->recent_scanned[0] /= 2;
2405		- reclaim_stat->recent_rotated[0] /= 2;
2406		- }
2407		-
2408		- if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
2409		- reclaim_stat->recent_scanned[1] /= 2;
2410		- reclaim_stat->recent_rotated[1] /= 2;
2411		- }
2412		-
2413		- /*
2414		- * The amount of pressure on anon vs file pages is inversely
2415		- * proportional to the fraction of recently scanned pages on
2416		- * each list that were recently referenced and in active use.
2417		- */
2418		- ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
2419		- ap /= reclaim_stat->recent_rotated[0] + 1;
2420		-
2421		- fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
2422		- fp /= reclaim_stat->recent_rotated[1] + 1;
2423		- spin_unlock_irq(&pgdat->lru_lock);
	2414	+ fp = (200 - swappiness) * (total_cost + 1);
	2415	+ fp /= file_cost + 1;
2424	2416
2425	2417	fraction[0] = ap;
2426	2418	fraction[1] = fp;
2427		- denominator = ap + fp + 1;
	2419	+ denominator = ap + fp;
2428	2420	out:
2429		- *lru_pages = 0;
	2421	+ trace_android_vh_tune_scan_type((char *)(&scan_balance));
2430	2422	for_each_evictable_lru(lru) {
2431	2423	int file = is_file_lru(lru);
2432		- unsigned long size;
	2424	+ unsigned long lruvec_size;
	2425	+ unsigned long low, min;
2433	2426	unsigned long scan;
2434	2427
2435		- size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2436		- scan = size >> sc->priority;
	2428	+ lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
	2429	+ mem_cgroup_protection(sc->target_mem_cgroup, memcg,
	2430	+ &min, &low);
	2431	+
	2432	+ if (min \|\| low) {
	2433	+ /*
	2434	+ * Scale a cgroup's reclaim pressure by proportioning
	2435	+ * its current usage to its memory.low or memory.min
	2436	+ * setting.
	2437	+ *
	2438	+ * This is important, as otherwise scanning aggression
	2439	+ * becomes extremely binary -- from nothing as we
	2440	+ * approach the memory protection threshold, to totally
	2441	+ * nominal as we exceed it. This results in requiring
	2442	+ * setting extremely liberal protection thresholds. It
	2443	+ * also means we simply get no protection at all if we
	2444	+ * set it too low, which is not ideal.
	2445	+ *
	2446	+ * If there is any protection in place, we reduce scan
	2447	+ * pressure by how much of the total memory used is
	2448	+ * within protection thresholds.
	2449	+ *
	2450	+ * There is one special case: in the first reclaim pass,
	2451	+ * we skip over all groups that are within their low
	2452	+ * protection. If that fails to reclaim enough pages to
	2453	+ * satisfy the reclaim goal, we come back and override
	2454	+ * the best-effort low protection. However, we still
	2455	+ * ideally want to honor how well-behaved groups are in
	2456	+ * that case instead of simply punishing them all
	2457	+ * equally. As such, we reclaim them based on how much
	2458	+ * memory they are using, reducing the scan pressure
	2459	+ * again by how much of the total memory used is under
	2460	+ * hard protection.
	2461	+ */
	2462	+ unsigned long cgroup_size = mem_cgroup_size(memcg);
	2463	+ unsigned long protection;
	2464	+
	2465	+ /* memory.low scaling, make sure we retry before OOM */
	2466	+ if (!sc->memcg_low_reclaim && low > min) {
	2467	+ protection = low;
	2468	+ sc->memcg_low_skipped = 1;
	2469	+ } else {
	2470	+ protection = min;
	2471	+ }
	2472	+
	2473	+ /* Avoid TOCTOU with earlier protection check */
	2474	+ cgroup_size = max(cgroup_size, protection);
	2475	+
	2476	+ scan = lruvec_size - lruvec_size * protection /
	2477	+ (cgroup_size + 1);
	2478	+
	2479	+ /*
	2480	+ * Minimally target SWAP_CLUSTER_MAX pages to keep
	2481	+ * reclaim moving forwards, avoiding decrementing
	2482	+ * sc->priority further than desirable.
	2483	+ */
	2484	+ scan = max(scan, SWAP_CLUSTER_MAX);
	2485	+ } else {
	2486	+ scan = lruvec_size;
	2487	+ }
	2488	+
	2489	+ scan >>= sc->priority;
	2490	+
2437	2491	/*
2438	2492	* If the cgroup's already been deleted, make sure to
2439	2493	* scrape out the remaining cache.
2440	2494	*/
2441	2495	if (!scan && !mem_cgroup_online(memcg))
2442		- scan = min(size, SWAP_CLUSTER_MAX);
	2496	+ scan = min(lruvec_size, SWAP_CLUSTER_MAX);
2443	2497
2444	2498	switch (scan_balance) {
2445	2499	case SCAN_EQUAL:
..	..	@@ -2461,38 +2515,30 @@
2461	2515	case SCAN_FILE:
2462	2516	case SCAN_ANON:
2463	2517	/* Scan one type exclusively */
2464		- if ((scan_balance == SCAN_FILE) != file) {
2465		- size = 0;
	2518	+ if ((scan_balance == SCAN_FILE) != file)
2466	2519	scan = 0;
2467		- }
2468	2520	break;
2469	2521	default:
2470	2522	/* Look ma, no brain */
2471	2523	BUG();
2472	2524	}
2473	2525
2474		- *lru_pages += size;
2475	2526	nr[lru] = scan;
2476	2527	}
2477	2528	}
2478	2529
2479		-/*
2480		- * This is a basic per-node page freer. Used by both kswapd and direct reclaim.
2481		- */
2482		-static void shrink_node_memcg(struct pglist_data pgdat, struct mem_cgroup memcg,
2483		- struct scan_control sc, unsigned long lru_pages)
	2530	+static void shrink_lruvec(struct lruvec lruvec, struct scan_control sc)
2484	2531	{
2485		- struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
2486	2532	unsigned long nr[NR_LRU_LISTS];
2487	2533	unsigned long targets[NR_LRU_LISTS];
2488	2534	unsigned long nr_to_scan;
2489	2535	enum lru_list lru;
2490	2536	unsigned long nr_reclaimed = 0;
2491	2537	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
	2538	+ bool proportional_reclaim;
2492	2539	struct blk_plug plug;
2493		- bool scan_adjusted;
2494	2540
2495		- get_scan_count(lruvec, memcg, sc, nr, lru_pages);
	2541	+ get_scan_count(lruvec, sc, nr);
2496	2542
2497	2543	/* Record the original scan target for proportional adjustments later */
2498	2544	memcpy(targets, nr, sizeof(nr));
..	..	@@ -2508,8 +2554,8 @@
2508	2554	* abort proportional reclaim if either the file or anon lru has already
2509	2555	* dropped to zero at the first pass.
2510	2556	*/
2511		- scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
2512		- sc->priority == DEF_PRIORITY);
	2557	+ proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
	2558	+ sc->priority == DEF_PRIORITY);
2513	2559
2514	2560	blk_start_plug(&plug);
2515	2561	while (nr[LRU_INACTIVE_ANON] \|\| nr[LRU_ACTIVE_FILE] \|\|
..	..	@@ -2529,7 +2575,7 @@
2529	2575
2530	2576	cond_resched();
2531	2577
2532		- if (nr_reclaimed < nr_to_reclaim \|\| scan_adjusted)
	2578	+ if (nr_reclaimed < nr_to_reclaim \|\| proportional_reclaim)
2533	2579	continue;
2534	2580
2535	2581	/*
..	..	@@ -2580,8 +2626,6 @@
2580	2626	nr_scanned = targets[lru] - nr[lru];
2581	2627	nr[lru] = targets[lru] * (100 - percentage) / 100;
2582	2628	nr[lru] -= min(nr[lru], nr_scanned);
2583		-
2584		- scan_adjusted = true;
2585	2629	}
2586	2630	blk_finish_plug(&plug);
2587	2631	sc->nr_reclaimed += nr_reclaimed;
..	..	@@ -2590,7 +2634,7 @@
2590	2634	* Even if we did not try to evict anon pages at all, we want to
2591	2635	* rebalance the anon lru active/inactive ratio.
2592	2636	*/
2593		- if (inactive_list_is_low(lruvec, false, sc, true))
	2637	+ if (total_swap_pages && inactive_is_low(lruvec, LRU_INACTIVE_ANON))
2594	2638	shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2595	2639	sc, LRU_ACTIVE_ANON);
2596	2640	}
..	..	@@ -2610,12 +2654,11 @@
2610	2654	* Reclaim/compaction is used for high-order allocation requests. It reclaims
2611	2655	* order-0 pages before compacting the zone. should_continue_reclaim() returns
2612	2656	* true if more pages should be reclaimed such that when the page allocator
2613		- * calls try_to_compact_zone() that it will have enough free pages to succeed.
	2657	+ * calls try_to_compact_pages() that it will have enough free pages to succeed.
2614	2658	* It will give up earlier than that if there is difficulty reclaiming pages.
2615	2659	*/
2616	2660	static inline bool should_continue_reclaim(struct pglist_data *pgdat,
2617	2661	unsigned long nr_reclaimed,
2618		- unsigned long nr_scanned,
2619	2662	struct scan_control *sc)
2620	2663	{
2621	2664	unsigned long pages_for_compaction;
..	..	@@ -2626,40 +2669,18 @@
2626	2669	if (!in_reclaim_compaction(sc))
2627	2670	return false;
2628	2671
2629		- /* Consider stopping depending on scan and reclaim activity */
2630		- if (sc->gfp_mask & __GFP_RETRY_MAYFAIL) {
2631		- /*
2632		- * For __GFP_RETRY_MAYFAIL allocations, stop reclaiming if the
2633		- * full LRU list has been scanned and we are still failing
2634		- * to reclaim pages. This full LRU scan is potentially
2635		- * expensive but a __GFP_RETRY_MAYFAIL caller really wants to succeed
2636		- */
2637		- if (!nr_reclaimed && !nr_scanned)
2638		- return false;
2639		- } else {
2640		- /*
2641		- * For non-__GFP_RETRY_MAYFAIL allocations which can presumably
2642		- * fail without consequence, stop if we failed to reclaim
2643		- * any pages from the last SWAP_CLUSTER_MAX number of
2644		- * pages that were scanned. This will return to the
2645		- * caller faster at the risk reclaim/compaction and
2646		- * the resulting allocation attempt fails
2647		- */
2648		- if (!nr_reclaimed)
2649		- return false;
2650		- }
2651		-
2652	2672	/*
2653		- * If we have not reclaimed enough pages for compaction and the
2654		- * inactive lists are large enough, continue reclaiming
	2673	+ * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
	2674	+ * number of pages that were scanned. This will return to the caller
	2675	+ * with the risk reclaim/compaction and the resulting allocation attempt
	2676	+ * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
	2677	+ * allocations through requiring that the full LRU list has been scanned
	2678	+ * first, by assuming that zero delta of sc->nr_scanned means full LRU
	2679	+ * scan, but that approximation was wrong, and there were corner cases
	2680	+ * where always a non-zero amount of pages were scanned.
2655	2681	*/
2656		- pages_for_compaction = compact_gap(sc->order);
2657		- inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
2658		- if (get_nr_swap_pages() > 0)
2659		- inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
2660		- if (sc->nr_reclaimed < pages_for_compaction &&
2661		- inactive_lru_pages > pages_for_compaction)
2662		- return true;
	2682	+ if (!nr_reclaimed)
	2683	+ return false;
2663	2684
2664	2685	/* If compaction would go ahead or the allocation would succeed, stop */
2665	2686	for (z = 0; z <= sc->reclaim_idx; z++) {
..	..	@@ -2676,179 +2697,262 @@
2676	2697	;
2677	2698	}
2678	2699	}
2679		- return true;
	2700	+
	2701	+ /*
	2702	+ * If we have not reclaimed enough pages for compaction and the
	2703	+ * inactive lists are large enough, continue reclaiming
	2704	+ */
	2705	+ pages_for_compaction = compact_gap(sc->order);
	2706	+ inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
	2707	+ if (get_nr_swap_pages() > 0)
	2708	+ inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
	2709	+
	2710	+ return inactive_lru_pages > pages_for_compaction;
2680	2711	}
2681	2712
2682		-static bool pgdat_memcg_congested(pg_data_t pgdat, struct mem_cgroup memcg)
	2713	+static void shrink_node_memcgs(pg_data_t pgdat, struct scan_control sc)
2683	2714	{
2684		- return test_bit(PGDAT_CONGESTED, &pgdat->flags) \|\|
2685		- (memcg && memcg_congested(pgdat, memcg));
	2715	+ struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
	2716	+ struct mem_cgroup *memcg;
	2717	+
	2718	+ memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
	2719	+ do {
	2720	+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
	2721	+ unsigned long reclaimed;
	2722	+ unsigned long scanned;
	2723	+ bool skip = false;
	2724	+
	2725	+ /*
	2726	+ * This loop can become CPU-bound when target memcgs
	2727	+ * aren't eligible for reclaim - either because they
	2728	+ * don't have any reclaimable pages, or because their
	2729	+ * memory is explicitly protected. Avoid soft lockups.
	2730	+ */
	2731	+ cond_resched();
	2732	+
	2733	+ trace_android_vh_shrink_node_memcgs(memcg, &skip);
	2734	+ if (skip)
	2735	+ continue;
	2736	+
	2737	+ mem_cgroup_calculate_protection(target_memcg, memcg);
	2738	+
	2739	+ if (mem_cgroup_below_min(memcg)) {
	2740	+ /*
	2741	+ * Hard protection.
	2742	+ * If there is no reclaimable memory, OOM.
	2743	+ */
	2744	+ continue;
	2745	+ } else if (mem_cgroup_below_low(memcg)) {
	2746	+ /*
	2747	+ * Soft protection.
	2748	+ * Respect the protection only as long as
	2749	+ * there is an unprotected supply
	2750	+ * of reclaimable memory from other cgroups.
	2751	+ */
	2752	+ if (!sc->memcg_low_reclaim) {
	2753	+ sc->memcg_low_skipped = 1;
	2754	+ continue;
	2755	+ }
	2756	+ memcg_memory_event(memcg, MEMCG_LOW);
	2757	+ }
	2758	+
	2759	+ reclaimed = sc->nr_reclaimed;
	2760	+ scanned = sc->nr_scanned;
	2761	+
	2762	+ shrink_lruvec(lruvec, sc);
	2763	+
	2764	+ shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
	2765	+ sc->priority);
	2766	+
	2767	+ /* Record the group's reclaim efficiency */
	2768	+ vmpressure(sc->gfp_mask, memcg, false,
	2769	+ sc->nr_scanned - scanned,
	2770	+ sc->nr_reclaimed - reclaimed);
	2771	+
	2772	+ } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
2686	2773	}
2687	2774
2688		-static bool shrink_node(pg_data_t pgdat, struct scan_control sc)
	2775	+static void shrink_node(pg_data_t pgdat, struct scan_control sc)
2689	2776	{
2690	2777	struct reclaim_state *reclaim_state = current->reclaim_state;
2691	2778	unsigned long nr_reclaimed, nr_scanned;
	2779	+ struct lruvec *target_lruvec;
2692	2780	bool reclaimable = false;
	2781	+ unsigned long file;
2693	2782
2694		- do {
2695		- struct mem_cgroup *root = sc->target_mem_cgroup;
2696		- struct mem_cgroup_reclaim_cookie reclaim = {
2697		- .pgdat = pgdat,
2698		- .priority = sc->priority,
2699		- };
2700		- unsigned long node_lru_pages = 0;
2701		- struct mem_cgroup *memcg;
	2783	+ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
2702	2784
2703		- memset(&sc->nr, 0, sizeof(sc->nr));
	2785	+again:
	2786	+ memset(&sc->nr, 0, sizeof(sc->nr));
2704	2787
2705		- nr_reclaimed = sc->nr_reclaimed;
2706		- nr_scanned = sc->nr_scanned;
	2788	+ nr_reclaimed = sc->nr_reclaimed;
	2789	+ nr_scanned = sc->nr_scanned;
2707	2790
2708		- memcg = mem_cgroup_iter(root, NULL, &reclaim);
2709		- do {
2710		- unsigned long lru_pages;
2711		- unsigned long reclaimed;
2712		- unsigned long scanned;
	2791	+ /*
	2792	+ * Determine the scan balance between anon and file LRUs.
	2793	+ */
	2794	+ spin_lock_irq(&pgdat->lru_lock);
	2795	+ sc->anon_cost = target_lruvec->anon_cost;
	2796	+ sc->file_cost = target_lruvec->file_cost;
	2797	+ spin_unlock_irq(&pgdat->lru_lock);
2713	2798
2714		- /*
2715		- * This loop can become CPU-bound when target memcgs
2716		- * aren't eligible for reclaim - either because they
2717		- * don't have any reclaimable pages, or because their
2718		- * memory is explicitly protected. Avoid soft lockups.
2719		- */
2720		- cond_resched();
	2799	+ /*
	2800	+ * Target desirable inactive:active list ratios for the anon
	2801	+ * and file LRU lists.
	2802	+ */
	2803	+ if (!sc->force_deactivate) {
	2804	+ unsigned long refaults;
2721	2805
2722		- switch (mem_cgroup_protected(root, memcg)) {
2723		- case MEMCG_PROT_MIN:
2724		- /*
2725		- * Hard protection.
2726		- * If there is no reclaimable memory, OOM.
2727		- */
	2806	+ refaults = lruvec_page_state(target_lruvec,
	2807	+ WORKINGSET_ACTIVATE_ANON);
	2808	+ if (refaults != target_lruvec->refaults[0] \|\|
	2809	+ inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
	2810	+ sc->may_deactivate \|= DEACTIVATE_ANON;
	2811	+ else
	2812	+ sc->may_deactivate &= ~DEACTIVATE_ANON;
	2813	+
	2814	+ /*
	2815	+ * When refaults are being observed, it means a new
	2816	+ * workingset is being established. Deactivate to get
	2817	+ * rid of any stale active pages quickly.
	2818	+ */
	2819	+ refaults = lruvec_page_state(target_lruvec,
	2820	+ WORKINGSET_ACTIVATE_FILE);
	2821	+ if (refaults != target_lruvec->refaults[1] \|\|
	2822	+ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
	2823	+ sc->may_deactivate \|= DEACTIVATE_FILE;
	2824	+ else
	2825	+ sc->may_deactivate &= ~DEACTIVATE_FILE;
	2826	+ } else
	2827	+ sc->may_deactivate = DEACTIVATE_ANON \| DEACTIVATE_FILE;
	2828	+
	2829	+ /*
	2830	+ * If we have plenty of inactive file pages that aren't
	2831	+ * thrashing, try to reclaim those first before touching
	2832	+ * anonymous pages.
	2833	+ */
	2834	+ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
	2835	+ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
	2836	+ sc->cache_trim_mode = 1;
	2837	+ else
	2838	+ sc->cache_trim_mode = 0;
	2839	+
	2840	+ /*
	2841	+ * Prevent the reclaimer from falling into the cache trap: as
	2842	+ * cache pages start out inactive, every cache fault will tip
	2843	+ * the scan balance towards the file LRU. And as the file LRU
	2844	+ * shrinks, so does the window for rotation from references.
	2845	+ * This means we have a runaway feedback loop where a tiny
	2846	+ * thrashing file LRU becomes infinitely more attractive than
	2847	+ * anon pages. Try to detect this based on file LRU size.
	2848	+ */
	2849	+ if (!cgroup_reclaim(sc)) {
	2850	+ unsigned long total_high_wmark = 0;
	2851	+ unsigned long free, anon;
	2852	+ int z;
	2853	+
	2854	+ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
	2855	+ file = node_page_state(pgdat, NR_ACTIVE_FILE) +
	2856	+ node_page_state(pgdat, NR_INACTIVE_FILE);
	2857	+
	2858	+ for (z = 0; z < MAX_NR_ZONES; z++) {
	2859	+ struct zone *zone = &pgdat->node_zones[z];
	2860	+ if (!managed_zone(zone))
2728	2861	continue;
2729		- case MEMCG_PROT_LOW:
2730		- /*
2731		- * Soft protection.
2732		- * Respect the protection only as long as
2733		- * there is an unprotected supply
2734		- * of reclaimable memory from other cgroups.
2735		- */
2736		- if (!sc->memcg_low_reclaim) {
2737		- sc->memcg_low_skipped = 1;
2738		- continue;
2739		- }
2740		- memcg_memory_event(memcg, MEMCG_LOW);
2741		- break;
2742		- case MEMCG_PROT_NONE:
2743		- break;
2744		- }
2745	2862
2746		- reclaimed = sc->nr_reclaimed;
2747		- scanned = sc->nr_scanned;
2748		- shrink_node_memcg(pgdat, memcg, sc, &lru_pages);
2749		- node_lru_pages += lru_pages;
2750		-
2751		- shrink_slab(sc->gfp_mask, pgdat->node_id,
2752		- memcg, sc->priority);
2753		-
2754		- /* Record the group's reclaim efficiency */
2755		- vmpressure(sc->gfp_mask, memcg, false,
2756		- sc->nr_scanned - scanned,
2757		- sc->nr_reclaimed - reclaimed);
2758		-
2759		- /*
2760		- * Direct reclaim and kswapd have to scan all memory
2761		- * cgroups to fulfill the overall scan target for the
2762		- * node.
2763		- *
2764		- * Limit reclaim, on the other hand, only cares about
2765		- * nr_to_reclaim pages to be reclaimed and it will
2766		- * retry with decreasing priority if one round over the
2767		- * whole hierarchy is not sufficient.
2768		- */
2769		- if (!global_reclaim(sc) &&
2770		- sc->nr_reclaimed >= sc->nr_to_reclaim) {
2771		- mem_cgroup_iter_break(root, memcg);
2772		- break;
2773		- }
2774		- } while ((memcg = mem_cgroup_iter(root, memcg, &reclaim)));
2775		-
2776		- if (reclaim_state) {
2777		- sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2778		- reclaim_state->reclaimed_slab = 0;
2779		- }
2780		-
2781		- /* Record the subtree's reclaim efficiency */
2782		- vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2783		- sc->nr_scanned - nr_scanned,
2784		- sc->nr_reclaimed - nr_reclaimed);
2785		-
2786		- if (sc->nr_reclaimed - nr_reclaimed)
2787		- reclaimable = true;
2788		-
2789		- if (current_is_kswapd()) {
2790		- /*
2791		- * If reclaim is isolating dirty pages under writeback,
2792		- * it implies that the long-lived page allocation rate
2793		- * is exceeding the page laundering rate. Either the
2794		- * global limits are not being effective at throttling
2795		- * processes due to the page distribution throughout
2796		- * zones or there is heavy usage of a slow backing
2797		- * device. The only option is to throttle from reclaim
2798		- * context which is not ideal as there is no guarantee
2799		- * the dirtying process is throttled in the same way
2800		- * balance_dirty_pages() manages.
2801		- *
2802		- * Once a node is flagged PGDAT_WRITEBACK, kswapd will
2803		- * count the number of pages under pages flagged for
2804		- * immediate reclaim and stall if any are encountered
2805		- * in the nr_immediate check below.
2806		- */
2807		- if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
2808		- set_bit(PGDAT_WRITEBACK, &pgdat->flags);
2809		-
2810		- /*
2811		- * Tag a node as congested if all the dirty pages
2812		- * scanned were backed by a congested BDI and
2813		- * wait_iff_congested will stall.
2814		- */
2815		- if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2816		- set_bit(PGDAT_CONGESTED, &pgdat->flags);
2817		-
2818		- /* Allow kswapd to start writing pages during reclaim.*/
2819		- if (sc->nr.unqueued_dirty == sc->nr.file_taken)
2820		- set_bit(PGDAT_DIRTY, &pgdat->flags);
2821		-
2822		- /*
2823		- * If kswapd scans pages marked marked for immediate
2824		- * reclaim and under writeback (nr_immediate), it
2825		- * implies that pages are cycling through the LRU
2826		- * faster than they are written so also forcibly stall.
2827		- */
2828		- if (sc->nr.immediate)
2829		- congestion_wait(BLK_RW_ASYNC, HZ/10);
	2863	+ total_high_wmark += high_wmark_pages(zone);
2830	2864	}
2831	2865
2832	2866	/*
2833		- * Legacy memcg will stall in page writeback so avoid forcibly
2834		- * stalling in wait_iff_congested().
	2867	+ * Consider anon: if that's low too, this isn't a
	2868	+ * runaway file reclaim problem, but rather just
	2869	+ * extreme pressure. Reclaim as per usual then.
2835	2870	*/
2836		- if (!global_reclaim(sc) && sane_reclaim(sc) &&
2837		- sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2838		- set_memcg_congestion(pgdat, root, true);
	2871	+ anon = node_page_state(pgdat, NR_INACTIVE_ANON);
	2872	+
	2873	+ sc->file_is_tiny =
	2874	+ file + free <= total_high_wmark &&
	2875	+ !(sc->may_deactivate & DEACTIVATE_ANON) &&
	2876	+ anon >> sc->priority;
	2877	+ }
	2878	+
	2879	+ shrink_node_memcgs(pgdat, sc);
	2880	+
	2881	+ if (reclaim_state) {
	2882	+ sc->nr_reclaimed += reclaim_state->reclaimed_slab;
	2883	+ reclaim_state->reclaimed_slab = 0;
	2884	+ }
	2885	+
	2886	+ /* Record the subtree's reclaim efficiency */
	2887	+ vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
	2888	+ sc->nr_scanned - nr_scanned,
	2889	+ sc->nr_reclaimed - nr_reclaimed);
	2890	+
	2891	+ if (sc->nr_reclaimed - nr_reclaimed)
	2892	+ reclaimable = true;
	2893	+
	2894	+ if (current_is_kswapd()) {
	2895	+ /*
	2896	+ * If reclaim is isolating dirty pages under writeback,
	2897	+ * it implies that the long-lived page allocation rate
	2898	+ * is exceeding the page laundering rate. Either the
	2899	+ * global limits are not being effective at throttling
	2900	+ * processes due to the page distribution throughout
	2901	+ * zones or there is heavy usage of a slow backing
	2902	+ * device. The only option is to throttle from reclaim
	2903	+ * context which is not ideal as there is no guarantee
	2904	+ * the dirtying process is throttled in the same way
	2905	+ * balance_dirty_pages() manages.
	2906	+ *
	2907	+ * Once a node is flagged PGDAT_WRITEBACK, kswapd will
	2908	+ * count the number of pages under pages flagged for
	2909	+ * immediate reclaim and stall if any are encountered
	2910	+ * in the nr_immediate check below.
	2911	+ */
	2912	+ if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
	2913	+ set_bit(PGDAT_WRITEBACK, &pgdat->flags);
	2914	+
	2915	+ /* Allow kswapd to start writing pages during reclaim.*/
	2916	+ if (sc->nr.unqueued_dirty == sc->nr.file_taken)
	2917	+ set_bit(PGDAT_DIRTY, &pgdat->flags);
2839	2918
2840	2919	/*
2841		- * Stall direct reclaim for IO completions if underlying BDIs
2842		- * and node is congested. Allow kswapd to continue until it
2843		- * starts encountering unqueued dirty pages or cycling through
2844		- * the LRU too quickly.
	2920	+ * If kswapd scans pages marked for immediate
	2921	+ * reclaim and under writeback (nr_immediate), it
	2922	+ * implies that pages are cycling through the LRU
	2923	+ * faster than they are written so also forcibly stall.
2845	2924	*/
2846		- if (!sc->hibernation_mode && !current_is_kswapd() &&
2847		- current_may_throttle() && pgdat_memcg_congested(pgdat, root))
2848		- wait_iff_congested(BLK_RW_ASYNC, HZ/10);
	2925	+ if (sc->nr.immediate)
	2926	+ congestion_wait(BLK_RW_ASYNC, HZ/10);
	2927	+ }
2849	2928
2850		- } while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
2851		- sc->nr_scanned - nr_scanned, sc));
	2929	+ /*
	2930	+ * Tag a node/memcg as congested if all the dirty pages
	2931	+ * scanned were backed by a congested BDI and
	2932	+ * wait_iff_congested will stall.
	2933	+ *
	2934	+ * Legacy memcg will stall in page writeback so avoid forcibly
	2935	+ * stalling in wait_iff_congested().
	2936	+ */
	2937	+ if ((current_is_kswapd() \|\|
	2938	+ (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
	2939	+ sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
	2940	+ set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
	2941	+
	2942	+ /*
	2943	+ * Stall direct reclaim for IO completions if underlying BDIs
	2944	+ * and node is congested. Allow kswapd to continue until it
	2945	+ * starts encountering unqueued dirty pages or cycling through
	2946	+ * the LRU too quickly.
	2947	+ */
	2948	+ if (!current_is_kswapd() && current_may_throttle() &&
	2949	+ !sc->hibernation_mode &&
	2950	+ test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
	2951	+ wait_iff_congested(BLK_RW_ASYNC, HZ/10);
	2952	+
	2953	+ if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
	2954	+ sc))
	2955	+ goto again;
2852	2956
2853	2957	/*
2854	2958	* Kswapd gives up on balancing particular nodes after too
..	..	@@ -2858,8 +2962,6 @@
2858	2962	*/
2859	2963	if (reclaimable)
2860	2964	pgdat->kswapd_failures = 0;
2861		-
2862		- return reclaimable;
2863	2965	}
2864	2966
2865	2967	/*
..	..	@@ -2928,7 +3030,7 @@
2928	3030	* Take care memory controller reclaiming has small influence
2929	3031	* to global LRU.
2930	3032	*/
2931		- if (global_reclaim(sc)) {
	3033	+ if (!cgroup_reclaim(sc)) {
2932	3034	if (!cpuset_zone_allowed(zone,
2933	3035	GFP_KERNEL \| __GFP_HARDWALL))
2934	3036	continue;
..	..	@@ -2987,19 +3089,17 @@
2987	3089	sc->gfp_mask = orig_mask;
2988	3090	}
2989	3091
2990		-static void snapshot_refaults(struct mem_cgroup root_memcg, pg_data_t pgdat)
	3092	+static void snapshot_refaults(struct mem_cgroup target_memcg, pg_data_t pgdat)
2991	3093	{
2992		- struct mem_cgroup *memcg;
	3094	+ struct lruvec *target_lruvec;
	3095	+ unsigned long refaults;
2993	3096
2994		- memcg = mem_cgroup_iter(root_memcg, NULL, NULL);
2995		- do {
2996		- unsigned long refaults;
2997		- struct lruvec *lruvec;
2998		-
2999		- lruvec = mem_cgroup_lruvec(pgdat, memcg);
3000		- refaults = lruvec_page_state(lruvec, WORKINGSET_ACTIVATE);
3001		- lruvec->refaults = refaults;
3002		- } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
	3097	+ target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
	3098	+ refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
	3099	+ target_lruvec->refaults[0] = refaults;
	3100	+ refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
	3101	+ target_lruvec->refaults[1] = refaults;
	3102	+ trace_android_vh_snapshot_refaults(target_lruvec);
3003	3103	}
3004	3104
3005	3105	/*
..	..	@@ -3028,7 +3128,7 @@
3028	3128	retry:
3029	3129	delayacct_freepages_start();
3030	3130
3031		- if (global_reclaim(sc))
	3131	+ if (!cgroup_reclaim(sc))
3032	3132	__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
3033	3133
3034	3134	do {
..	..	@@ -3057,8 +3157,16 @@
3057	3157	if (zone->zone_pgdat == last_pgdat)
3058	3158	continue;
3059	3159	last_pgdat = zone->zone_pgdat;
	3160	+
3060	3161	snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3061		- set_memcg_congestion(last_pgdat, sc->target_mem_cgroup, false);
	3162	+
	3163	+ if (cgroup_reclaim(sc)) {
	3164	+ struct lruvec *lruvec;
	3165	+
	3166	+ lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
	3167	+ zone->zone_pgdat);
	3168	+ clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
	3169	+ }
3062	3170	}
3063	3171
3064	3172	delayacct_freepages_end();
..	..	@@ -3070,9 +3178,26 @@
3070	3178	if (sc->compaction_ready)
3071	3179	return 1;
3072	3180
	3181	+ /*
	3182	+ * We make inactive:active ratio decisions based on the node's
	3183	+ * composition of memory, but a restrictive reclaim_idx or a
	3184	+ * memory.low cgroup setting can exempt large amounts of
	3185	+ * memory from reclaim. Neither of which are very common, so
	3186	+ * instead of doing costly eligibility calculations of the
	3187	+ * entire cgroup subtree up front, we assume the estimates are
	3188	+ * good, and retry with forcible deactivation if that fails.
	3189	+ */
	3190	+ if (sc->skipped_deactivate) {
	3191	+ sc->priority = initial_priority;
	3192	+ sc->force_deactivate = 1;
	3193	+ sc->skipped_deactivate = 0;
	3194	+ goto retry;
	3195	+ }
	3196	+
3073	3197	/* Untapped cgroup reserves? Don't OOM, retry. */
3074	3198	if (sc->memcg_low_skipped) {
3075	3199	sc->priority = initial_priority;
	3200	+ sc->force_deactivate = 0;
3076	3201	sc->memcg_low_reclaim = 1;
3077	3202	sc->memcg_low_skipped = 0;
3078	3203	goto retry;
..	..	@@ -3112,8 +3237,8 @@
3112	3237
3113	3238	/* kswapd must be awake if processes are being throttled */
3114	3239	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
3115		- if (READ_ONCE(pgdat->kswapd_classzone_idx) > ZONE_NORMAL)
3116		- WRITE_ONCE(pgdat->kswapd_classzone_idx, ZONE_NORMAL);
	3240	+ if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
	3241	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);
3117	3242
3118	3243	wake_up_interruptible(&pgdat->kswapd_wait);
3119	3244	}
..	..	@@ -3246,25 +3371,26 @@
3246	3371	if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3247	3372	return 1;
3248	3373
3249		- trace_mm_vmscan_direct_reclaim_begin(order,
3250		- sc.may_writepage,
3251		- sc.gfp_mask,
3252		- sc.reclaim_idx);
	3374	+ set_task_reclaim_state(current, &sc.reclaim_state);
	3375	+ trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask);
3253	3376
3254	3377	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3255	3378
3256	3379	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
	3380	+ set_task_reclaim_state(current, NULL);
3257	3381
3258	3382	return nr_reclaimed;
3259	3383	}
3260	3384
3261	3385	#ifdef CONFIG_MEMCG
3262	3386
	3387	+/* Only used by soft limit reclaim. Do not reuse for anything else. */
3263	3388	unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3264	3389	gfp_t gfp_mask, bool noswap,
3265	3390	pg_data_t *pgdat,
3266	3391	unsigned long *nr_scanned)
3267	3392	{
	3393	+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
3268	3394	struct scan_control sc = {
3269	3395	.nr_to_reclaim = SWAP_CLUSTER_MAX,
3270	3396	.target_mem_cgroup = memcg,
..	..	@@ -3273,15 +3399,14 @@
3273	3399	.reclaim_idx = MAX_NR_ZONES - 1,
3274	3400	.may_swap = !noswap,
3275	3401	};
3276		- unsigned long lru_pages;
	3402	+
	3403	+ WARN_ON_ONCE(!current->reclaim_state);
3277	3404
3278	3405	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) \|
3279	3406	(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3280	3407
3281	3408	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
3282		- sc.may_writepage,
3283		- sc.gfp_mask,
3284		- sc.reclaim_idx);
	3409	+ sc.gfp_mask);
3285	3410
3286	3411	/*
3287	3412	* NOTE: Although we can get the priority field, using it
..	..	@@ -3290,11 +3415,12 @@
3290	3415	* will pick up pages from other mem cgroup's as well. We hack
3291	3416	* the priority and make it zero.
3292	3417	*/
3293		- shrink_node_memcg(pgdat, memcg, &sc, &lru_pages);
	3418	+ shrink_lruvec(lruvec, &sc);
3294	3419
3295	3420	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3296	3421
3297	3422	*nr_scanned = sc.nr_scanned;
	3423	+
3298	3424	return sc.nr_reclaimed;
3299	3425	}
3300	3426
..	..	@@ -3303,10 +3429,7 @@
3303	3429	gfp_t gfp_mask,
3304	3430	bool may_swap)
3305	3431	{
3306		- struct zonelist *zonelist;
3307	3432	unsigned long nr_reclaimed;
3308		- unsigned long pflags;
3309		- int nid;
3310	3433	unsigned int noreclaim_flag;
3311	3434	struct scan_control sc = {
3312	3435	.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
..	..	@@ -3319,78 +3442,101 @@
3319	3442	.may_unmap = 1,
3320	3443	.may_swap = may_swap,
3321	3444	};
3322		-
3323	3445	/*
3324		- * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
3325		- * take care of from where we get pages. So the node where we start the
3326		- * scan does not need to be the current node.
	3446	+ * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
	3447	+ * equal pressure on all the nodes. This is based on the assumption that
	3448	+ * the reclaim does not bail out early.
3327	3449	*/
3328		- nid = mem_cgroup_select_victim_node(memcg);
	3450	+ struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3329	3451
3330		- zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
3331		-
3332		- trace_mm_vmscan_memcg_reclaim_begin(0,
3333		- sc.may_writepage,
3334		- sc.gfp_mask,
3335		- sc.reclaim_idx);
3336		-
3337		- psi_memstall_enter(&pflags);
	3452	+ set_task_reclaim_state(current, &sc.reclaim_state);
	3453	+ trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
3338	3454	noreclaim_flag = memalloc_noreclaim_save();
3339	3455
3340	3456	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3341	3457
3342	3458	memalloc_noreclaim_restore(noreclaim_flag);
3343		- psi_memstall_leave(&pflags);
3344		-
3345	3459	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
	3460	+ set_task_reclaim_state(current, NULL);
3346	3461
3347	3462	return nr_reclaimed;
3348	3463	}
	3464	+EXPORT_SYMBOL_GPL(try_to_free_mem_cgroup_pages);
3349	3465	#endif
3350	3466
3351	3467	static void age_active_anon(struct pglist_data *pgdat,
3352	3468	struct scan_control *sc)
3353	3469	{
3354	3470	struct mem_cgroup *memcg;
	3471	+ struct lruvec *lruvec;
3355	3472
3356	3473	if (!total_swap_pages)
3357	3474	return;
3358	3475
	3476	+ lruvec = mem_cgroup_lruvec(NULL, pgdat);
	3477	+ if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
	3478	+ return;
	3479	+
3359	3480	memcg = mem_cgroup_iter(NULL, NULL, NULL);
3360	3481	do {
3361		- struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
3362		-
3363		- if (inactive_list_is_low(lruvec, false, sc, true))
3364		- shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3365		- sc, LRU_ACTIVE_ANON);
3366		-
	3482	+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
	3483	+ shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
	3484	+ sc, LRU_ACTIVE_ANON);
3367	3485	memcg = mem_cgroup_iter(NULL, memcg, NULL);
3368	3486	} while (memcg);
3369	3487	}
3370	3488
	3489	+static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
	3490	+{
	3491	+ int i;
	3492	+ struct zone *zone;
	3493	+
	3494	+ /*
	3495	+ * Check for watermark boosts top-down as the higher zones
	3496	+ * are more likely to be boosted. Both watermarks and boosts
	3497	+ * should not be checked at the same time as reclaim would
	3498	+ * start prematurely when there is no boosting and a lower
	3499	+ * zone is balanced.
	3500	+ */
	3501	+ for (i = highest_zoneidx; i >= 0; i--) {
	3502	+ zone = pgdat->node_zones + i;
	3503	+ if (!managed_zone(zone))
	3504	+ continue;
	3505	+
	3506	+ if (zone->watermark_boost)
	3507	+ return true;
	3508	+ }
	3509	+
	3510	+ return false;
	3511	+}
	3512	+
3371	3513	/*
3372	3514	* Returns true if there is an eligible zone balanced for the request order
3373		- * and classzone_idx
	3515	+ * and highest_zoneidx
3374	3516	*/
3375		-static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx)
	3517	+static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
3376	3518	{
3377	3519	int i;
3378	3520	unsigned long mark = -1;
3379	3521	struct zone *zone;
3380	3522
3381		- for (i = 0; i <= classzone_idx; i++) {
	3523	+ /*
	3524	+ * Check watermarks bottom-up as lower zones are more likely to
	3525	+ * meet watermarks.
	3526	+ */
	3527	+ for (i = 0; i <= highest_zoneidx; i++) {
3382	3528	zone = pgdat->node_zones + i;
3383	3529
3384	3530	if (!managed_zone(zone))
3385	3531	continue;
3386	3532
3387	3533	mark = high_wmark_pages(zone);
3388		- if (zone_watermark_ok_safe(zone, order, mark, classzone_idx))
	3534	+ if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx))
3389	3535	return true;
3390	3536	}
3391	3537
3392	3538	/*
3393		- * If a node has no populated zone within classzone_idx, it does not
	3539	+ * If a node has no populated zone within highest_zoneidx, it does not
3394	3540	* need balancing by definition. This can happen if a zone-restricted
3395	3541	* allocation tries to wake a remote kswapd.
3396	3542	*/
..	..	@@ -3403,7 +3549,9 @@
3403	3549	/* Clear pgdat state for congested, dirty or under writeback. */
3404	3550	static void clear_pgdat_congested(pg_data_t *pgdat)
3405	3551	{
3406		- clear_bit(PGDAT_CONGESTED, &pgdat->flags);
	3552	+ struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
	3553	+
	3554	+ clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3407	3555	clear_bit(PGDAT_DIRTY, &pgdat->flags);
3408	3556	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
3409	3557	}
..	..	@@ -3414,7 +3562,8 @@
3414	3562	*
3415	3563	* Returns true if kswapd is ready to sleep
3416	3564	*/
3417		-static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, int classzone_idx)
	3565	+static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
	3566	+ int highest_zoneidx)
3418	3567	{
3419	3568	/*
3420	3569	* The throttled processes are normally woken up in balance_pgdat() as
..	..	@@ -3436,7 +3585,7 @@
3436	3585	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3437	3586	return true;
3438	3587
3439		- if (pgdat_balanced(pgdat, order, classzone_idx)) {
	3588	+ if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
3440	3589	clear_pgdat_congested(pgdat);
3441	3590	return true;
3442	3591	}
..	..	@@ -3496,37 +3645,57 @@
3496	3645	*
3497	3646	* kswapd scans the zones in the highmem->normal->dma direction. It skips
3498	3647	* zones which have free_pages > high_wmark_pages(zone), but once a zone is
3499		- * found to have free_pages <= high_wmark_pages(zone), any page is that zone
	3648	+ * found to have free_pages <= high_wmark_pages(zone), any page in that zone
3500	3649	* or lower is eligible for reclaim until at least one usable zone is
3501	3650	* balanced.
3502	3651	*/
3503		-static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
	3652	+static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
3504	3653	{
3505	3654	int i;
3506	3655	unsigned long nr_soft_reclaimed;
3507	3656	unsigned long nr_soft_scanned;
3508	3657	unsigned long pflags;
	3658	+ unsigned long nr_boost_reclaim;
	3659	+ unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
	3660	+ bool boosted;
3509	3661	struct zone *zone;
3510	3662	struct scan_control sc = {
3511	3663	.gfp_mask = GFP_KERNEL,
3512	3664	.order = order,
3513		- .priority = DEF_PRIORITY,
3514		- .may_writepage = !laptop_mode,
3515	3665	.may_unmap = 1,
3516		- .may_swap = 1,
3517	3666	};
3518	3667
	3668	+ set_task_reclaim_state(current, &sc.reclaim_state);
3519	3669	psi_memstall_enter(&pflags);
3520	3670	__fs_reclaim_acquire();
3521	3671
3522	3672	count_vm_event(PAGEOUTRUN);
3523	3673
	3674	+ /*
	3675	+ * Account for the reclaim boost. Note that the zone boost is left in
	3676	+ * place so that parallel allocations that are near the watermark will
	3677	+ * stall or direct reclaim until kswapd is finished.
	3678	+ */
	3679	+ nr_boost_reclaim = 0;
	3680	+ for (i = 0; i <= highest_zoneidx; i++) {
	3681	+ zone = pgdat->node_zones + i;
	3682	+ if (!managed_zone(zone))
	3683	+ continue;
	3684	+
	3685	+ nr_boost_reclaim += zone->watermark_boost;
	3686	+ zone_boosts[i] = zone->watermark_boost;
	3687	+ }
	3688	+ boosted = nr_boost_reclaim;
	3689	+
	3690	+restart:
	3691	+ sc.priority = DEF_PRIORITY;
3524	3692	do {
3525	3693	unsigned long nr_reclaimed = sc.nr_reclaimed;
3526	3694	bool raise_priority = true;
	3695	+ bool balanced;
3527	3696	bool ret;
3528	3697
3529		- sc.reclaim_idx = classzone_idx;
	3698	+ sc.reclaim_idx = highest_zoneidx;
3530	3699
3531	3700	/*
3532	3701	* If the number of buffer_heads exceeds the maximum allowed
..	..	@@ -3550,12 +3719,38 @@
3550	3719	}
3551	3720
3552	3721	/*
3553		- * Only reclaim if there are no eligible zones. Note that
3554		- * sc.reclaim_idx is not used as buffer_heads_over_limit may
3555		- * have adjusted it.
	3722	+ * If the pgdat is imbalanced then ignore boosting and preserve
	3723	+ * the watermarks for a later time and restart. Note that the
	3724	+ * zone watermarks will be still reset at the end of balancing
	3725	+ * on the grounds that the normal reclaim should be enough to
	3726	+ * re-evaluate if boosting is required when kswapd next wakes.
3556	3727	*/
3557		- if (pgdat_balanced(pgdat, sc.order, classzone_idx))
	3728	+ balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);
	3729	+ if (!balanced && nr_boost_reclaim) {
	3730	+ nr_boost_reclaim = 0;
	3731	+ goto restart;
	3732	+ }
	3733	+
	3734	+ /*
	3735	+ * If boosting is not active then only reclaim if there are no
	3736	+ * eligible zones. Note that sc.reclaim_idx is not used as
	3737	+ * buffer_heads_over_limit may have adjusted it.
	3738	+ */
	3739	+ if (!nr_boost_reclaim && balanced)
3558	3740	goto out;
	3741	+
	3742	+ /* Limit the priority of boosting to avoid reclaim writeback */
	3743	+ if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
	3744	+ raise_priority = false;
	3745	+
	3746	+ /*
	3747	+ * Do not writeback or swap pages for boosted reclaim. The
	3748	+ * intent is to relieve pressure not issue sub-optimal IO
	3749	+ * from reclaim context. If no pages are reclaimed, the
	3750	+ * reclaim will be aborted.
	3751	+ */
	3752	+ sc.may_writepage = !laptop_mode && !nr_boost_reclaim;
	3753	+ sc.may_swap = !nr_boost_reclaim;
3559	3754
3560	3755	/*
3561	3756	* Do some background aging of the anon list, to give
..	..	@@ -3608,6 +3803,16 @@
3608	3803	* progress in reclaiming pages
3609	3804	*/
3610	3805	nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
	3806	+ nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
	3807	+
	3808	+ /*
	3809	+ * If reclaim made no progress for a boost, stop reclaim as
	3810	+ * IO cannot be queued and it could be an infinite loop in
	3811	+ * extreme circumstances.
	3812	+ */
	3813	+ if (nr_boost_reclaim && !nr_reclaimed)
	3814	+ break;
	3815	+
3611	3816	if (raise_priority \|\| !nr_reclaimed)
3612	3817	sc.priority--;
3613	3818	} while (sc.priority >= 1);
..	..	@@ -3616,9 +3821,33 @@
3616	3821	pgdat->kswapd_failures++;
3617	3822
3618	3823	out:
	3824	+ /* If reclaim was boosted, account for the reclaim done in this pass */
	3825	+ if (boosted) {
	3826	+ unsigned long flags;
	3827	+
	3828	+ for (i = 0; i <= highest_zoneidx; i++) {
	3829	+ if (!zone_boosts[i])
	3830	+ continue;
	3831	+
	3832	+ /* Increments are under the zone lock */
	3833	+ zone = pgdat->node_zones + i;
	3834	+ spin_lock_irqsave(&zone->lock, flags);
	3835	+ zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
	3836	+ spin_unlock_irqrestore(&zone->lock, flags);
	3837	+ }
	3838	+
	3839	+ /*
	3840	+ * As there is now likely space, wakeup kcompact to defragment
	3841	+ * pageblocks.
	3842	+ */
	3843	+ wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);
	3844	+ }
	3845	+
3619	3846	snapshot_refaults(NULL, pgdat);
3620	3847	__fs_reclaim_release();
3621	3848	psi_memstall_leave(&pflags);
	3849	+ set_task_reclaim_state(current, NULL);
	3850	+
3622	3851	/*
3623	3852	* Return the order kswapd stopped reclaiming at as
3624	3853	* prepare_kswapd_sleep() takes it into account. If another caller
..	..	@@ -3629,22 +3858,22 @@
3629	3858	}
3630	3859
3631	3860	/*
3632		- * The pgdat->kswapd_classzone_idx is used to pass the highest zone index to be
3633		- * reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is not
3634		- * a valid index then either kswapd runs for first time or kswapd couldn't sleep
3635		- * after previous reclaim attempt (node is still unbalanced). In that case
3636		- * return the zone index of the previous kswapd reclaim cycle.
	3861	+ * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
	3862	+ * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
	3863	+ * not a valid index then either kswapd runs for first time or kswapd couldn't
	3864	+ * sleep after previous reclaim attempt (node is still unbalanced). In that
	3865	+ * case return the zone index of the previous kswapd reclaim cycle.
3637	3866	*/
3638		-static enum zone_type kswapd_classzone_idx(pg_data_t *pgdat,
3639		- enum zone_type prev_classzone_idx)
	3867	+static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,
	3868	+ enum zone_type prev_highest_zoneidx)
3640	3869	{
3641		- enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_classzone_idx);
	3870	+ enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
3642	3871
3643		- return curr_idx == MAX_NR_ZONES ? prev_classzone_idx : curr_idx;
	3872	+ return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx;
3644	3873	}
3645	3874
3646	3875	static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
3647		- unsigned int classzone_idx)
	3876	+ unsigned int highest_zoneidx)
3648	3877	{
3649	3878	long remaining = 0;
3650	3879	DEFINE_WAIT(wait);
..	..	@@ -3661,7 +3890,7 @@
3661	3890	* eligible zone balanced that it's also unlikely that compaction will
3662	3891	* succeed.
3663	3892	*/
3664		- if (prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
	3893	+ if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
3665	3894	/*
3666	3895	* Compaction records what page blocks it recently failed to
3667	3896	* isolate pages from and skips them in the future scanning.
..	..	@@ -3674,18 +3903,19 @@
3674	3903	* We have freed the memory, now we should compact it to make
3675	3904	* allocation of the requested order possible.
3676	3905	*/
3677		- wakeup_kcompactd(pgdat, alloc_order, classzone_idx);
	3906	+ wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);
3678	3907
3679	3908	remaining = schedule_timeout(HZ/10);
3680	3909
3681	3910	/*
3682		- * If woken prematurely then reset kswapd_classzone_idx and
	3911	+ * If woken prematurely then reset kswapd_highest_zoneidx and
3683	3912	* order. The values will either be from a wakeup request or
3684	3913	* the previous request that slept prematurely.
3685	3914	*/
3686	3915	if (remaining) {
3687		- WRITE_ONCE(pgdat->kswapd_classzone_idx,
3688		- kswapd_classzone_idx(pgdat, classzone_idx));
	3916	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx,
	3917	+ kswapd_highest_zoneidx(pgdat,
	3918	+ highest_zoneidx));
3689	3919
3690	3920	if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)
3691	3921	WRITE_ONCE(pgdat->kswapd_order, reclaim_order);
..	..	@@ -3700,7 +3930,7 @@
3700	3930	* go fully to sleep until explicitly woken up.
3701	3931	*/
3702	3932	if (!remaining &&
3703		- prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
	3933	+ prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
3704	3934	trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
3705	3935
3706	3936	/*
..	..	@@ -3742,18 +3972,13 @@
3742	3972	static int kswapd(void *p)
3743	3973	{
3744	3974	unsigned int alloc_order, reclaim_order;
3745		- unsigned int classzone_idx = MAX_NR_ZONES - 1;
	3975	+ unsigned int highest_zoneidx = MAX_NR_ZONES - 1;
3746	3976	pg_data_t pgdat = (pg_data_t)p;
3747	3977	struct task_struct *tsk = current;
3748		-
3749		- struct reclaim_state reclaim_state = {
3750		- .reclaimed_slab = 0,
3751		- };
3752	3978	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3753	3979
3754	3980	if (!cpumask_empty(cpumask))
3755	3981	set_cpus_allowed_ptr(tsk, cpumask);
3756		- current->reclaim_state = &reclaim_state;
3757	3982
3758	3983	/*
3759	3984	* Tell the memory management that we're a "memory allocator",
..	..	@@ -3771,22 +3996,24 @@
3771	3996	set_freezable();
3772	3997
3773	3998	WRITE_ONCE(pgdat->kswapd_order, 0);
3774		- WRITE_ONCE(pgdat->kswapd_classzone_idx, MAX_NR_ZONES);
	3999	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
3775	4000	for ( ; ; ) {
3776	4001	bool ret;
3777	4002
3778	4003	alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
3779		- classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx);
	4004	+ highest_zoneidx = kswapd_highest_zoneidx(pgdat,
	4005	+ highest_zoneidx);
3780	4006
3781	4007	kswapd_try_sleep:
3782	4008	kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
3783		- classzone_idx);
	4009	+ highest_zoneidx);
3784	4010
3785		- /* Read the new order and classzone_idx */
	4011	+ /* Read the new order and highest_zoneidx */
3786	4012	alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
3787		- classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx);
	4013	+ highest_zoneidx = kswapd_highest_zoneidx(pgdat,
	4014	+ highest_zoneidx);
3788	4015	WRITE_ONCE(pgdat->kswapd_order, 0);
3789		- WRITE_ONCE(pgdat->kswapd_classzone_idx, MAX_NR_ZONES);
	4016	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
3790	4017
3791	4018	ret = try_to_freeze();
3792	4019	if (kthread_should_stop())
..	..	@@ -3807,17 +4034,57 @@
3807	4034	* but kcompactd is woken to compact for the original
3808	4035	* request (alloc_order).
3809	4036	*/
3810		- trace_mm_vmscan_kswapd_wake(pgdat->node_id, classzone_idx,
	4037	+ trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,
3811	4038	alloc_order);
3812		- reclaim_order = balance_pgdat(pgdat, alloc_order, classzone_idx);
	4039	+ reclaim_order = balance_pgdat(pgdat, alloc_order,
	4040	+ highest_zoneidx);
3813	4041	if (reclaim_order < alloc_order)
3814	4042	goto kswapd_try_sleep;
3815	4043	}
3816	4044
3817	4045	tsk->flags &= ~(PF_MEMALLOC \| PF_SWAPWRITE \| PF_KSWAPD);
3818		- current->reclaim_state = NULL;
3819	4046
3820	4047	return 0;
	4048	+}
	4049	+
	4050	+static int kswapd_per_node_run(int nid)
	4051	+{
	4052	+ pg_data_t *pgdat = NODE_DATA(nid);
	4053	+ int hid;
	4054	+ int ret = 0;
	4055	+
	4056	+ for (hid = 0; hid < kswapd_threads; ++hid) {
	4057	+ pgdat->mkswapd[hid] = kthread_run(kswapd, pgdat, "kswapd%d:%d",
	4058	+ nid, hid);
	4059	+ if (IS_ERR(pgdat->mkswapd[hid])) {
	4060	+ /* failure at boot is fatal */
	4061	+ WARN_ON(system_state < SYSTEM_RUNNING);
	4062	+ pr_err("Failed to start kswapd%d on node %d\n",
	4063	+ hid, nid);
	4064	+ ret = PTR_ERR(pgdat->mkswapd[hid]);
	4065	+ pgdat->mkswapd[hid] = NULL;
	4066	+ continue;
	4067	+ }
	4068	+ if (!pgdat->kswapd)
	4069	+ pgdat->kswapd = pgdat->mkswapd[hid];
	4070	+ }
	4071	+
	4072	+ return ret;
	4073	+}
	4074	+
	4075	+static void kswapd_per_node_stop(int nid)
	4076	+{
	4077	+ int hid = 0;
	4078	+ struct task_struct *kswapd;
	4079	+
	4080	+ for (hid = 0; hid < kswapd_threads; hid++) {
	4081	+ kswapd = NODE_DATA(nid)->mkswapd[hid];
	4082	+ if (kswapd) {
	4083	+ kthread_stop(kswapd);
	4084	+ NODE_DATA(nid)->mkswapd[hid] = NULL;
	4085	+ }
	4086	+ }
	4087	+ NODE_DATA(nid)->kswapd = NULL;
3821	4088	}
3822	4089
3823	4090	/*
..	..	@@ -3828,7 +4095,7 @@
3828	4095	* needed.
3829	4096	*/
3830	4097	void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
3831		- enum zone_type classzone_idx)
	4098	+ enum zone_type highest_zoneidx)
3832	4099	{
3833	4100	pg_data_t *pgdat;
3834	4101	enum zone_type curr_idx;
..	..	@@ -3840,10 +4107,10 @@
3840	4107	return;
3841	4108
3842	4109	pgdat = zone->zone_pgdat;
3843		- curr_idx = READ_ONCE(pgdat->kswapd_classzone_idx);
	4110	+ curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
3844	4111
3845		- if (curr_idx == MAX_NR_ZONES \|\| curr_idx < classzone_idx)
3846		- WRITE_ONCE(pgdat->kswapd_classzone_idx, classzone_idx);
	4112	+ if (curr_idx == MAX_NR_ZONES \|\| curr_idx < highest_zoneidx)
	4113	+ WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);
3847	4114
3848	4115	if (READ_ONCE(pgdat->kswapd_order) < order)
3849	4116	WRITE_ONCE(pgdat->kswapd_order, order);
..	..	@@ -3853,7 +4120,8 @@
3853	4120
3854	4121	/* Hopeless node, leave it to direct reclaim if possible */
3855	4122	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES \|\|
3856		- pgdat_balanced(pgdat, order, classzone_idx)) {
	4123	+ (pgdat_balanced(pgdat, order, highest_zoneidx) &&
	4124	+ !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
3857	4125	/*
3858	4126	* There may be plenty of free memory available, but it's too
3859	4127	* fragmented for high-order allocations. Wake up kcompactd
..	..	@@ -3862,11 +4130,11 @@
3862	4130	* ratelimit its work.
3863	4131	*/
3864	4132	if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
3865		- wakeup_kcompactd(pgdat, order, classzone_idx);
	4133	+ wakeup_kcompactd(pgdat, order, highest_zoneidx);
3866	4134	return;
3867	4135	}
3868	4136
3869		- trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, classzone_idx, order,
	4137	+ trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,
3870	4138	gfp_flags);
3871	4139	wake_up_interruptible(&pgdat->kswapd_wait);
3872	4140	}
..	..	@@ -3882,7 +4150,6 @@
3882	4150	*/
3883	4151	unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3884	4152	{
3885		- struct reclaim_state reclaim_state;
3886	4153	struct scan_control sc = {
3887	4154	.nr_to_reclaim = nr_to_reclaim,
3888	4155	.gfp_mask = GFP_HIGHUSER_MOVABLE,
..	..	@@ -3894,45 +4161,22 @@
3894	4161	.hibernation_mode = 1,
3895	4162	};
3896	4163	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3897		- struct task_struct *p = current;
3898	4164	unsigned long nr_reclaimed;
3899	4165	unsigned int noreclaim_flag;
3900	4166
3901	4167	fs_reclaim_acquire(sc.gfp_mask);
3902	4168	noreclaim_flag = memalloc_noreclaim_save();
3903		- reclaim_state.reclaimed_slab = 0;
3904		- p->reclaim_state = &reclaim_state;
	4169	+ set_task_reclaim_state(current, &sc.reclaim_state);
3905	4170
3906	4171	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3907	4172
3908		- p->reclaim_state = NULL;
	4173	+ set_task_reclaim_state(current, NULL);
3909	4174	memalloc_noreclaim_restore(noreclaim_flag);
3910	4175	fs_reclaim_release(sc.gfp_mask);
3911	4176
3912	4177	return nr_reclaimed;
3913	4178	}
3914	4179	#endif /* CONFIG_HIBERNATION */
3915		-
3916		-/* It's optimal to keep kswapds on the same CPUs as their memory, but
3917		- not required for correctness. So if the last cpu in a node goes
3918		- away, we get changed to run anywhere: as the first one comes back,
3919		- restore their cpu bindings. */
3920		-static int kswapd_cpu_online(unsigned int cpu)
3921		-{
3922		- int nid;
3923		-
3924		- for_each_node_state(nid, N_MEMORY) {
3925		- pg_data_t *pgdat = NODE_DATA(nid);
3926		- const struct cpumask *mask;
3927		-
3928		- mask = cpumask_of_node(pgdat->node_id);
3929		-
3930		- if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3931		- /* One of our CPUs online: restore mask */
3932		- set_cpus_allowed_ptr(pgdat->kswapd, mask);
3933		- }
3934		- return 0;
3935		-}
3936	4180
3937	4181	/*
3938	4182	* This kswapd start function will be called by init and node-hot-add.
..	..	@@ -3945,6 +4189,9 @@
3945	4189
3946	4190	if (pgdat->kswapd)
3947	4191	return 0;
	4192	+
	4193	+ if (kswapd_threads > 1)
	4194	+ return kswapd_per_node_run(nid);
3948	4195
3949	4196	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
3950	4197	if (IS_ERR(pgdat->kswapd)) {
..	..	@@ -3965,6 +4212,11 @@
3965	4212	{
3966	4213	struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
3967	4214
	4215	+ if (kswapd_threads > 1) {
	4216	+ kswapd_per_node_stop(nid);
	4217	+ return;
	4218	+ }
	4219	+
3968	4220	if (kswapd) {
3969	4221	kthread_stop(kswapd);
3970	4222	NODE_DATA(nid)->kswapd = NULL;
..	..	@@ -3973,15 +4225,11 @@
3973	4225
3974	4226	static int __init kswapd_init(void)
3975	4227	{
3976		- int nid, ret;
	4228	+ int nid;
3977	4229
3978	4230	swap_setup();
3979	4231	for_each_node_state(nid, N_MEMORY)
3980	4232	kswapd_run(nid);
3981		- ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
3982		- "mm/vmscan:online", kswapd_cpu_online,
3983		- NULL);
3984		- WARN_ON(ret < 0);
3985	4233	return 0;
3986	4234	}
3987	4235
..	..	@@ -3996,10 +4244,13 @@
3996	4244	*/
3997	4245	int node_reclaim_mode __read_mostly;
3998	4246
3999		-#define RECLAIM_OFF 0
4000		-#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */
4001		-#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
4002		-#define RECLAIM_UNMAP (1<<2) /* Unmap pages during reclaim */
	4247	+/*
	4248	+ * These bit locations are exposed in the vm.zone_reclaim_mode sysctl
	4249	+ * ABI. New bits are OK, but existing bits can never change.
	4250	+ */
	4251	+#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */
	4252	+#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
	4253	+#define RECLAIM_UNMAP (1<<2) /* Unmap pages during reclaim */
4003	4254
4004	4255	/*
4005	4256	* Priority for NODE_RECLAIM. This determines the fraction of pages
..	..	@@ -4070,7 +4321,6 @@
4070	4321	/* Minimum pages needed in order to stay on node */
4071	4322	const unsigned long nr_pages = 1 << order;
4072	4323	struct task_struct *p = current;
4073		- struct reclaim_state reclaim_state;
4074	4324	unsigned int noreclaim_flag;
4075	4325	struct scan_control sc = {
4076	4326	.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
..	..	@@ -4083,6 +4333,9 @@
4083	4333	.reclaim_idx = gfp_zone(gfp_mask),
4084	4334	};
4085	4335
	4336	+ trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,
	4337	+ sc.gfp_mask);
	4338	+
4086	4339	cond_resched();
4087	4340	fs_reclaim_acquire(sc.gfp_mask);
4088	4341	/*
..	..	@@ -4092,8 +4345,7 @@
4092	4345	*/
4093	4346	noreclaim_flag = memalloc_noreclaim_save();
4094	4347	p->flags \|= PF_SWAPWRITE;
4095		- reclaim_state.reclaimed_slab = 0;
4096		- p->reclaim_state = &reclaim_state;
	4348	+ set_task_reclaim_state(p, &sc.reclaim_state);
4097	4349
4098	4350	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
4099	4351	/*
..	..	@@ -4105,10 +4357,13 @@
4105	4357	} while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
4106	4358	}
4107	4359
4108		- p->reclaim_state = NULL;
	4360	+ set_task_reclaim_state(p, NULL);
4109	4361	current->flags &= ~PF_SWAPWRITE;
4110	4362	memalloc_noreclaim_restore(noreclaim_flag);
4111	4363	fs_reclaim_release(sc.gfp_mask);
	4364	+
	4365	+ trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed);
	4366	+
4112	4367	return sc.nr_reclaimed >= nr_pages;
4113	4368	}
4114	4369
..	..	@@ -4127,7 +4382,8 @@
4127	4382	* unmapped file backed pages.
4128	4383	*/
4129	4384	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
4130		- node_page_state(pgdat, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
	4385	+ node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
	4386	+ pgdat->min_slab_pages)
4131	4387	return NODE_RECLAIM_FULL;
4132	4388
4133	4389	/*
..	..	@@ -4158,29 +4414,6 @@
4158	4414	}
4159	4415	#endif
4160	4416
4161		-/*
4162		- * page_evictable - test whether a page is evictable
4163		- * @page: the page to test
4164		- *
4165		- * Test whether page is evictable--i.e., should be placed on active/inactive
4166		- * lists vs unevictable list.
4167		- *
4168		- * Reasons page might not be evictable:
4169		- * (1) page's mapping marked unevictable
4170		- * (2) page is part of an mlocked VMA
4171		- *
4172		- */
4173		-int page_evictable(struct page *page)
4174		-{
4175		- int ret;
4176		-
4177		- /* Prevent address_space of inode and swap cache from being freed */
4178		- rcu_read_lock();
4179		- ret = !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
4180		- rcu_read_unlock();
4181		- return ret;
4182		-}
4183		-
4184	4417	/**
4185	4418	* check_move_unevictable_pages - check pages for evictability and move to
4186	4419	* appropriate zone lru list
..	..	@@ -4201,8 +4434,14 @@
4201	4434	for (i = 0; i < pvec->nr; i++) {
4202	4435	struct page *page = pvec->pages[i];
4203	4436	struct pglist_data *pagepgdat = page_pgdat(page);
	4437	+ int nr_pages;
4204	4438
4205		- pgscanned++;
	4439	+ if (PageTransTail(page))
	4440	+ continue;
	4441	+
	4442	+ nr_pages = thp_nr_pages(page);
	4443	+ pgscanned += nr_pages;
	4444	+
4206	4445	if (pagepgdat != pgdat) {
4207	4446	if (pgdat)
4208	4447	spin_unlock_irq(&pgdat->lru_lock);
..	..	@@ -4221,7 +4460,7 @@
4221	4460	ClearPageUnevictable(page);
4222	4461	del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
4223	4462	add_page_to_lru_list(page, lruvec, lru);
4224		- pgrescued++;
	4463	+ pgrescued += nr_pages;
4225	4464	}
4226	4465	}
4227	4466