write in 30M

hc

2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc

 kernel/mm/swap_slots.c
..
..
@@ -33,6 +33,7 @@
33
33
 #include <linux/vmalloc.h>
34
34
 #include <linux/mutex.h>
35
35
 #include <linux/mm.h>
36
+#include <trace/hooks/mm.h>
36
37
 
37
38
 static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
38
39
 static bool	swap_slot_cache_active;
..
..
@@ -46,8 +47,7 @@
46
47
 static void deactivate_swap_slots_cache(void);
47
48
 static void reactivate_swap_slots_cache(void);
48
49
 
49
-#define use_swap_slot_cache (swap_slot_cache_active && \
50
-		swap_slot_cache_enabled && swap_slot_cache_initialized)
50
+#define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
51
51
 #define SLOTS_CACHE 0x1
52
52
 #define SLOTS_CACHE_RET 0x2
53
53
 
..
..
@@ -55,6 +55,7 @@
55
55
 {
56
56
 	mutex_lock(&swap_slots_cache_mutex);
57
57
 	swap_slot_cache_active = false;
58
+	trace_android_vh_swap_slot_cache_active(false);
58
59
 	__drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
59
60
 	mutex_unlock(&swap_slots_cache_mutex);
60
61
 }
..
..
@@ -63,6 +64,7 @@
63
64
 {
64
65
 	mutex_lock(&swap_slots_cache_mutex);
65
66
 	swap_slot_cache_active = true;
67
+	trace_android_vh_swap_slot_cache_active(true);
66
68
 	mutex_unlock(&swap_slots_cache_mutex);
67
69
 }
68
70
 
..
..
@@ -90,11 +92,17 @@
90
92
 	mutex_unlock(&swap_slots_cache_enable_mutex);
91
93
 }
92
94
 
93
-static bool check_cache_active(void)
95
+bool is_swap_slot_cache_enabled(void)
96
+{
97
+	return swap_slot_cache_enabled;
98
+}
99
+EXPORT_SYMBOL_GPL(is_swap_slot_cache_enabled);
100
+
101
+bool check_cache_active(void)
94
102
 {
95
103
 	long pages;
96
104
 
97
-	if (!swap_slot_cache_enabled || !swap_slot_cache_initialized)
105
+	if (!swap_slot_cache_enabled)
98
106
 		return false;
99
107
 
100
108
 	pages = get_nr_swap_pages();
..
..
@@ -111,17 +119,26 @@
111
119
 out:
112
120
 	return swap_slot_cache_active;
113
121
 }
122
+EXPORT_SYMBOL_GPL(check_cache_active);
114
123
 
115
124
 static int alloc_swap_slot_cache(unsigned int cpu)
116
125
 {
117
126
 	struct swap_slots_cache *cache;
118
127
 	swp_entry_t *slots, *slots_ret;
128
+	bool skip = false;
129
+	int ret = 0;
119
130
 
120
131
 	/*
121
132
 	 * Do allocation outside swap_slots_cache_mutex
122
133
 	 * as kvzalloc could trigger reclaim and get_swap_page,
123
134
 	 * which can lock swap_slots_cache_mutex.
124
135
 	 */
136
+	trace_android_rvh_alloc_swap_slot_cache(&per_cpu(swp_slots, cpu),
137
+		&ret, &skip);
138
+	trace_android_vh_alloc_swap_slot_cache(&per_cpu(swp_slots, cpu),
139
+		&ret, &skip);
140
+	if (skip)
141
+		return ret;
125
142
 	slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
126
143
 			 GFP_KERNEL);
127
144
 	if (!slots)
..
..
@@ -136,9 +153,16 @@
136
153
 
137
154
 	mutex_lock(&swap_slots_cache_mutex);
138
155
 	cache = &per_cpu(swp_slots, cpu);
139
-	if (cache->slots || cache->slots_ret)
156
+	if (cache->slots || cache->slots_ret) {
140
157
 		/* cache already allocated */
141
-		goto out;
158
+		mutex_unlock(&swap_slots_cache_mutex);
159
+
160
+		kvfree(slots);
161
+		kvfree(slots_ret);
162
+
163
+		return 0;
164
+	}
165
+
142
166
 	if (!cache->lock_initialized) {
143
167
 		mutex_init(&cache->alloc_lock);
144
168
 		spin_lock_init(&cache->free_lock);
..
..
@@ -155,15 +179,8 @@
155
179
 	 */
156
180
 	mb();
157
181
 	cache->slots = slots;
158
-	slots = NULL;
159
182
 	cache->slots_ret = slots_ret;
160
-	slots_ret = NULL;
161
-out:
162
183
 	mutex_unlock(&swap_slots_cache_mutex);
163
-	if (slots)
164
-		kvfree(slots);
165
-	if (slots_ret)
166
-		kvfree(slots_ret);
167
184
 	return 0;
168
185
 }
169
186
 
..
..
@@ -172,8 +189,15 @@
172
189
 {
173
190
 	struct swap_slots_cache *cache;
174
191
 	swp_entry_t *slots = NULL;
192
+	bool skip = false;
175
193
 
176
194
 	cache = &per_cpu(swp_slots, cpu);
195
+	trace_android_rvh_drain_slots_cache_cpu(cache, type,
196
+		free_slots, &skip);
197
+	trace_android_vh_drain_slots_cache_cpu(cache, type,
198
+		free_slots, &skip);
199
+	if (skip)
200
+		return;
177
201
 	if ((type & SLOTS_CACHE) && cache->slots) {
178
202
 		mutex_lock(&cache->alloc_lock);
179
203
 		swapcache_free_entries(cache->slots + cache->cur, cache->nr);
..
..
@@ -238,27 +262,24 @@
238
262
 	return 0;
239
263
 }
240
264
 
241
-int enable_swap_slots_cache(void)
265
+void enable_swap_slots_cache(void)
242
266
 {
243
-	int ret = 0;
244
-
245
267
 	mutex_lock(&swap_slots_cache_enable_mutex);
246
-	if (swap_slot_cache_initialized) {
247
-		__reenable_swap_slots_cache();
248
-		goto out_unlock;
268
+	if (!swap_slot_cache_initialized) {
269
+		int ret;
270
+
271
+		ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
272
+					alloc_swap_slot_cache, free_slot_cache);
273
+		if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
274
+				       "without swap slots cache.\n", __func__))
275
+			goto out_unlock;
276
+
277
+		swap_slot_cache_initialized = true;
249
278
 	}
250
279
 
251
-	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
252
-				alloc_swap_slot_cache, free_slot_cache);
253
-	if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
254
-			       "without swap slots cache.\n", __func__))
255
-		goto out_unlock;
256
-
257
-	swap_slot_cache_initialized = true;
258
280
 	__reenable_swap_slots_cache();
259
281
 out_unlock:
260
282
 	mutex_unlock(&swap_slots_cache_enable_mutex);
261
-	return 0;
262
283
 }
263
284
 
264
285
 /* called with swap slot cache's alloc lock held */
..
..
@@ -278,8 +299,13 @@
278
299
 int free_swap_slot(swp_entry_t entry)
279
300
 {
280
301
 	struct swap_slots_cache *cache;
302
+	bool skip = false;
281
303
 
282
304
 	cache = raw_cpu_ptr(&swp_slots);
305
+	trace_android_rvh_free_swap_slot(entry, cache, &skip);
306
+	trace_android_vh_free_swap_slot(entry, cache, &skip);
307
+	if (skip)
308
+		return 0;
283
309
 	if (likely(use_swap_slot_cache && cache->slots_ret)) {
284
310
 		spin_lock_irq(&cache->free_lock);
285
311
 		/* Swap slots cache may be deactivated before acquiring lock */
..
..
@@ -309,10 +335,15 @@
309
335
 
310
336
 swp_entry_t get_swap_page(struct page *page)
311
337
 {
312
-	swp_entry_t entry, *pentry;
338
+	swp_entry_t entry;
313
339
 	struct swap_slots_cache *cache;
314
-
340
+	bool found = false;
315
341
 	entry.val = 0;
342
+
343
+	trace_android_rvh_get_swap_page(page, &entry, raw_cpu_ptr(&swp_slots), &found);
344
+	trace_android_vh_get_swap_page(page, &entry, raw_cpu_ptr(&swp_slots), &found);
345
+	if (found)
346
+		goto out;
316
347
 
317
348
 	if (PageTransHuge(page)) {
318
349
 		if (IS_ENABLED(CONFIG_THP_SWAP))
..
..
@@ -336,13 +367,11 @@
336
367
 		if (cache->slots) {
337
368
 repeat:
338
369
 			if (cache->nr) {
339
-				pentry = &cache->slots[cache->cur++];
340
-				entry = *pentry;
341
-				pentry->val = 0;
370
+				entry = cache->slots[cache->cur];
371
+				cache->slots[cache->cur++].val = 0;
342
372
 				cache->nr--;
343
-			} else {
344
-				if (refill_swap_slots_cache(cache))
345
-					goto repeat;
373
+			} else if (refill_swap_slots_cache(cache)) {
374
+				goto repeat;
346
375
 			}
347
376
 		}
348
377
 		mutex_unlock(&cache->alloc_lock);