hc
2024-05-10 37f49e37ab4cb5d0bc4c60eb5c6d4dd57db767bb
kernel/include/linux/swap.h
....@@ -12,7 +12,6 @@
1212 #include <linux/fs.h>
1313 #include <linux/atomic.h>
1414 #include <linux/page-flags.h>
15
-#include <linux/locallock.h>
1615 #include <asm/page.h>
1716
1817 struct notifier_block;
....@@ -149,7 +148,7 @@
149148 * We always assume that blocks are of size PAGE_SIZE.
150149 */
151150 struct swap_extent {
152
- struct list_head list;
151
+ struct rb_node rb_node;
153152 pgoff_t start_page;
154153 pgoff_t nr_pages;
155154 sector_t start_block;
....@@ -170,24 +169,31 @@
170169 SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */
171170 SWP_CONTINUED = (1 << 5), /* swap_map has count continuation */
172171 SWP_BLKDEV = (1 << 6), /* its a block device */
173
- SWP_FILE = (1 << 7), /* set after swap_activate success */
174
- SWP_AREA_DISCARD = (1 << 8), /* single-time swap area discards */
175
- SWP_PAGE_DISCARD = (1 << 9), /* freed swap page-cluster discards */
176
- SWP_STABLE_WRITES = (1 << 10), /* no overwrite PG_writeback pages */
177
- SWP_SYNCHRONOUS_IO = (1 << 11), /* synchronous IO is efficient */
172
+ SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */
173
+ SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */
174
+ SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */
175
+ SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */
176
+ SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */
177
+ SWP_SYNCHRONOUS_IO = (1 << 12), /* synchronous IO is efficient */
178
+ SWP_VALID = (1 << 13), /* swap is valid to be operated on? */
178179 /* add others here before... */
179
- SWP_SCANNING = (1 << 12), /* refcount in scan_swap_map */
180
+ SWP_SCANNING = (1 << 14), /* refcount in scan_swap_map */
180181 };
181182
182183 #define SWAP_CLUSTER_MAX 32UL
183184 #define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX
184185
185
-#define SWAP_MAP_MAX 0x3e /* Max duplication count, in first swap_map */
186
-#define SWAP_MAP_BAD 0x3f /* Note pageblock is bad, in first swap_map */
186
+/* Bit flag in swap_map */
187187 #define SWAP_HAS_CACHE 0x40 /* Flag page is cached, in first swap_map */
188
-#define SWAP_CONT_MAX 0x7f /* Max count, in each swap_map continuation */
189
-#define COUNT_CONTINUED 0x80 /* See swap_map continuation for full count */
190
-#define SWAP_MAP_SHMEM 0xbf /* Owned by shmem/tmpfs, in first swap_map */
188
+#define COUNT_CONTINUED 0x80 /* Flag swap_map continuation for full count */
189
+
190
+/* Special value in first swap_map */
191
+#define SWAP_MAP_MAX 0x3e /* Max count */
192
+#define SWAP_MAP_BAD 0x3f /* Note page is bad */
193
+#define SWAP_MAP_SHMEM 0xbf /* Owned by shmem/tmpfs */
194
+
195
+/* Special value in each swap_map continuation */
196
+#define SWAP_CONT_MAX 0x7f /* Max count */
191197
192198 /*
193199 * We use this to track usage of a cluster. A cluster is a block of swap disk
....@@ -246,9 +252,9 @@
246252 unsigned int inuse_pages; /* number of those currently in use */
247253 unsigned int cluster_next; /* likely index for next allocation */
248254 unsigned int cluster_nr; /* countdown to next cluster search */
255
+ unsigned int __percpu *cluster_next_cpu; /*percpu index for next allocation */
249256 struct percpu_cluster __percpu *percpu_cluster; /* per cpu's swap location */
250
- struct swap_extent *curr_swap_extent;
251
- struct swap_extent first_swap_extent;
257
+ struct rb_root swap_extent_root;/* root of the swap extent rbtree */
252258 struct block_device *bdev; /* swap device or bdev of swap file */
253259 struct file *swap_file; /* seldom referenced */
254260 unsigned int old_block_size; /* seldom referenced */
....@@ -275,9 +281,7 @@
275281 */
276282 struct work_struct discard_work; /* discard worker */
277283 struct swap_cluster_list discard_clusters; /* discard clusters list */
278
- unsigned int write_pending;
279
- unsigned int max_writes;
280
- struct plist_node avail_lists[0]; /*
284
+ struct plist_node avail_lists[]; /*
281285 * entries in swap_avail_heads, one
282286 * entry per node.
283287 * Must be last as the number of the
....@@ -309,51 +313,57 @@
309313 };
310314
311315 /* linux/mm/workingset.c */
312
-void *workingset_eviction(struct address_space *mapping, struct page *page);
316
+void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
317
+void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg);
313318 void workingset_refault(struct page *page, void *shadow);
314319 void workingset_activation(struct page *page);
315320
316
-/* Do not use directly, use workingset_lookup_update */
317
-void workingset_update_node(struct radix_tree_node *node);
318
-
319
-/* Returns workingset_update_node() if the mapping has shadow entries. */
320
-#define workingset_lookup_update(mapping) \
321
-({ \
322
- radix_tree_update_node_t __helper = workingset_update_node; \
323
- if (dax_mapping(mapping) || shmem_mapping(mapping)) \
324
- __helper = NULL; \
325
- __helper; \
326
-})
321
+/* Only track the nodes of mappings with shadow entries */
322
+void workingset_update_node(struct xa_node *node);
323
+#define mapping_set_update(xas, mapping) do { \
324
+ if (!dax_mapping(mapping) && !shmem_mapping(mapping)) \
325
+ xas_set_update(xas, workingset_update_node); \
326
+} while (0)
327327
328328 /* linux/mm/page_alloc.c */
329
-extern unsigned long totalram_pages;
330329 extern unsigned long totalreserve_pages;
331330 extern unsigned long nr_free_buffer_pages(void);
332
-extern unsigned long nr_free_pagecache_pages(void);
333331
334332 /* Definition of global_zone_page_state not available yet */
335333 #define nr_free_pages() global_zone_page_state(NR_FREE_PAGES)
336334
337335
338336 /* linux/mm/swap.c */
339
-DECLARE_LOCAL_IRQ_LOCK(swapvec_lock);
337
+extern void lru_note_cost(struct lruvec *lruvec, bool file,
338
+ unsigned int nr_pages);
339
+extern void lru_note_cost_page(struct page *);
340340 extern void lru_cache_add(struct page *);
341
-extern void lru_cache_add_anon(struct page *page);
342
-extern void lru_cache_add_file(struct page *page);
343341 extern void lru_add_page_tail(struct page *page, struct page *page_tail,
344342 struct lruvec *lruvec, struct list_head *head);
345
-extern void activate_page(struct page *);
346343 extern void mark_page_accessed(struct page *);
344
+
345
+extern bool lru_cache_disabled(void);
346
+extern void lru_cache_disable(void);
347
+extern void lru_cache_enable(void);
347348 extern void lru_add_drain(void);
348349 extern void lru_add_drain_cpu(int cpu);
350
+extern void lru_add_drain_cpu_zone(struct zone *zone);
349351 extern void lru_add_drain_all(void);
350352 extern void rotate_reclaimable_page(struct page *page);
351353 extern void deactivate_file_page(struct page *page);
354
+extern void deactivate_page(struct page *page);
352355 extern void mark_page_lazyfree(struct page *page);
356
+extern void mark_page_lazyfree_movetail(struct page *page, bool tail);
353357 extern void swap_setup(void);
354358
355
-extern void lru_cache_add_active_or_unevictable(struct page *page,
356
- struct vm_area_struct *vma);
359
+extern void __lru_cache_add_inactive_or_unevictable(struct page *page,
360
+ unsigned long vma_flags);
361
+
362
+static inline void lru_cache_add_inactive_or_unevictable(struct page *page,
363
+ struct vm_area_struct *vma)
364
+{
365
+ return __lru_cache_add_inactive_or_unevictable(page, vma->vm_flags);
366
+}
357367
358368 /* linux/mm/vmscan.c */
359369 extern unsigned long zone_reclaimable_pages(struct zone *zone);
....@@ -371,8 +381,8 @@
371381 extern unsigned long shrink_all_memory(unsigned long nr_pages);
372382 extern int vm_swappiness;
373383 extern int remove_mapping(struct address_space *mapping, struct page *page);
374
-extern unsigned long vm_total_pages;
375384
385
+extern unsigned long reclaim_pages(struct list_head *page_list);
376386 #ifdef CONFIG_NUMA
377387 extern int node_reclaim_mode;
378388 extern int sysctl_min_unmapped_ratio;
....@@ -381,7 +391,6 @@
381391 #define node_reclaim_mode 0
382392 #endif
383393
384
-extern int page_evictable(struct page *page);
385394 extern void check_move_unevictable_pages(struct pagevec *pvec);
386395
387396 extern int kswapd_run(int nid);
....@@ -415,15 +424,20 @@
415424 extern unsigned long total_swapcache_pages(void);
416425 extern void show_swap_cache_info(void);
417426 extern int add_to_swap(struct page *page);
418
-extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t);
419
-extern int __add_to_swap_cache(struct page *page, swp_entry_t entry);
420
-extern void __delete_from_swap_cache(struct page *);
427
+extern void *get_shadow_from_swap_cache(swp_entry_t entry);
428
+extern int add_to_swap_cache(struct page *page, swp_entry_t entry,
429
+ gfp_t gfp, void **shadowp);
430
+extern void __delete_from_swap_cache(struct page *page,
431
+ swp_entry_t entry, void *shadow);
421432 extern void delete_from_swap_cache(struct page *);
433
+extern void clear_shadow_from_swap_cache(int type, unsigned long begin,
434
+ unsigned long end);
422435 extern void free_page_and_swap_cache(struct page *);
423436 extern void free_pages_and_swap_cache(struct page **, int);
424437 extern struct page *lookup_swap_cache(swp_entry_t entry,
425438 struct vm_area_struct *vma,
426439 unsigned long addr);
440
+struct page *find_get_incore_page(struct address_space *mapping, pgoff_t index);
427441 extern struct page *read_swap_cache_async(swp_entry_t, gfp_t,
428442 struct vm_area_struct *vma, unsigned long addr,
429443 bool do_poll);
....@@ -464,12 +478,13 @@
464478 extern void swap_free(swp_entry_t);
465479 extern void swapcache_free_entries(swp_entry_t *entries, int n);
466480 extern int free_swap_and_cache(swp_entry_t);
467
-extern int swap_type_of(dev_t, sector_t, struct block_device **);
481
+int swap_type_of(dev_t device, sector_t offset);
482
+int find_first_swap(dev_t *device);
468483 extern unsigned int count_swap_pages(int, int);
469484 extern sector_t map_swap_page(struct page *, struct block_device **);
470485 extern sector_t swapdev_block(int, pgoff_t);
471486 extern int page_swapcount(struct page *);
472
-extern int __swap_count(struct swap_info_struct *si, swp_entry_t entry);
487
+extern int __swap_count(swp_entry_t entry);
473488 extern int __swp_swapcount(swp_entry_t entry);
474489 extern int swp_swapcount(swp_entry_t entry);
475490 extern struct swap_info_struct *page_swap_info(struct page *);
....@@ -479,6 +494,13 @@
479494 struct backing_dev_info;
480495 extern int init_swap_address_space(unsigned int type, unsigned long nr_pages);
481496 extern void exit_swap_address_space(unsigned int type);
497
+extern struct swap_info_struct *get_swap_device(swp_entry_t entry);
498
+sector_t swap_page_sector(struct page *page);
499
+
500
+static inline void put_swap_device(struct swap_info_struct *si)
501
+{
502
+ rcu_read_unlock();
503
+}
482504
483505 #else /* CONFIG_SWAP */
484506
....@@ -560,22 +582,39 @@
560582 return NULL;
561583 }
562584
585
+static inline
586
+struct page *find_get_incore_page(struct address_space *mapping, pgoff_t index)
587
+{
588
+ return find_get_page(mapping, index);
589
+}
590
+
563591 static inline int add_to_swap(struct page *page)
564592 {
565593 return 0;
566594 }
567595
596
+static inline void *get_shadow_from_swap_cache(swp_entry_t entry)
597
+{
598
+ return NULL;
599
+}
600
+
568601 static inline int add_to_swap_cache(struct page *page, swp_entry_t entry,
569
- gfp_t gfp_mask)
602
+ gfp_t gfp_mask, void **shadowp)
570603 {
571604 return -1;
572605 }
573606
574
-static inline void __delete_from_swap_cache(struct page *page)
607
+static inline void __delete_from_swap_cache(struct page *page,
608
+ swp_entry_t entry, void *shadow)
575609 {
576610 }
577611
578612 static inline void delete_from_swap_cache(struct page *page)
613
+{
614
+}
615
+
616
+static inline void clear_shadow_from_swap_cache(int type, unsigned long begin,
617
+ unsigned long end)
579618 {
580619 }
581620
....@@ -584,7 +623,7 @@
584623 return 0;
585624 }
586625
587
-static inline int __swap_count(struct swap_info_struct *si, swp_entry_t entry)
626
+static inline int __swap_count(swp_entry_t entry)
588627 {
589628 return 0;
590629 }
....@@ -633,7 +672,7 @@
633672 return vm_swappiness;
634673
635674 /* root ? */
636
- if (mem_cgroup_disabled() || !memcg->css.parent)
675
+ if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg))
637676 return vm_swappiness;
638677
639678 return memcg->swappiness;
....@@ -646,19 +685,37 @@
646685 #endif
647686
648687 #if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
649
-extern void mem_cgroup_throttle_swaprate(struct mem_cgroup *memcg, int node,
650
- gfp_t gfp_mask);
688
+extern void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask);
689
+static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
690
+{
691
+ if (mem_cgroup_disabled())
692
+ return;
693
+ __cgroup_throttle_swaprate(page, gfp_mask);
694
+}
651695 #else
652
-static inline void mem_cgroup_throttle_swaprate(struct mem_cgroup *memcg,
653
- int node, gfp_t gfp_mask)
696
+static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
654697 {
655698 }
656699 #endif
657700
658701 #ifdef CONFIG_MEMCG_SWAP
659702 extern void mem_cgroup_swapout(struct page *page, swp_entry_t entry);
660
-extern int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
661
-extern void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
703
+extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
704
+static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
705
+{
706
+ if (mem_cgroup_disabled())
707
+ return 0;
708
+ return __mem_cgroup_try_charge_swap(page, entry);
709
+}
710
+
711
+extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
712
+static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
713
+{
714
+ if (mem_cgroup_disabled())
715
+ return;
716
+ __mem_cgroup_uncharge_swap(entry, nr_pages);
717
+}
718
+
662719 extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
663720 extern bool mem_cgroup_swap_full(struct page *page);
664721 #else