.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * mm_init.c - Memory initialisation verification and debugging |
---|
3 | 4 | * |
---|
.. | .. |
---|
12 | 13 | #include <linux/memory.h> |
---|
13 | 14 | #include <linux/notifier.h> |
---|
14 | 15 | #include <linux/sched.h> |
---|
| 16 | +#include <linux/mman.h> |
---|
15 | 17 | #include "internal.h" |
---|
16 | 18 | |
---|
17 | 19 | #ifdef CONFIG_DEBUG_MEMORY_INIT |
---|
.. | .. |
---|
36 | 38 | struct zonelist *zonelist; |
---|
37 | 39 | int i, listid, zoneid; |
---|
38 | 40 | |
---|
39 | | - BUG_ON(MAX_ZONELISTS > 2); |
---|
| 41 | + BUILD_BUG_ON(MAX_ZONELISTS > 2); |
---|
40 | 42 | for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { |
---|
41 | 43 | |
---|
42 | 44 | /* Identify the zone and nodelist */ |
---|
.. | .. |
---|
66 | 68 | unsigned long or_mask, add_mask; |
---|
67 | 69 | |
---|
68 | 70 | shift = 8 * sizeof(unsigned long); |
---|
69 | | - width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - LAST_CPUPID_SHIFT; |
---|
| 71 | + width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH |
---|
| 72 | + - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; |
---|
70 | 73 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", |
---|
71 | | - "Section %d Node %d Zone %d Lastcpupid %d Flags %d\n", |
---|
| 74 | + "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", |
---|
72 | 75 | SECTIONS_WIDTH, |
---|
73 | 76 | NODES_WIDTH, |
---|
74 | 77 | ZONES_WIDTH, |
---|
75 | 78 | LAST_CPUPID_WIDTH, |
---|
| 79 | + KASAN_TAG_WIDTH, |
---|
76 | 80 | NR_PAGEFLAGS); |
---|
77 | 81 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", |
---|
78 | | - "Section %d Node %d Zone %d Lastcpupid %d\n", |
---|
| 82 | + "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", |
---|
79 | 83 | SECTIONS_SHIFT, |
---|
80 | 84 | NODES_SHIFT, |
---|
81 | 85 | ZONES_SHIFT, |
---|
82 | | - LAST_CPUPID_SHIFT); |
---|
| 86 | + LAST_CPUPID_SHIFT, |
---|
| 87 | + KASAN_TAG_WIDTH); |
---|
83 | 88 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts", |
---|
84 | | - "Section %lu Node %lu Zone %lu Lastcpupid %lu\n", |
---|
| 89 | + "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n", |
---|
85 | 90 | (unsigned long)SECTIONS_PGSHIFT, |
---|
86 | 91 | (unsigned long)NODES_PGSHIFT, |
---|
87 | 92 | (unsigned long)ZONES_PGSHIFT, |
---|
88 | | - (unsigned long)LAST_CPUPID_PGSHIFT); |
---|
| 93 | + (unsigned long)LAST_CPUPID_PGSHIFT, |
---|
| 94 | + (unsigned long)KASAN_TAG_PGSHIFT); |
---|
89 | 95 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid", |
---|
90 | 96 | "Node/Zone ID: %lu -> %lu\n", |
---|
91 | 97 | (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT), |
---|
.. | .. |
---|
139 | 145 | #ifdef CONFIG_SMP |
---|
140 | 146 | s32 vm_committed_as_batch = 32; |
---|
141 | 147 | |
---|
142 | | -static void __meminit mm_compute_batch(void) |
---|
| 148 | +void mm_compute_batch(int overcommit_policy) |
---|
143 | 149 | { |
---|
144 | 150 | u64 memsized_batch; |
---|
145 | 151 | s32 nr = num_present_cpus(); |
---|
146 | 152 | s32 batch = max_t(s32, nr*2, 32); |
---|
| 153 | + unsigned long ram_pages = totalram_pages(); |
---|
147 | 154 | |
---|
148 | | - /* batch size set to 0.4% of (total memory/#cpus), or max int32 */ |
---|
149 | | - memsized_batch = min_t(u64, (totalram_pages/nr)/256, 0x7fffffff); |
---|
| 155 | + /* |
---|
| 156 | + * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of |
---|
| 157 | + * (total memory/#cpus), and lift it to 25% for other policies |
---|
| 158 | + * to easy the possible lock contention for percpu_counter |
---|
| 159 | + * vm_committed_as, while the max limit is INT_MAX |
---|
| 160 | + */ |
---|
| 161 | + if (overcommit_policy == OVERCOMMIT_NEVER) |
---|
| 162 | + memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX); |
---|
| 163 | + else |
---|
| 164 | + memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX); |
---|
150 | 165 | |
---|
151 | 166 | vm_committed_as_batch = max_t(s32, memsized_batch, batch); |
---|
152 | 167 | } |
---|
.. | .. |
---|
157 | 172 | switch (action) { |
---|
158 | 173 | case MEM_ONLINE: |
---|
159 | 174 | case MEM_OFFLINE: |
---|
160 | | - mm_compute_batch(); |
---|
| 175 | + mm_compute_batch(sysctl_overcommit_memory); |
---|
161 | 176 | default: |
---|
162 | 177 | break; |
---|
163 | 178 | } |
---|
.. | .. |
---|
171 | 186 | |
---|
172 | 187 | static int __init mm_compute_batch_init(void) |
---|
173 | 188 | { |
---|
174 | | - mm_compute_batch(); |
---|
| 189 | + mm_compute_batch(sysctl_overcommit_memory); |
---|
175 | 190 | register_hotmemory_notifier(&compute_batch_nb); |
---|
176 | 191 | |
---|
177 | 192 | return 0; |
---|