| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * mm_init.c - Memory initialisation verification and debugging |
|---|
| 3 | 4 | * |
|---|
| .. | .. |
|---|
| 12 | 13 | #include <linux/memory.h> |
|---|
| 13 | 14 | #include <linux/notifier.h> |
|---|
| 14 | 15 | #include <linux/sched.h> |
|---|
| 16 | +#include <linux/mman.h> |
|---|
| 15 | 17 | #include "internal.h" |
|---|
| 16 | 18 | |
|---|
| 17 | 19 | #ifdef CONFIG_DEBUG_MEMORY_INIT |
|---|
| .. | .. |
|---|
| 36 | 38 | struct zonelist *zonelist; |
|---|
| 37 | 39 | int i, listid, zoneid; |
|---|
| 38 | 40 | |
|---|
| 39 | | - BUG_ON(MAX_ZONELISTS > 2); |
|---|
| 41 | + BUILD_BUG_ON(MAX_ZONELISTS > 2); |
|---|
| 40 | 42 | for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { |
|---|
| 41 | 43 | |
|---|
| 42 | 44 | /* Identify the zone and nodelist */ |
|---|
| .. | .. |
|---|
| 66 | 68 | unsigned long or_mask, add_mask; |
|---|
| 67 | 69 | |
|---|
| 68 | 70 | shift = 8 * sizeof(unsigned long); |
|---|
| 69 | | - width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - LAST_CPUPID_SHIFT; |
|---|
| 71 | + width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH |
|---|
| 72 | + - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; |
|---|
| 70 | 73 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", |
|---|
| 71 | | - "Section %d Node %d Zone %d Lastcpupid %d Flags %d\n", |
|---|
| 74 | + "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", |
|---|
| 72 | 75 | SECTIONS_WIDTH, |
|---|
| 73 | 76 | NODES_WIDTH, |
|---|
| 74 | 77 | ZONES_WIDTH, |
|---|
| 75 | 78 | LAST_CPUPID_WIDTH, |
|---|
| 79 | + KASAN_TAG_WIDTH, |
|---|
| 76 | 80 | NR_PAGEFLAGS); |
|---|
| 77 | 81 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", |
|---|
| 78 | | - "Section %d Node %d Zone %d Lastcpupid %d\n", |
|---|
| 82 | + "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", |
|---|
| 79 | 83 | SECTIONS_SHIFT, |
|---|
| 80 | 84 | NODES_SHIFT, |
|---|
| 81 | 85 | ZONES_SHIFT, |
|---|
| 82 | | - LAST_CPUPID_SHIFT); |
|---|
| 86 | + LAST_CPUPID_SHIFT, |
|---|
| 87 | + KASAN_TAG_WIDTH); |
|---|
| 83 | 88 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts", |
|---|
| 84 | | - "Section %lu Node %lu Zone %lu Lastcpupid %lu\n", |
|---|
| 89 | + "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n", |
|---|
| 85 | 90 | (unsigned long)SECTIONS_PGSHIFT, |
|---|
| 86 | 91 | (unsigned long)NODES_PGSHIFT, |
|---|
| 87 | 92 | (unsigned long)ZONES_PGSHIFT, |
|---|
| 88 | | - (unsigned long)LAST_CPUPID_PGSHIFT); |
|---|
| 93 | + (unsigned long)LAST_CPUPID_PGSHIFT, |
|---|
| 94 | + (unsigned long)KASAN_TAG_PGSHIFT); |
|---|
| 89 | 95 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid", |
|---|
| 90 | 96 | "Node/Zone ID: %lu -> %lu\n", |
|---|
| 91 | 97 | (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT), |
|---|
| .. | .. |
|---|
| 139 | 145 | #ifdef CONFIG_SMP |
|---|
| 140 | 146 | s32 vm_committed_as_batch = 32; |
|---|
| 141 | 147 | |
|---|
| 142 | | -static void __meminit mm_compute_batch(void) |
|---|
| 148 | +void mm_compute_batch(int overcommit_policy) |
|---|
| 143 | 149 | { |
|---|
| 144 | 150 | u64 memsized_batch; |
|---|
| 145 | 151 | s32 nr = num_present_cpus(); |
|---|
| 146 | 152 | s32 batch = max_t(s32, nr*2, 32); |
|---|
| 153 | + unsigned long ram_pages = totalram_pages(); |
|---|
| 147 | 154 | |
|---|
| 148 | | - /* batch size set to 0.4% of (total memory/#cpus), or max int32 */ |
|---|
| 149 | | - memsized_batch = min_t(u64, (totalram_pages/nr)/256, 0x7fffffff); |
|---|
| 155 | + /* |
|---|
| 156 | + * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of |
|---|
| 157 | + * (total memory/#cpus), and lift it to 25% for other policies |
|---|
| 158 | + * to easy the possible lock contention for percpu_counter |
|---|
| 159 | + * vm_committed_as, while the max limit is INT_MAX |
|---|
| 160 | + */ |
|---|
| 161 | + if (overcommit_policy == OVERCOMMIT_NEVER) |
|---|
| 162 | + memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX); |
|---|
| 163 | + else |
|---|
| 164 | + memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX); |
|---|
| 150 | 165 | |
|---|
| 151 | 166 | vm_committed_as_batch = max_t(s32, memsized_batch, batch); |
|---|
| 152 | 167 | } |
|---|
| .. | .. |
|---|
| 157 | 172 | switch (action) { |
|---|
| 158 | 173 | case MEM_ONLINE: |
|---|
| 159 | 174 | case MEM_OFFLINE: |
|---|
| 160 | | - mm_compute_batch(); |
|---|
| 175 | + mm_compute_batch(sysctl_overcommit_memory); |
|---|
| 161 | 176 | default: |
|---|
| 162 | 177 | break; |
|---|
| 163 | 178 | } |
|---|
| .. | .. |
|---|
| 171 | 186 | |
|---|
| 172 | 187 | static int __init mm_compute_batch_init(void) |
|---|
| 173 | 188 | { |
|---|
| 174 | | - mm_compute_batch(); |
|---|
| 189 | + mm_compute_batch(sysctl_overcommit_memory); |
|---|
| 175 | 190 | register_hotmemory_notifier(&compute_batch_nb); |
|---|
| 176 | 191 | |
|---|
| 177 | 192 | return 0; |
|---|