| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * linux/mm/vmstat.c |
|---|
| 3 | 4 | * |
|---|
| .. | .. |
|---|
| 75 | 76 | static DEFINE_MUTEX(vm_numa_stat_lock); |
|---|
| 76 | 77 | |
|---|
| 77 | 78 | int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write, |
|---|
| 78 | | - void __user *buffer, size_t *length, loff_t *ppos) |
|---|
| 79 | + void *buffer, size_t *length, loff_t *ppos) |
|---|
| 79 | 80 | { |
|---|
| 80 | 81 | int ret, oldval; |
|---|
| 81 | 82 | |
|---|
| .. | .. |
|---|
| 227 | 228 | * 125 1024 10 16-32 GB 9 |
|---|
| 228 | 229 | */ |
|---|
| 229 | 230 | |
|---|
| 230 | | - mem = zone->managed_pages >> (27 - PAGE_SHIFT); |
|---|
| 231 | + mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); |
|---|
| 231 | 232 | |
|---|
| 232 | 233 | threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); |
|---|
| 233 | 234 | |
|---|
| .. | .. |
|---|
| 320 | 321 | long x; |
|---|
| 321 | 322 | long t; |
|---|
| 322 | 323 | |
|---|
| 323 | | - preempt_disable_rt(); |
|---|
| 324 | 324 | x = delta + __this_cpu_read(*p); |
|---|
| 325 | 325 | |
|---|
| 326 | 326 | t = __this_cpu_read(pcp->stat_threshold); |
|---|
| 327 | 327 | |
|---|
| 328 | | - if (unlikely(x > t || x < -t)) { |
|---|
| 328 | + if (unlikely(abs(x) > t)) { |
|---|
| 329 | 329 | zone_page_state_add(x, zone, item); |
|---|
| 330 | 330 | x = 0; |
|---|
| 331 | 331 | } |
|---|
| 332 | 332 | __this_cpu_write(*p, x); |
|---|
| 333 | | - preempt_enable_rt(); |
|---|
| 334 | 333 | } |
|---|
| 335 | 334 | EXPORT_SYMBOL(__mod_zone_page_state); |
|---|
| 336 | 335 | |
|---|
| .. | .. |
|---|
| 342 | 341 | long x; |
|---|
| 343 | 342 | long t; |
|---|
| 344 | 343 | |
|---|
| 345 | | - preempt_disable_rt(); |
|---|
| 344 | + if (vmstat_item_in_bytes(item)) { |
|---|
| 345 | + VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); |
|---|
| 346 | + delta >>= PAGE_SHIFT; |
|---|
| 347 | + } |
|---|
| 348 | + |
|---|
| 346 | 349 | x = delta + __this_cpu_read(*p); |
|---|
| 347 | 350 | |
|---|
| 348 | 351 | t = __this_cpu_read(pcp->stat_threshold); |
|---|
| 349 | 352 | |
|---|
| 350 | | - if (unlikely(x > t || x < -t)) { |
|---|
| 353 | + if (unlikely(abs(x) > t)) { |
|---|
| 351 | 354 | node_page_state_add(x, pgdat, item); |
|---|
| 352 | 355 | x = 0; |
|---|
| 353 | 356 | } |
|---|
| 354 | 357 | __this_cpu_write(*p, x); |
|---|
| 355 | | - preempt_enable_rt(); |
|---|
| 356 | 358 | } |
|---|
| 357 | 359 | EXPORT_SYMBOL(__mod_node_page_state); |
|---|
| 358 | 360 | |
|---|
| .. | .. |
|---|
| 385 | 387 | s8 __percpu *p = pcp->vm_stat_diff + item; |
|---|
| 386 | 388 | s8 v, t; |
|---|
| 387 | 389 | |
|---|
| 388 | | - preempt_disable_rt(); |
|---|
| 389 | 390 | v = __this_cpu_inc_return(*p); |
|---|
| 390 | 391 | t = __this_cpu_read(pcp->stat_threshold); |
|---|
| 391 | 392 | if (unlikely(v > t)) { |
|---|
| .. | .. |
|---|
| 394 | 395 | zone_page_state_add(v + overstep, zone, item); |
|---|
| 395 | 396 | __this_cpu_write(*p, -overstep); |
|---|
| 396 | 397 | } |
|---|
| 397 | | - preempt_enable_rt(); |
|---|
| 398 | 398 | } |
|---|
| 399 | 399 | |
|---|
| 400 | 400 | void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) |
|---|
| .. | .. |
|---|
| 403 | 403 | s8 __percpu *p = pcp->vm_node_stat_diff + item; |
|---|
| 404 | 404 | s8 v, t; |
|---|
| 405 | 405 | |
|---|
| 406 | | - preempt_disable_rt(); |
|---|
| 406 | + VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); |
|---|
| 407 | + |
|---|
| 407 | 408 | v = __this_cpu_inc_return(*p); |
|---|
| 408 | 409 | t = __this_cpu_read(pcp->stat_threshold); |
|---|
| 409 | 410 | if (unlikely(v > t)) { |
|---|
| .. | .. |
|---|
| 412 | 413 | node_page_state_add(v + overstep, pgdat, item); |
|---|
| 413 | 414 | __this_cpu_write(*p, -overstep); |
|---|
| 414 | 415 | } |
|---|
| 415 | | - preempt_enable_rt(); |
|---|
| 416 | 416 | } |
|---|
| 417 | 417 | |
|---|
| 418 | 418 | void __inc_zone_page_state(struct page *page, enum zone_stat_item item) |
|---|
| .. | .. |
|---|
| 433 | 433 | s8 __percpu *p = pcp->vm_stat_diff + item; |
|---|
| 434 | 434 | s8 v, t; |
|---|
| 435 | 435 | |
|---|
| 436 | | - preempt_disable_rt(); |
|---|
| 437 | 436 | v = __this_cpu_dec_return(*p); |
|---|
| 438 | 437 | t = __this_cpu_read(pcp->stat_threshold); |
|---|
| 439 | 438 | if (unlikely(v < - t)) { |
|---|
| .. | .. |
|---|
| 442 | 441 | zone_page_state_add(v - overstep, zone, item); |
|---|
| 443 | 442 | __this_cpu_write(*p, overstep); |
|---|
| 444 | 443 | } |
|---|
| 445 | | - preempt_enable_rt(); |
|---|
| 446 | 444 | } |
|---|
| 447 | 445 | |
|---|
| 448 | 446 | void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) |
|---|
| .. | .. |
|---|
| 451 | 449 | s8 __percpu *p = pcp->vm_node_stat_diff + item; |
|---|
| 452 | 450 | s8 v, t; |
|---|
| 453 | 451 | |
|---|
| 454 | | - preempt_disable_rt(); |
|---|
| 452 | + VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); |
|---|
| 453 | + |
|---|
| 455 | 454 | v = __this_cpu_dec_return(*p); |
|---|
| 456 | 455 | t = __this_cpu_read(pcp->stat_threshold); |
|---|
| 457 | 456 | if (unlikely(v < - t)) { |
|---|
| .. | .. |
|---|
| 460 | 459 | node_page_state_add(v - overstep, pgdat, item); |
|---|
| 461 | 460 | __this_cpu_write(*p, overstep); |
|---|
| 462 | 461 | } |
|---|
| 463 | | - preempt_enable_rt(); |
|---|
| 464 | 462 | } |
|---|
| 465 | 463 | |
|---|
| 466 | 464 | void __dec_zone_page_state(struct page *page, enum zone_stat_item item) |
|---|
| .. | .. |
|---|
| 513 | 511 | o = this_cpu_read(*p); |
|---|
| 514 | 512 | n = delta + o; |
|---|
| 515 | 513 | |
|---|
| 516 | | - if (n > t || n < -t) { |
|---|
| 514 | + if (abs(n) > t) { |
|---|
| 517 | 515 | int os = overstep_mode * (t >> 1) ; |
|---|
| 518 | 516 | |
|---|
| 519 | 517 | /* Overflow must be added to zone counters */ |
|---|
| .. | .. |
|---|
| 552 | 550 | s8 __percpu *p = pcp->vm_node_stat_diff + item; |
|---|
| 553 | 551 | long o, n, t, z; |
|---|
| 554 | 552 | |
|---|
| 553 | + if (vmstat_item_in_bytes(item)) { |
|---|
| 554 | + VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); |
|---|
| 555 | + delta >>= PAGE_SHIFT; |
|---|
| 556 | + } |
|---|
| 557 | + |
|---|
| 555 | 558 | do { |
|---|
| 556 | 559 | z = 0; /* overflow to node counters */ |
|---|
| 557 | 560 | |
|---|
| .. | .. |
|---|
| 570 | 573 | o = this_cpu_read(*p); |
|---|
| 571 | 574 | n = delta + o; |
|---|
| 572 | 575 | |
|---|
| 573 | | - if (n > t || n < -t) { |
|---|
| 576 | + if (abs(n) > t) { |
|---|
| 574 | 577 | int os = overstep_mode * (t >> 1) ; |
|---|
| 575 | 578 | |
|---|
| 576 | 579 | /* Overflow must be added to node counters */ |
|---|
| .. | .. |
|---|
| 1000 | 1003 | /* |
|---|
| 1001 | 1004 | * Determine the per node value of a stat item. |
|---|
| 1002 | 1005 | */ |
|---|
| 1003 | | -unsigned long node_page_state(struct pglist_data *pgdat, |
|---|
| 1004 | | - enum node_stat_item item) |
|---|
| 1006 | +unsigned long node_page_state_pages(struct pglist_data *pgdat, |
|---|
| 1007 | + enum node_stat_item item) |
|---|
| 1005 | 1008 | { |
|---|
| 1006 | 1009 | long x = atomic_long_read(&pgdat->vm_stat[item]); |
|---|
| 1007 | 1010 | #ifdef CONFIG_SMP |
|---|
| .. | .. |
|---|
| 1009 | 1012 | x = 0; |
|---|
| 1010 | 1013 | #endif |
|---|
| 1011 | 1014 | return x; |
|---|
| 1015 | +} |
|---|
| 1016 | + |
|---|
| 1017 | +unsigned long node_page_state(struct pglist_data *pgdat, |
|---|
| 1018 | + enum node_stat_item item) |
|---|
| 1019 | +{ |
|---|
| 1020 | + VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); |
|---|
| 1021 | + |
|---|
| 1022 | + return node_page_state_pages(pgdat, item); |
|---|
| 1012 | 1023 | } |
|---|
| 1013 | 1024 | #endif |
|---|
| 1014 | 1025 | |
|---|
| .. | .. |
|---|
| 1085 | 1096 | return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); |
|---|
| 1086 | 1097 | } |
|---|
| 1087 | 1098 | |
|---|
| 1099 | +/* |
|---|
| 1100 | + * Calculates external fragmentation within a zone wrt the given order. |
|---|
| 1101 | + * It is defined as the percentage of pages found in blocks of size |
|---|
| 1102 | + * less than 1 << order. It returns values in range [0, 100]. |
|---|
| 1103 | + */ |
|---|
| 1104 | +unsigned int extfrag_for_order(struct zone *zone, unsigned int order) |
|---|
| 1105 | +{ |
|---|
| 1106 | + struct contig_page_info info; |
|---|
| 1107 | + |
|---|
| 1108 | + fill_contig_page_info(zone, order, &info); |
|---|
| 1109 | + if (info.free_pages == 0) |
|---|
| 1110 | + return 0; |
|---|
| 1111 | + |
|---|
| 1112 | + return div_u64((info.free_pages - |
|---|
| 1113 | + (info.free_blocks_suitable << order)) * 100, |
|---|
| 1114 | + info.free_pages); |
|---|
| 1115 | +} |
|---|
| 1116 | + |
|---|
| 1088 | 1117 | /* Same as __fragmentation index but allocs contig_page_info on stack */ |
|---|
| 1089 | 1118 | int fragmentation_index(struct zone *zone, unsigned int order) |
|---|
| 1090 | 1119 | { |
|---|
| .. | .. |
|---|
| 1095 | 1124 | } |
|---|
| 1096 | 1125 | #endif |
|---|
| 1097 | 1126 | |
|---|
| 1098 | | -#if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || defined(CONFIG_NUMA) |
|---|
| 1127 | +#if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \ |
|---|
| 1128 | + defined(CONFIG_NUMA) || defined(CONFIG_MEMCG) |
|---|
| 1099 | 1129 | #ifdef CONFIG_ZONE_DMA |
|---|
| 1100 | 1130 | #define TEXT_FOR_DMA(xx) xx "_dma", |
|---|
| 1101 | 1131 | #else |
|---|
| .. | .. |
|---|
| 1118 | 1148 | TEXT_FOR_HIGHMEM(xx) xx "_movable", |
|---|
| 1119 | 1149 | |
|---|
| 1120 | 1150 | const char * const vmstat_text[] = { |
|---|
| 1121 | | - /* enum zone_stat_item countes */ |
|---|
| 1151 | + /* enum zone_stat_item counters */ |
|---|
| 1122 | 1152 | "nr_free_pages", |
|---|
| 1123 | 1153 | "nr_zone_inactive_anon", |
|---|
| 1124 | 1154 | "nr_zone_active_anon", |
|---|
| .. | .. |
|---|
| 1128 | 1158 | "nr_zone_write_pending", |
|---|
| 1129 | 1159 | "nr_mlock", |
|---|
| 1130 | 1160 | "nr_page_table_pages", |
|---|
| 1131 | | - "nr_kernel_stack", |
|---|
| 1132 | | -#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) |
|---|
| 1133 | | - "nr_shadow_call_stack_bytes", |
|---|
| 1134 | | -#endif |
|---|
| 1135 | 1161 | "nr_bounce", |
|---|
| 1136 | | -#if IS_ENABLED(CONFIG_ZSMALLOC) |
|---|
| 1137 | 1162 | "nr_zspages", |
|---|
| 1138 | | -#endif |
|---|
| 1139 | 1163 | "nr_free_cma", |
|---|
| 1140 | 1164 | |
|---|
| 1141 | 1165 | /* enum numa_stat_item counters */ |
|---|
| .. | .. |
|---|
| 1148 | 1172 | "numa_other", |
|---|
| 1149 | 1173 | #endif |
|---|
| 1150 | 1174 | |
|---|
| 1151 | | - /* Node-based counters */ |
|---|
| 1175 | + /* enum node_stat_item counters */ |
|---|
| 1152 | 1176 | "nr_inactive_anon", |
|---|
| 1153 | 1177 | "nr_active_anon", |
|---|
| 1154 | 1178 | "nr_inactive_file", |
|---|
| .. | .. |
|---|
| 1158 | 1182 | "nr_slab_unreclaimable", |
|---|
| 1159 | 1183 | "nr_isolated_anon", |
|---|
| 1160 | 1184 | "nr_isolated_file", |
|---|
| 1161 | | - "workingset_refault", |
|---|
| 1162 | | - "workingset_activate", |
|---|
| 1163 | | - "workingset_restore", |
|---|
| 1185 | + "workingset_nodes", |
|---|
| 1186 | + "workingset_refault_anon", |
|---|
| 1187 | + "workingset_refault_file", |
|---|
| 1188 | + "workingset_activate_anon", |
|---|
| 1189 | + "workingset_activate_file", |
|---|
| 1190 | + "workingset_restore_anon", |
|---|
| 1191 | + "workingset_restore_file", |
|---|
| 1164 | 1192 | "workingset_nodereclaim", |
|---|
| 1165 | 1193 | "nr_anon_pages", |
|---|
| 1166 | 1194 | "nr_mapped", |
|---|
| .. | .. |
|---|
| 1171 | 1199 | "nr_shmem", |
|---|
| 1172 | 1200 | "nr_shmem_hugepages", |
|---|
| 1173 | 1201 | "nr_shmem_pmdmapped", |
|---|
| 1202 | + "nr_file_hugepages", |
|---|
| 1203 | + "nr_file_pmdmapped", |
|---|
| 1174 | 1204 | "nr_anon_transparent_hugepages", |
|---|
| 1175 | | - "nr_unstable", |
|---|
| 1176 | 1205 | "nr_vmscan_write", |
|---|
| 1177 | 1206 | "nr_vmscan_immediate_reclaim", |
|---|
| 1178 | 1207 | "nr_dirtied", |
|---|
| 1179 | 1208 | "nr_written", |
|---|
| 1180 | 1209 | "nr_kernel_misc_reclaimable", |
|---|
| 1181 | | - "nr_unreclaimable_pages", |
|---|
| 1210 | + "nr_foll_pin_acquired", |
|---|
| 1211 | + "nr_foll_pin_released", |
|---|
| 1212 | + "nr_kernel_stack", |
|---|
| 1213 | +#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) |
|---|
| 1214 | + "nr_shadow_call_stack", |
|---|
| 1215 | +#endif |
|---|
| 1182 | 1216 | |
|---|
| 1183 | | - |
|---|
| 1184 | | - "nr_ion_heap", |
|---|
| 1185 | | - "nr_ion_heap_pool", |
|---|
| 1186 | | - "nr_gpu_heap", |
|---|
| 1187 | 1217 | /* enum writeback_stat_item counters */ |
|---|
| 1188 | 1218 | "nr_dirty_threshold", |
|---|
| 1189 | 1219 | "nr_dirty_background_threshold", |
|---|
| 1190 | 1220 | |
|---|
| 1191 | | -#ifdef CONFIG_VM_EVENT_COUNTERS |
|---|
| 1221 | +#if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG) |
|---|
| 1192 | 1222 | /* enum vm_event_item counters */ |
|---|
| 1193 | 1223 | "pgpgin", |
|---|
| 1194 | 1224 | "pgpgout", |
|---|
| 1195 | | - "pgpgoutclean", |
|---|
| 1196 | 1225 | "pswpin", |
|---|
| 1197 | 1226 | "pswpout", |
|---|
| 1198 | 1227 | |
|---|
| .. | .. |
|---|
| 1210 | 1239 | "pglazyfreed", |
|---|
| 1211 | 1240 | |
|---|
| 1212 | 1241 | "pgrefill", |
|---|
| 1242 | + "pgreuse", |
|---|
| 1213 | 1243 | "pgsteal_kswapd", |
|---|
| 1214 | 1244 | "pgsteal_direct", |
|---|
| 1215 | 1245 | "pgscan_kswapd", |
|---|
| 1216 | 1246 | "pgscan_direct", |
|---|
| 1217 | 1247 | "pgscan_direct_throttle", |
|---|
| 1248 | + "pgscan_anon", |
|---|
| 1249 | + "pgscan_file", |
|---|
| 1250 | + "pgsteal_anon", |
|---|
| 1251 | + "pgsteal_file", |
|---|
| 1218 | 1252 | |
|---|
| 1219 | 1253 | #ifdef CONFIG_NUMA |
|---|
| 1220 | 1254 | "zone_reclaim_failed", |
|---|
| .. | .. |
|---|
| 1242 | 1276 | #ifdef CONFIG_MIGRATION |
|---|
| 1243 | 1277 | "pgmigrate_success", |
|---|
| 1244 | 1278 | "pgmigrate_fail", |
|---|
| 1279 | + "thp_migration_success", |
|---|
| 1280 | + "thp_migration_fail", |
|---|
| 1281 | + "thp_migration_split", |
|---|
| 1245 | 1282 | #endif |
|---|
| 1246 | 1283 | #ifdef CONFIG_COMPACTION |
|---|
| 1247 | 1284 | "compact_migrate_scanned", |
|---|
| .. | .. |
|---|
| 1259 | 1296 | "htlb_buddy_alloc_success", |
|---|
| 1260 | 1297 | "htlb_buddy_alloc_fail", |
|---|
| 1261 | 1298 | #endif |
|---|
| 1299 | +#ifdef CONFIG_CMA |
|---|
| 1300 | + "cma_alloc_success", |
|---|
| 1301 | + "cma_alloc_fail", |
|---|
| 1302 | +#endif |
|---|
| 1262 | 1303 | "unevictable_pgs_culled", |
|---|
| 1263 | 1304 | "unevictable_pgs_scanned", |
|---|
| 1264 | 1305 | "unevictable_pgs_rescued", |
|---|
| .. | .. |
|---|
| 1270 | 1311 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
|---|
| 1271 | 1312 | "thp_fault_alloc", |
|---|
| 1272 | 1313 | "thp_fault_fallback", |
|---|
| 1314 | + "thp_fault_fallback_charge", |
|---|
| 1273 | 1315 | "thp_collapse_alloc", |
|---|
| 1274 | 1316 | "thp_collapse_alloc_failed", |
|---|
| 1275 | 1317 | "thp_file_alloc", |
|---|
| 1318 | + "thp_file_fallback", |
|---|
| 1319 | + "thp_file_fallback_charge", |
|---|
| 1276 | 1320 | "thp_file_mapped", |
|---|
| 1277 | 1321 | "thp_split_page", |
|---|
| 1278 | 1322 | "thp_split_page_failed", |
|---|
| .. | .. |
|---|
| 1308 | 1352 | "swap_ra", |
|---|
| 1309 | 1353 | "swap_ra_hit", |
|---|
| 1310 | 1354 | #endif |
|---|
| 1311 | | -#endif /* CONFIG_VM_EVENTS_COUNTERS */ |
|---|
| 1355 | +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT |
|---|
| 1356 | + "speculative_pgfault", |
|---|
| 1357 | + "speculative_pgfault_file" |
|---|
| 1358 | +#endif |
|---|
| 1359 | +#endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */ |
|---|
| 1312 | 1360 | }; |
|---|
| 1313 | | -#endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA */ |
|---|
| 1361 | +#endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ |
|---|
| 1314 | 1362 | |
|---|
| 1315 | 1363 | #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ |
|---|
| 1316 | 1364 | defined(CONFIG_PROC_FS) |
|---|
| .. | .. |
|---|
| 1400 | 1448 | unsigned long freecount = 0; |
|---|
| 1401 | 1449 | struct free_area *area; |
|---|
| 1402 | 1450 | struct list_head *curr; |
|---|
| 1451 | + bool overflow = false; |
|---|
| 1403 | 1452 | |
|---|
| 1404 | 1453 | area = &(zone->free_area[order]); |
|---|
| 1405 | 1454 | |
|---|
| 1406 | | - list_for_each(curr, &area->free_list[mtype]) |
|---|
| 1407 | | - freecount++; |
|---|
| 1408 | | - seq_printf(m, "%6lu ", freecount); |
|---|
| 1455 | + list_for_each(curr, &area->free_list[mtype]) { |
|---|
| 1456 | + /* |
|---|
| 1457 | + * Cap the free_list iteration because it might |
|---|
| 1458 | + * be really large and we are under a spinlock |
|---|
| 1459 | + * so a long time spent here could trigger a |
|---|
| 1460 | + * hard lockup detector. Anyway this is a |
|---|
| 1461 | + * debugging tool so knowing there is a handful |
|---|
| 1462 | + * of pages of this order should be more than |
|---|
| 1463 | + * sufficient. |
|---|
| 1464 | + */ |
|---|
| 1465 | + if (++freecount >= 100000) { |
|---|
| 1466 | + overflow = true; |
|---|
| 1467 | + break; |
|---|
| 1468 | + } |
|---|
| 1469 | + } |
|---|
| 1470 | + seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount); |
|---|
| 1409 | 1471 | spin_unlock_irq(&zone->lock); |
|---|
| 1410 | 1472 | cond_resched(); |
|---|
| 1411 | 1473 | spin_lock_irq(&zone->lock); |
|---|
| .. | .. |
|---|
| 1445 | 1507 | |
|---|
| 1446 | 1508 | page = pfn_to_online_page(pfn); |
|---|
| 1447 | 1509 | if (!page) |
|---|
| 1448 | | - continue; |
|---|
| 1449 | | - |
|---|
| 1450 | | - /* Watch for unexpected holes punched in the memmap */ |
|---|
| 1451 | | - if (!memmap_valid_within(pfn, page, zone)) |
|---|
| 1452 | 1510 | continue; |
|---|
| 1453 | 1511 | |
|---|
| 1454 | 1512 | if (page_zone(page) != zone) |
|---|
| .. | .. |
|---|
| 1567 | 1625 | if (is_zone_first_populated(pgdat, zone)) { |
|---|
| 1568 | 1626 | seq_printf(m, "\n per-node stats"); |
|---|
| 1569 | 1627 | for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { |
|---|
| 1570 | | - /* Skip hidden vmstat items. */ |
|---|
| 1571 | | - if (*vmstat_text[i + NR_VM_ZONE_STAT_ITEMS + |
|---|
| 1572 | | - NR_VM_NUMA_STAT_ITEMS] == '\0') |
|---|
| 1573 | | - continue; |
|---|
| 1574 | | - seq_printf(m, "\n %-12s %lu", |
|---|
| 1575 | | - vmstat_text[i + NR_VM_ZONE_STAT_ITEMS + |
|---|
| 1576 | | - NR_VM_NUMA_STAT_ITEMS], |
|---|
| 1577 | | - node_page_state(pgdat, i)); |
|---|
| 1628 | + seq_printf(m, "\n %-12s %lu", node_stat_name(i), |
|---|
| 1629 | + node_page_state_pages(pgdat, i)); |
|---|
| 1578 | 1630 | } |
|---|
| 1579 | 1631 | } |
|---|
| 1580 | 1632 | seq_printf(m, |
|---|
| .. | .. |
|---|
| 1584 | 1636 | "\n high %lu" |
|---|
| 1585 | 1637 | "\n spanned %lu" |
|---|
| 1586 | 1638 | "\n present %lu" |
|---|
| 1587 | | - "\n managed %lu", |
|---|
| 1639 | + "\n managed %lu" |
|---|
| 1640 | + "\n cma %lu", |
|---|
| 1588 | 1641 | zone_page_state(zone, NR_FREE_PAGES), |
|---|
| 1589 | 1642 | min_wmark_pages(zone), |
|---|
| 1590 | 1643 | low_wmark_pages(zone), |
|---|
| 1591 | 1644 | high_wmark_pages(zone), |
|---|
| 1592 | 1645 | zone->spanned_pages, |
|---|
| 1593 | 1646 | zone->present_pages, |
|---|
| 1594 | | - zone->managed_pages); |
|---|
| 1647 | + zone_managed_pages(zone), |
|---|
| 1648 | + zone_cma_pages(zone)); |
|---|
| 1595 | 1649 | |
|---|
| 1596 | 1650 | seq_printf(m, |
|---|
| 1597 | 1651 | "\n protection: (%ld", |
|---|
| .. | .. |
|---|
| 1607 | 1661 | } |
|---|
| 1608 | 1662 | |
|---|
| 1609 | 1663 | for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) |
|---|
| 1610 | | - seq_printf(m, "\n %-12s %lu", vmstat_text[i], |
|---|
| 1611 | | - zone_page_state(zone, i)); |
|---|
| 1664 | + seq_printf(m, "\n %-12s %lu", zone_stat_name(i), |
|---|
| 1665 | + zone_page_state(zone, i)); |
|---|
| 1612 | 1666 | |
|---|
| 1613 | 1667 | #ifdef CONFIG_NUMA |
|---|
| 1614 | 1668 | for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) |
|---|
| 1615 | | - seq_printf(m, "\n %-12s %lu", |
|---|
| 1616 | | - vmstat_text[i + NR_VM_ZONE_STAT_ITEMS], |
|---|
| 1617 | | - zone_numa_state_snapshot(zone, i)); |
|---|
| 1669 | + seq_printf(m, "\n %-12s %lu", numa_stat_name(i), |
|---|
| 1670 | + zone_numa_state_snapshot(zone, i)); |
|---|
| 1618 | 1671 | #endif |
|---|
| 1619 | 1672 | |
|---|
| 1620 | 1673 | seq_printf(m, "\n pagesets"); |
|---|
| .. | .. |
|---|
| 1665 | 1718 | .show = zoneinfo_show, |
|---|
| 1666 | 1719 | }; |
|---|
| 1667 | 1720 | |
|---|
| 1668 | | -enum writeback_stat_item { |
|---|
| 1669 | | - NR_DIRTY_THRESHOLD, |
|---|
| 1670 | | - NR_DIRTY_BG_THRESHOLD, |
|---|
| 1671 | | - NR_VM_WRITEBACK_STAT_ITEMS, |
|---|
| 1672 | | -}; |
|---|
| 1721 | +#define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \ |
|---|
| 1722 | + NR_VM_NUMA_STAT_ITEMS + \ |
|---|
| 1723 | + NR_VM_NODE_STAT_ITEMS + \ |
|---|
| 1724 | + NR_VM_WRITEBACK_STAT_ITEMS + \ |
|---|
| 1725 | + (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \ |
|---|
| 1726 | + NR_VM_EVENT_ITEMS : 0)) |
|---|
| 1673 | 1727 | |
|---|
| 1674 | 1728 | static void *vmstat_start(struct seq_file *m, loff_t *pos) |
|---|
| 1675 | 1729 | { |
|---|
| 1676 | 1730 | unsigned long *v; |
|---|
| 1677 | | - int i, stat_items_size; |
|---|
| 1731 | + int i; |
|---|
| 1678 | 1732 | |
|---|
| 1679 | | - if (*pos >= ARRAY_SIZE(vmstat_text)) |
|---|
| 1733 | + if (*pos >= NR_VMSTAT_ITEMS) |
|---|
| 1680 | 1734 | return NULL; |
|---|
| 1681 | | - stat_items_size = NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long) + |
|---|
| 1682 | | - NR_VM_NUMA_STAT_ITEMS * sizeof(unsigned long) + |
|---|
| 1683 | | - NR_VM_NODE_STAT_ITEMS * sizeof(unsigned long) + |
|---|
| 1684 | | - NR_VM_WRITEBACK_STAT_ITEMS * sizeof(unsigned long); |
|---|
| 1685 | 1735 | |
|---|
| 1686 | | -#ifdef CONFIG_VM_EVENT_COUNTERS |
|---|
| 1687 | | - stat_items_size += sizeof(struct vm_event_state); |
|---|
| 1688 | | -#endif |
|---|
| 1689 | | - |
|---|
| 1690 | | - v = kmalloc(stat_items_size, GFP_KERNEL); |
|---|
| 1736 | + BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) < NR_VMSTAT_ITEMS); |
|---|
| 1737 | + v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL); |
|---|
| 1691 | 1738 | m->private = v; |
|---|
| 1692 | 1739 | if (!v) |
|---|
| 1693 | 1740 | return ERR_PTR(-ENOMEM); |
|---|
| .. | .. |
|---|
| 1702 | 1749 | #endif |
|---|
| 1703 | 1750 | |
|---|
| 1704 | 1751 | for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) |
|---|
| 1705 | | - v[i] = global_node_page_state(i); |
|---|
| 1752 | + v[i] = global_node_page_state_pages(i); |
|---|
| 1706 | 1753 | v += NR_VM_NODE_STAT_ITEMS; |
|---|
| 1707 | 1754 | |
|---|
| 1708 | 1755 | global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, |
|---|
| .. | .. |
|---|
| 1720 | 1767 | static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) |
|---|
| 1721 | 1768 | { |
|---|
| 1722 | 1769 | (*pos)++; |
|---|
| 1723 | | - //nr_gpu_heap is out-of-tree now so we don't want to export it. |
|---|
| 1724 | | - if (*pos == NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_STAT_ITEMS + NR_GPU_HEAP) |
|---|
| 1725 | | - (*pos)++; |
|---|
| 1726 | | - if (*pos >= ARRAY_SIZE(vmstat_text)) |
|---|
| 1770 | + if (*pos >= NR_VMSTAT_ITEMS) |
|---|
| 1727 | 1771 | return NULL; |
|---|
| 1728 | 1772 | return (unsigned long *)m->private + *pos; |
|---|
| 1729 | 1773 | } |
|---|
| .. | .. |
|---|
| 1736 | 1780 | seq_puts(m, vmstat_text[off]); |
|---|
| 1737 | 1781 | seq_put_decimal_ull(m, " ", *l); |
|---|
| 1738 | 1782 | seq_putc(m, '\n'); |
|---|
| 1783 | + |
|---|
| 1784 | + if (off == NR_VMSTAT_ITEMS - 1) { |
|---|
| 1785 | + /* |
|---|
| 1786 | + * We've come to the end - add any deprecated counters to avoid |
|---|
| 1787 | + * breaking userspace which might depend on them being present. |
|---|
| 1788 | + */ |
|---|
| 1789 | + seq_puts(m, "nr_unstable 0\n"); |
|---|
| 1790 | + } |
|---|
| 1739 | 1791 | return 0; |
|---|
| 1740 | 1792 | } |
|---|
| 1741 | 1793 | |
|---|
| .. | .. |
|---|
| 1764 | 1816 | } |
|---|
| 1765 | 1817 | |
|---|
| 1766 | 1818 | int vmstat_refresh(struct ctl_table *table, int write, |
|---|
| 1767 | | - void __user *buffer, size_t *lenp, loff_t *ppos) |
|---|
| 1819 | + void *buffer, size_t *lenp, loff_t *ppos) |
|---|
| 1768 | 1820 | { |
|---|
| 1769 | 1821 | long val; |
|---|
| 1770 | 1822 | int err; |
|---|
| .. | .. |
|---|
| 1789 | 1841 | val = atomic_long_read(&vm_zone_stat[i]); |
|---|
| 1790 | 1842 | if (val < 0) { |
|---|
| 1791 | 1843 | pr_warn("%s: %s %ld\n", |
|---|
| 1792 | | - __func__, vmstat_text[i], val); |
|---|
| 1844 | + __func__, zone_stat_name(i), val); |
|---|
| 1793 | 1845 | err = -EINVAL; |
|---|
| 1794 | 1846 | } |
|---|
| 1795 | 1847 | } |
|---|
| .. | .. |
|---|
| 1798 | 1850 | val = atomic_long_read(&vm_numa_stat[i]); |
|---|
| 1799 | 1851 | if (val < 0) { |
|---|
| 1800 | 1852 | pr_warn("%s: %s %ld\n", |
|---|
| 1801 | | - __func__, vmstat_text[i + NR_VM_ZONE_STAT_ITEMS], val); |
|---|
| 1853 | + __func__, numa_stat_name(i), val); |
|---|
| 1802 | 1854 | err = -EINVAL; |
|---|
| 1803 | 1855 | } |
|---|
| 1804 | 1856 | } |
|---|
| .. | .. |
|---|
| 2068 | 2120 | return 0; |
|---|
| 2069 | 2121 | } |
|---|
| 2070 | 2122 | |
|---|
| 2071 | | -static const struct seq_operations unusable_op = { |
|---|
| 2123 | +static const struct seq_operations unusable_sops = { |
|---|
| 2072 | 2124 | .start = frag_start, |
|---|
| 2073 | 2125 | .next = frag_next, |
|---|
| 2074 | 2126 | .stop = frag_stop, |
|---|
| 2075 | 2127 | .show = unusable_show, |
|---|
| 2076 | 2128 | }; |
|---|
| 2077 | 2129 | |
|---|
| 2078 | | -static int unusable_open(struct inode *inode, struct file *file) |
|---|
| 2079 | | -{ |
|---|
| 2080 | | - return seq_open(file, &unusable_op); |
|---|
| 2081 | | -} |
|---|
| 2082 | | - |
|---|
| 2083 | | -static const struct file_operations unusable_file_ops = { |
|---|
| 2084 | | - .open = unusable_open, |
|---|
| 2085 | | - .read = seq_read, |
|---|
| 2086 | | - .llseek = seq_lseek, |
|---|
| 2087 | | - .release = seq_release, |
|---|
| 2088 | | -}; |
|---|
| 2130 | +DEFINE_SEQ_ATTRIBUTE(unusable); |
|---|
| 2089 | 2131 | |
|---|
| 2090 | 2132 | static void extfrag_show_print(struct seq_file *m, |
|---|
| 2091 | 2133 | pg_data_t *pgdat, struct zone *zone) |
|---|
| .. | .. |
|---|
| 2120 | 2162 | return 0; |
|---|
| 2121 | 2163 | } |
|---|
| 2122 | 2164 | |
|---|
| 2123 | | -static const struct seq_operations extfrag_op = { |
|---|
| 2165 | +static const struct seq_operations extfrag_sops = { |
|---|
| 2124 | 2166 | .start = frag_start, |
|---|
| 2125 | 2167 | .next = frag_next, |
|---|
| 2126 | 2168 | .stop = frag_stop, |
|---|
| 2127 | 2169 | .show = extfrag_show, |
|---|
| 2128 | 2170 | }; |
|---|
| 2129 | 2171 | |
|---|
| 2130 | | -static int extfrag_open(struct inode *inode, struct file *file) |
|---|
| 2131 | | -{ |
|---|
| 2132 | | - return seq_open(file, &extfrag_op); |
|---|
| 2133 | | -} |
|---|
| 2134 | | - |
|---|
| 2135 | | -static const struct file_operations extfrag_file_ops = { |
|---|
| 2136 | | - .open = extfrag_open, |
|---|
| 2137 | | - .read = seq_read, |
|---|
| 2138 | | - .llseek = seq_lseek, |
|---|
| 2139 | | - .release = seq_release, |
|---|
| 2140 | | -}; |
|---|
| 2172 | +DEFINE_SEQ_ATTRIBUTE(extfrag); |
|---|
| 2141 | 2173 | |
|---|
| 2142 | 2174 | static int __init extfrag_debug_init(void) |
|---|
| 2143 | 2175 | { |
|---|
| 2144 | 2176 | struct dentry *extfrag_debug_root; |
|---|
| 2145 | 2177 | |
|---|
| 2146 | 2178 | extfrag_debug_root = debugfs_create_dir("extfrag", NULL); |
|---|
| 2147 | | - if (!extfrag_debug_root) |
|---|
| 2148 | | - return -ENOMEM; |
|---|
| 2149 | 2179 | |
|---|
| 2150 | | - if (!debugfs_create_file("unusable_index", 0444, |
|---|
| 2151 | | - extfrag_debug_root, NULL, &unusable_file_ops)) |
|---|
| 2152 | | - goto fail; |
|---|
| 2180 | + debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL, |
|---|
| 2181 | + &unusable_fops); |
|---|
| 2153 | 2182 | |
|---|
| 2154 | | - if (!debugfs_create_file("extfrag_index", 0444, |
|---|
| 2155 | | - extfrag_debug_root, NULL, &extfrag_file_ops)) |
|---|
| 2156 | | - goto fail; |
|---|
| 2183 | + debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL, |
|---|
| 2184 | + &extfrag_fops); |
|---|
| 2157 | 2185 | |
|---|
| 2158 | 2186 | return 0; |
|---|
| 2159 | | -fail: |
|---|
| 2160 | | - debugfs_remove_recursive(extfrag_debug_root); |
|---|
| 2161 | | - return -ENOMEM; |
|---|
| 2162 | 2187 | } |
|---|
| 2163 | 2188 | |
|---|
| 2164 | 2189 | module_init(extfrag_debug_init); |
|---|