| .. | .. |
|---|
| 27 | 27 | #ifndef _LINUX_TOPOLOGY_H |
|---|
| 28 | 28 | #define _LINUX_TOPOLOGY_H |
|---|
| 29 | 29 | |
|---|
| 30 | +#include <linux/arch_topology.h> |
|---|
| 30 | 31 | #include <linux/cpumask.h> |
|---|
| 31 | 32 | #include <linux/bitops.h> |
|---|
| 32 | 33 | #include <linux/mmzone.h> |
|---|
| .. | .. |
|---|
| 47 | 48 | /* Conform to ACPI 2.0 SLIT distance definitions */ |
|---|
| 48 | 49 | #define LOCAL_DISTANCE 10 |
|---|
| 49 | 50 | #define REMOTE_DISTANCE 20 |
|---|
| 51 | +#define DISTANCE_BITS 8 |
|---|
| 50 | 52 | #ifndef node_distance |
|---|
| 51 | 53 | #define node_distance(from,to) ((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE) |
|---|
| 52 | 54 | #endif |
|---|
| .. | .. |
|---|
| 59 | 61 | */ |
|---|
| 60 | 62 | #define RECLAIM_DISTANCE 30 |
|---|
| 61 | 63 | #endif |
|---|
| 64 | + |
|---|
| 65 | +/* |
|---|
| 66 | + * The following tunable allows platforms to override the default node |
|---|
| 67 | + * reclaim distance (RECLAIM_DISTANCE) if remote memory accesses are |
|---|
| 68 | + * sufficiently fast that the default value actually hurts |
|---|
| 69 | + * performance. |
|---|
| 70 | + * |
|---|
| 71 | + * AMD EPYC machines use this because even though the 2-hop distance |
|---|
| 72 | + * is 32 (3.2x slower than a local memory access) performance actually |
|---|
| 73 | + * *improves* if allowed to reclaim memory and load balance tasks |
|---|
| 74 | + * between NUMA nodes 2-hops apart. |
|---|
| 75 | + */ |
|---|
| 76 | +extern int __read_mostly node_reclaim_distance; |
|---|
| 77 | + |
|---|
| 62 | 78 | #ifndef PENALTY_FOR_NODE_WITH_CPUS |
|---|
| 63 | 79 | #define PENALTY_FOR_NODE_WITH_CPUS (1) |
|---|
| 64 | 80 | #endif |
|---|
| .. | .. |
|---|
| 115 | 131 | * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem(). |
|---|
| 116 | 132 | */ |
|---|
| 117 | 133 | DECLARE_PER_CPU(int, _numa_mem_); |
|---|
| 118 | | -extern int _node_numa_mem_[MAX_NUMNODES]; |
|---|
| 119 | 134 | |
|---|
| 120 | 135 | #ifndef set_numa_mem |
|---|
| 121 | 136 | static inline void set_numa_mem(int node) |
|---|
| 122 | 137 | { |
|---|
| 123 | 138 | this_cpu_write(_numa_mem_, node); |
|---|
| 124 | | - _node_numa_mem_[numa_node_id()] = node; |
|---|
| 125 | | -} |
|---|
| 126 | | -#endif |
|---|
| 127 | | - |
|---|
| 128 | | -#ifndef node_to_mem_node |
|---|
| 129 | | -static inline int node_to_mem_node(int node) |
|---|
| 130 | | -{ |
|---|
| 131 | | - return _node_numa_mem_[node]; |
|---|
| 132 | 139 | } |
|---|
| 133 | 140 | #endif |
|---|
| 134 | 141 | |
|---|
| .. | .. |
|---|
| 151 | 158 | static inline void set_cpu_numa_mem(int cpu, int node) |
|---|
| 152 | 159 | { |
|---|
| 153 | 160 | per_cpu(_numa_mem_, cpu) = node; |
|---|
| 154 | | - _node_numa_mem_[cpu_to_node(cpu)] = node; |
|---|
| 155 | 161 | } |
|---|
| 156 | 162 | #endif |
|---|
| 157 | 163 | |
|---|
| .. | .. |
|---|
| 162 | 168 | static inline int numa_mem_id(void) |
|---|
| 163 | 169 | { |
|---|
| 164 | 170 | return numa_node_id(); |
|---|
| 165 | | -} |
|---|
| 166 | | -#endif |
|---|
| 167 | | - |
|---|
| 168 | | -#ifndef node_to_mem_node |
|---|
| 169 | | -static inline int node_to_mem_node(int node) |
|---|
| 170 | | -{ |
|---|
| 171 | | - return node; |
|---|
| 172 | 171 | } |
|---|
| 173 | 172 | #endif |
|---|
| 174 | 173 | |
|---|
| .. | .. |
|---|
| 184 | 183 | #ifndef topology_physical_package_id |
|---|
| 185 | 184 | #define topology_physical_package_id(cpu) ((void)(cpu), -1) |
|---|
| 186 | 185 | #endif |
|---|
| 186 | +#ifndef topology_die_id |
|---|
| 187 | +#define topology_die_id(cpu) ((void)(cpu), -1) |
|---|
| 188 | +#endif |
|---|
| 187 | 189 | #ifndef topology_core_id |
|---|
| 188 | 190 | #define topology_core_id(cpu) ((void)(cpu), 0) |
|---|
| 189 | 191 | #endif |
|---|
| .. | .. |
|---|
| 193 | 195 | #ifndef topology_core_cpumask |
|---|
| 194 | 196 | #define topology_core_cpumask(cpu) cpumask_of(cpu) |
|---|
| 195 | 197 | #endif |
|---|
| 198 | +#ifndef topology_die_cpumask |
|---|
| 199 | +#define topology_die_cpumask(cpu) cpumask_of(cpu) |
|---|
| 200 | +#endif |
|---|
| 196 | 201 | |
|---|
| 197 | | -#ifdef CONFIG_SCHED_SMT |
|---|
| 202 | +#if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask) |
|---|
| 198 | 203 | static inline const struct cpumask *cpu_smt_mask(int cpu) |
|---|
| 199 | 204 | { |
|---|
| 200 | 205 | return topology_sibling_cpumask(cpu); |
|---|