| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Housekeeping management. Manage the targets for routine code that can run on |
|---|
| 3 | 4 | * any CPU: unbound workqueues, timers, kthreads and any offloadable work. |
|---|
| .. | .. |
|---|
| 8 | 9 | */ |
|---|
| 9 | 10 | #include "sched.h" |
|---|
| 10 | 11 | |
|---|
| 11 | | -DEFINE_STATIC_KEY_FALSE(housekeeping_overriden); |
|---|
| 12 | | -EXPORT_SYMBOL_GPL(housekeeping_overriden); |
|---|
| 12 | +DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); |
|---|
| 13 | +EXPORT_SYMBOL_GPL(housekeeping_overridden); |
|---|
| 13 | 14 | static cpumask_var_t housekeeping_mask; |
|---|
| 14 | 15 | static unsigned int housekeeping_flags; |
|---|
| 15 | 16 | |
|---|
| 17 | +bool housekeeping_enabled(enum hk_flags flags) |
|---|
| 18 | +{ |
|---|
| 19 | + return !!(housekeeping_flags & flags); |
|---|
| 20 | +} |
|---|
| 21 | +EXPORT_SYMBOL_GPL(housekeeping_enabled); |
|---|
| 22 | + |
|---|
| 16 | 23 | int housekeeping_any_cpu(enum hk_flags flags) |
|---|
| 17 | 24 | { |
|---|
| 18 | | - if (static_branch_unlikely(&housekeeping_overriden)) |
|---|
| 19 | | - if (housekeeping_flags & flags) |
|---|
| 25 | + int cpu; |
|---|
| 26 | + |
|---|
| 27 | + if (static_branch_unlikely(&housekeeping_overridden)) { |
|---|
| 28 | + if (housekeeping_flags & flags) { |
|---|
| 29 | + cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id()); |
|---|
| 30 | + if (cpu < nr_cpu_ids) |
|---|
| 31 | + return cpu; |
|---|
| 32 | + |
|---|
| 20 | 33 | return cpumask_any_and(housekeeping_mask, cpu_online_mask); |
|---|
| 34 | + } |
|---|
| 35 | + } |
|---|
| 21 | 36 | return smp_processor_id(); |
|---|
| 22 | 37 | } |
|---|
| 23 | 38 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); |
|---|
| 24 | 39 | |
|---|
| 25 | 40 | const struct cpumask *housekeeping_cpumask(enum hk_flags flags) |
|---|
| 26 | 41 | { |
|---|
| 27 | | - if (static_branch_unlikely(&housekeeping_overriden)) |
|---|
| 42 | + if (static_branch_unlikely(&housekeeping_overridden)) |
|---|
| 28 | 43 | if (housekeeping_flags & flags) |
|---|
| 29 | 44 | return housekeeping_mask; |
|---|
| 30 | 45 | return cpu_possible_mask; |
|---|
| .. | .. |
|---|
| 33 | 48 | |
|---|
| 34 | 49 | void housekeeping_affine(struct task_struct *t, enum hk_flags flags) |
|---|
| 35 | 50 | { |
|---|
| 36 | | - if (static_branch_unlikely(&housekeeping_overriden)) |
|---|
| 51 | + if (static_branch_unlikely(&housekeeping_overridden)) |
|---|
| 37 | 52 | if (housekeeping_flags & flags) |
|---|
| 38 | 53 | set_cpus_allowed_ptr(t, housekeeping_mask); |
|---|
| 39 | 54 | } |
|---|
| .. | .. |
|---|
| 41 | 56 | |
|---|
| 42 | 57 | bool housekeeping_test_cpu(int cpu, enum hk_flags flags) |
|---|
| 43 | 58 | { |
|---|
| 44 | | - if (static_branch_unlikely(&housekeeping_overriden)) |
|---|
| 59 | + if (static_branch_unlikely(&housekeeping_overridden)) |
|---|
| 45 | 60 | if (housekeeping_flags & flags) |
|---|
| 46 | 61 | return cpumask_test_cpu(cpu, housekeeping_mask); |
|---|
| 47 | 62 | return true; |
|---|
| .. | .. |
|---|
| 53 | 68 | if (!housekeeping_flags) |
|---|
| 54 | 69 | return; |
|---|
| 55 | 70 | |
|---|
| 56 | | - static_branch_enable(&housekeeping_overriden); |
|---|
| 71 | + static_branch_enable(&housekeeping_overridden); |
|---|
| 57 | 72 | |
|---|
| 58 | 73 | if (housekeeping_flags & HK_FLAG_TICK) |
|---|
| 59 | 74 | sched_tick_offload_init(); |
|---|
| .. | .. |
|---|
| 65 | 80 | static int __init housekeeping_setup(char *str, enum hk_flags flags) |
|---|
| 66 | 81 | { |
|---|
| 67 | 82 | cpumask_var_t non_housekeeping_mask; |
|---|
| 83 | + cpumask_var_t tmp; |
|---|
| 68 | 84 | int err; |
|---|
| 69 | 85 | |
|---|
| 70 | 86 | alloc_bootmem_cpumask_var(&non_housekeeping_mask); |
|---|
| .. | .. |
|---|
| 75 | 91 | return 0; |
|---|
| 76 | 92 | } |
|---|
| 77 | 93 | |
|---|
| 94 | + alloc_bootmem_cpumask_var(&tmp); |
|---|
| 78 | 95 | if (!housekeeping_flags) { |
|---|
| 79 | 96 | alloc_bootmem_cpumask_var(&housekeeping_mask); |
|---|
| 80 | 97 | cpumask_andnot(housekeeping_mask, |
|---|
| 81 | 98 | cpu_possible_mask, non_housekeeping_mask); |
|---|
| 82 | | - if (cpumask_empty(housekeeping_mask)) |
|---|
| 83 | | - cpumask_set_cpu(smp_processor_id(), housekeeping_mask); |
|---|
| 84 | | - } else { |
|---|
| 85 | | - cpumask_var_t tmp; |
|---|
| 86 | 99 | |
|---|
| 87 | | - alloc_bootmem_cpumask_var(&tmp); |
|---|
| 100 | + cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); |
|---|
| 101 | + if (cpumask_empty(tmp)) { |
|---|
| 102 | + pr_warn("Housekeeping: must include one present CPU, " |
|---|
| 103 | + "using boot CPU:%d\n", smp_processor_id()); |
|---|
| 104 | + __cpumask_set_cpu(smp_processor_id(), housekeeping_mask); |
|---|
| 105 | + __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); |
|---|
| 106 | + } |
|---|
| 107 | + } else { |
|---|
| 108 | + cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); |
|---|
| 109 | + if (cpumask_empty(tmp)) |
|---|
| 110 | + __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); |
|---|
| 88 | 111 | cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); |
|---|
| 89 | 112 | if (!cpumask_equal(tmp, housekeeping_mask)) { |
|---|
| 90 | 113 | pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); |
|---|
| .. | .. |
|---|
| 92 | 115 | free_bootmem_cpumask_var(non_housekeeping_mask); |
|---|
| 93 | 116 | return 0; |
|---|
| 94 | 117 | } |
|---|
| 95 | | - free_bootmem_cpumask_var(tmp); |
|---|
| 96 | 118 | } |
|---|
| 119 | + free_bootmem_cpumask_var(tmp); |
|---|
| 97 | 120 | |
|---|
| 98 | 121 | if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { |
|---|
| 99 | 122 | if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { |
|---|
| .. | .. |
|---|
| 117 | 140 | { |
|---|
| 118 | 141 | unsigned int flags; |
|---|
| 119 | 142 | |
|---|
| 120 | | - flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC; |
|---|
| 143 | + flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | |
|---|
| 144 | + HK_FLAG_MISC | HK_FLAG_KTHREAD; |
|---|
| 121 | 145 | |
|---|
| 122 | 146 | return housekeeping_setup(str, flags); |
|---|
| 123 | 147 | } |
|---|
| .. | .. |
|---|
| 126 | 150 | static int __init housekeeping_isolcpus_setup(char *str) |
|---|
| 127 | 151 | { |
|---|
| 128 | 152 | unsigned int flags = 0; |
|---|
| 153 | + bool illegal = false; |
|---|
| 154 | + char *par; |
|---|
| 155 | + int len; |
|---|
| 129 | 156 | |
|---|
| 130 | 157 | while (isalpha(*str)) { |
|---|
| 131 | 158 | if (!strncmp(str, "nohz,", 5)) { |
|---|
| .. | .. |
|---|
| 140 | 167 | continue; |
|---|
| 141 | 168 | } |
|---|
| 142 | 169 | |
|---|
| 143 | | - pr_warn("isolcpus: Error, unknown flag\n"); |
|---|
| 144 | | - return 0; |
|---|
| 170 | + if (!strncmp(str, "managed_irq,", 12)) { |
|---|
| 171 | + str += 12; |
|---|
| 172 | + flags |= HK_FLAG_MANAGED_IRQ; |
|---|
| 173 | + continue; |
|---|
| 174 | + } |
|---|
| 175 | + |
|---|
| 176 | + /* |
|---|
| 177 | + * Skip unknown sub-parameter and validate that it is not |
|---|
| 178 | + * containing an invalid character. |
|---|
| 179 | + */ |
|---|
| 180 | + for (par = str, len = 0; *str && *str != ','; str++, len++) { |
|---|
| 181 | + if (!isalpha(*str) && *str != '_') |
|---|
| 182 | + illegal = true; |
|---|
| 183 | + } |
|---|
| 184 | + |
|---|
| 185 | + if (illegal) { |
|---|
| 186 | + pr_warn("isolcpus: Invalid flag %.*s\n", len, par); |
|---|
| 187 | + return 0; |
|---|
| 188 | + } |
|---|
| 189 | + |
|---|
| 190 | + pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); |
|---|
| 191 | + str++; |
|---|
| 145 | 192 | } |
|---|
| 146 | 193 | |
|---|
| 147 | 194 | /* Default behaviour for isolcpus without flags */ |
|---|