.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra |
---|
3 | 4 | * |
---|
.. | .. |
---|
28 | 29 | */ |
---|
29 | 30 | static bool irq_work_claim(struct irq_work *work) |
---|
30 | 31 | { |
---|
31 | | - unsigned long flags, oflags, nflags; |
---|
| 32 | + int oflags; |
---|
32 | 33 | |
---|
| 34 | + oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->flags); |
---|
33 | 35 | /* |
---|
34 | | - * Start with our best wish as a premise but only trust any |
---|
35 | | - * flag value after cmpxchg() result. |
---|
| 36 | + * If the work is already pending, no need to raise the IPI. |
---|
| 37 | + * The pairing atomic_fetch_andnot() in irq_work_run() makes sure |
---|
| 38 | + * everything we did before is visible. |
---|
36 | 39 | */ |
---|
37 | | - flags = work->flags & ~IRQ_WORK_PENDING; |
---|
38 | | - for (;;) { |
---|
39 | | - nflags = flags | IRQ_WORK_CLAIMED; |
---|
40 | | - oflags = cmpxchg(&work->flags, flags, nflags); |
---|
41 | | - if (oflags == flags) |
---|
42 | | - break; |
---|
43 | | - if (oflags & IRQ_WORK_PENDING) |
---|
44 | | - return false; |
---|
45 | | - flags = oflags; |
---|
46 | | - cpu_relax(); |
---|
47 | | - } |
---|
48 | | - |
---|
| 40 | + if (oflags & IRQ_WORK_PENDING) |
---|
| 41 | + return false; |
---|
49 | 42 | return true; |
---|
50 | 43 | } |
---|
51 | 44 | |
---|
.. | .. |
---|
60 | 53 | static void __irq_work_queue_local(struct irq_work *work) |
---|
61 | 54 | { |
---|
62 | 55 | /* If the work is "lazy", handle it from next tick if any */ |
---|
63 | | - if (work->flags & IRQ_WORK_LAZY) { |
---|
| 56 | + if (atomic_read(&work->flags) & IRQ_WORK_LAZY) { |
---|
64 | 57 | if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) && |
---|
65 | 58 | tick_nohz_tick_stopped()) |
---|
66 | 59 | arch_irq_work_raise(); |
---|
.. | .. |
---|
109 | 102 | if (cpu != smp_processor_id()) { |
---|
110 | 103 | /* Arch remote IPI send/receive backend aren't NMI safe */ |
---|
111 | 104 | WARN_ON_ONCE(in_nmi()); |
---|
112 | | - if (llist_add(&work->llnode, &per_cpu(raised_list, cpu))) |
---|
113 | | - arch_send_call_function_single_ipi(cpu); |
---|
| 105 | + __smp_call_single_queue(cpu, &work->llnode); |
---|
114 | 106 | } else { |
---|
115 | 107 | __irq_work_queue_local(work); |
---|
116 | 108 | } |
---|
.. | .. |
---|
119 | 111 | return true; |
---|
120 | 112 | #endif /* CONFIG_SMP */ |
---|
121 | 113 | } |
---|
122 | | - |
---|
| 114 | +EXPORT_SYMBOL_GPL(irq_work_queue_on); |
---|
123 | 115 | |
---|
124 | 116 | bool irq_work_needs_cpu(void) |
---|
125 | 117 | { |
---|
.. | .. |
---|
138 | 130 | return true; |
---|
139 | 131 | } |
---|
140 | 132 | |
---|
| 133 | +void irq_work_single(void *arg) |
---|
| 134 | +{ |
---|
| 135 | + struct irq_work *work = arg; |
---|
| 136 | + int flags; |
---|
| 137 | + |
---|
| 138 | + /* |
---|
| 139 | + * Clear the PENDING bit, after this point the @work |
---|
| 140 | + * can be re-used. |
---|
| 141 | + * Make it immediately visible so that other CPUs trying |
---|
| 142 | + * to claim that work don't rely on us to handle their data |
---|
| 143 | + * while we are in the middle of the func. |
---|
| 144 | + */ |
---|
| 145 | + flags = atomic_fetch_andnot(IRQ_WORK_PENDING, &work->flags); |
---|
| 146 | + |
---|
| 147 | + lockdep_irq_work_enter(work); |
---|
| 148 | + work->func(work); |
---|
| 149 | + lockdep_irq_work_exit(work); |
---|
| 150 | + /* |
---|
| 151 | + * Clear the BUSY bit and return to the free state if |
---|
| 152 | + * no-one else claimed it meanwhile. |
---|
| 153 | + */ |
---|
| 154 | + flags &= ~IRQ_WORK_PENDING; |
---|
| 155 | + (void)atomic_cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY); |
---|
| 156 | +} |
---|
| 157 | + |
---|
141 | 158 | static void irq_work_run_list(struct llist_head *list) |
---|
142 | 159 | { |
---|
143 | 160 | struct irq_work *work, *tmp; |
---|
144 | 161 | struct llist_node *llnode; |
---|
145 | | - unsigned long flags; |
---|
146 | 162 | |
---|
147 | 163 | BUG_ON(!irqs_disabled()); |
---|
148 | 164 | |
---|
.. | .. |
---|
150 | 166 | return; |
---|
151 | 167 | |
---|
152 | 168 | llnode = llist_del_all(list); |
---|
153 | | - llist_for_each_entry_safe(work, tmp, llnode, llnode) { |
---|
154 | | - /* |
---|
155 | | - * Clear the PENDING bit, after this point the @work |
---|
156 | | - * can be re-used. |
---|
157 | | - * Make it immediately visible so that other CPUs trying |
---|
158 | | - * to claim that work don't rely on us to handle their data |
---|
159 | | - * while we are in the middle of the func. |
---|
160 | | - */ |
---|
161 | | - flags = work->flags & ~IRQ_WORK_PENDING; |
---|
162 | | - xchg(&work->flags, flags); |
---|
163 | | - |
---|
164 | | - work->func(work); |
---|
165 | | - /* |
---|
166 | | - * Clear the BUSY bit and return to the free state if |
---|
167 | | - * no-one else claimed it meanwhile. |
---|
168 | | - */ |
---|
169 | | - (void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY); |
---|
170 | | - } |
---|
| 169 | + llist_for_each_entry_safe(work, tmp, llnode, llnode) |
---|
| 170 | + irq_work_single(work); |
---|
171 | 171 | } |
---|
172 | 172 | |
---|
173 | 173 | /* |
---|
.. | .. |
---|
198 | 198 | { |
---|
199 | 199 | lockdep_assert_irqs_enabled(); |
---|
200 | 200 | |
---|
201 | | - while (work->flags & IRQ_WORK_BUSY) |
---|
| 201 | + while (atomic_read(&work->flags) & IRQ_WORK_BUSY) |
---|
202 | 202 | cpu_relax(); |
---|
203 | 203 | } |
---|
204 | 204 | EXPORT_SYMBOL_GPL(irq_work_sync); |
---|