| .. | .. |
|---|
| 10 | 10 | #include <linux/module.h> |
|---|
| 11 | 11 | #include <linux/ftrace.h> |
|---|
| 12 | 12 | #include <linux/kprobes.h> |
|---|
| 13 | +#include <linux/irq_pipeline.h> |
|---|
| 13 | 14 | #include "trace.h" |
|---|
| 14 | 15 | |
|---|
| 15 | 16 | #define CREATE_TRACE_POINTS |
|---|
| .. | .. |
|---|
| 133 | 134 | } |
|---|
| 134 | 135 | EXPORT_SYMBOL(trace_hardirqs_off_caller); |
|---|
| 135 | 136 | NOKPROBE_SYMBOL(trace_hardirqs_off_caller); |
|---|
| 137 | + |
|---|
| 138 | +#ifdef CONFIG_IRQ_PIPELINE |
|---|
| 139 | + |
|---|
| 140 | +void trace_hardirqs_off_pipelined(void) |
|---|
| 141 | +{ |
|---|
| 142 | + WARN_ON(irq_pipeline_debug() && !hard_irqs_disabled()); |
|---|
| 143 | + |
|---|
| 144 | + if (running_inband()) |
|---|
| 145 | + trace_hardirqs_off(); |
|---|
| 146 | +} |
|---|
| 147 | +EXPORT_SYMBOL(trace_hardirqs_off_pipelined); |
|---|
| 148 | +NOKPROBE_SYMBOL(trace_hardirqs_off_pipelined); |
|---|
| 149 | + |
|---|
| 150 | +void trace_hardirqs_on_pipelined(void) |
|---|
| 151 | +{ |
|---|
| 152 | + WARN_ON(irq_pipeline_debug() && !hard_irqs_disabled()); |
|---|
| 153 | + |
|---|
| 154 | + /* |
|---|
| 155 | + * If the in-band stage of the kernel is current but the IRQ |
|---|
| 156 | + * was not delivered because the latter is stalled, keep the |
|---|
| 157 | + * tracing logic unaware of the receipt, so that no false |
|---|
| 158 | + * positive is triggered in lockdep (e.g. IN-HARDIRQ-W -> |
|---|
| 159 | + * HARDIRQ-ON-W). |
|---|
| 160 | + */ |
|---|
| 161 | + if (running_inband() && !irqs_disabled()) { |
|---|
| 162 | + stall_inband(); |
|---|
| 163 | + trace_hardirqs_on(); |
|---|
| 164 | + unstall_inband_nocheck(); |
|---|
| 165 | + } |
|---|
| 166 | +} |
|---|
| 167 | +EXPORT_SYMBOL(trace_hardirqs_on_pipelined); |
|---|
| 168 | +NOKPROBE_SYMBOL(trace_hardirqs_on_pipelined); |
|---|
| 169 | + |
|---|
| 170 | +#else |
|---|
| 171 | + |
|---|
| 172 | +void trace_hardirqs_off_pipelined(void) |
|---|
| 173 | +{ |
|---|
| 174 | + trace_hardirqs_off(); |
|---|
| 175 | +} |
|---|
| 176 | +EXPORT_SYMBOL(trace_hardirqs_off_pipelined); |
|---|
| 177 | +NOKPROBE_SYMBOL(trace_hardirqs_off_pipelined); |
|---|
| 178 | + |
|---|
| 179 | +void trace_hardirqs_on_pipelined(void) |
|---|
| 180 | +{ |
|---|
| 181 | + trace_hardirqs_on(); |
|---|
| 182 | +} |
|---|
| 183 | +EXPORT_SYMBOL(trace_hardirqs_on_pipelined); |
|---|
| 184 | +NOKPROBE_SYMBOL(trace_hardirqs_on_pipelined); |
|---|
| 185 | + |
|---|
| 186 | +#endif |
|---|
| 187 | + |
|---|
| 136 | 188 | #endif /* CONFIG_TRACE_IRQFLAGS */ |
|---|
| 137 | 189 | |
|---|
| 138 | 190 | #ifdef CONFIG_TRACE_PREEMPT_TOGGLE |
|---|