.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0 |
---|
1 | 2 | /* |
---|
2 | 3 | * Block stat tracking code |
---|
3 | 4 | * |
---|
.. | .. |
---|
52 | 53 | struct request_queue *q = rq->q; |
---|
53 | 54 | struct blk_stat_callback *cb; |
---|
54 | 55 | struct blk_rq_stat *stat; |
---|
55 | | - int bucket; |
---|
| 56 | + int bucket, cpu; |
---|
56 | 57 | u64 value; |
---|
57 | 58 | |
---|
58 | 59 | value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0; |
---|
.. | .. |
---|
60 | 61 | blk_throtl_stat_add(rq, value); |
---|
61 | 62 | |
---|
62 | 63 | rcu_read_lock(); |
---|
| 64 | + cpu = get_cpu(); |
---|
63 | 65 | list_for_each_entry_rcu(cb, &q->stats->callbacks, list) { |
---|
64 | 66 | if (!blk_stat_is_active(cb)) |
---|
65 | 67 | continue; |
---|
.. | .. |
---|
68 | 70 | if (bucket < 0) |
---|
69 | 71 | continue; |
---|
70 | 72 | |
---|
71 | | - stat = &get_cpu_ptr(cb->cpu_stat)[bucket]; |
---|
| 73 | + stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket]; |
---|
72 | 74 | blk_rq_stat_add(stat, value); |
---|
73 | | - put_cpu_ptr(cb->cpu_stat); |
---|
74 | 75 | } |
---|
| 76 | + put_cpu(); |
---|
75 | 77 | rcu_read_unlock(); |
---|
76 | 78 | } |
---|
77 | 79 | |
---|
.. | .. |
---|
130 | 132 | |
---|
131 | 133 | return cb; |
---|
132 | 134 | } |
---|
133 | | -EXPORT_SYMBOL_GPL(blk_stat_alloc_callback); |
---|
134 | 135 | |
---|
135 | 136 | void blk_stat_add_callback(struct request_queue *q, |
---|
136 | 137 | struct blk_stat_callback *cb) |
---|
137 | 138 | { |
---|
138 | 139 | unsigned int bucket; |
---|
| 140 | + unsigned long flags; |
---|
139 | 141 | int cpu; |
---|
140 | 142 | |
---|
141 | 143 | for_each_possible_cpu(cpu) { |
---|
.. | .. |
---|
146 | 148 | blk_rq_stat_init(&cpu_stat[bucket]); |
---|
147 | 149 | } |
---|
148 | 150 | |
---|
149 | | - spin_lock(&q->stats->lock); |
---|
| 151 | + spin_lock_irqsave(&q->stats->lock, flags); |
---|
150 | 152 | list_add_tail_rcu(&cb->list, &q->stats->callbacks); |
---|
151 | 153 | blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
---|
152 | | - spin_unlock(&q->stats->lock); |
---|
| 154 | + spin_unlock_irqrestore(&q->stats->lock, flags); |
---|
153 | 155 | } |
---|
154 | | -EXPORT_SYMBOL_GPL(blk_stat_add_callback); |
---|
155 | 156 | |
---|
156 | 157 | void blk_stat_remove_callback(struct request_queue *q, |
---|
157 | 158 | struct blk_stat_callback *cb) |
---|
158 | 159 | { |
---|
159 | | - spin_lock(&q->stats->lock); |
---|
| 160 | + unsigned long flags; |
---|
| 161 | + |
---|
| 162 | + spin_lock_irqsave(&q->stats->lock, flags); |
---|
160 | 163 | list_del_rcu(&cb->list); |
---|
161 | 164 | if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting) |
---|
162 | 165 | blk_queue_flag_clear(QUEUE_FLAG_STATS, q); |
---|
163 | | - spin_unlock(&q->stats->lock); |
---|
| 166 | + spin_unlock_irqrestore(&q->stats->lock, flags); |
---|
164 | 167 | |
---|
165 | 168 | del_timer_sync(&cb->timer); |
---|
166 | 169 | } |
---|
167 | | -EXPORT_SYMBOL_GPL(blk_stat_remove_callback); |
---|
168 | 170 | |
---|
169 | 171 | static void blk_stat_free_callback_rcu(struct rcu_head *head) |
---|
170 | 172 | { |
---|
.. | .. |
---|
181 | 183 | if (cb) |
---|
182 | 184 | call_rcu(&cb->rcu, blk_stat_free_callback_rcu); |
---|
183 | 185 | } |
---|
184 | | -EXPORT_SYMBOL_GPL(blk_stat_free_callback); |
---|
185 | 186 | |
---|
186 | 187 | void blk_stat_enable_accounting(struct request_queue *q) |
---|
187 | 188 | { |
---|
188 | | - spin_lock(&q->stats->lock); |
---|
| 189 | + unsigned long flags; |
---|
| 190 | + |
---|
| 191 | + spin_lock_irqsave(&q->stats->lock, flags); |
---|
189 | 192 | q->stats->enable_accounting = true; |
---|
190 | 193 | blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
---|
191 | | - spin_unlock(&q->stats->lock); |
---|
| 194 | + spin_unlock_irqrestore(&q->stats->lock, flags); |
---|
192 | 195 | } |
---|
| 196 | +EXPORT_SYMBOL_GPL(blk_stat_enable_accounting); |
---|
193 | 197 | |
---|
194 | 198 | struct blk_queue_stats *blk_alloc_queue_stats(void) |
---|
195 | 199 | { |
---|