hc
2024-01-05 071106ecf68c401173c58808b1cf5f68cc50d390
kernel/block/blk-stat.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0
12 /*
23 * Block stat tracking code
34 *
....@@ -52,7 +53,7 @@
5253 struct request_queue *q = rq->q;
5354 struct blk_stat_callback *cb;
5455 struct blk_rq_stat *stat;
55
- int bucket;
56
+ int bucket, cpu;
5657 u64 value;
5758
5859 value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
....@@ -60,6 +61,7 @@
6061 blk_throtl_stat_add(rq, value);
6162
6263 rcu_read_lock();
64
+ cpu = get_cpu();
6365 list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
6466 if (!blk_stat_is_active(cb))
6567 continue;
....@@ -68,10 +70,10 @@
6870 if (bucket < 0)
6971 continue;
7072
71
- stat = &get_cpu_ptr(cb->cpu_stat)[bucket];
73
+ stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket];
7274 blk_rq_stat_add(stat, value);
73
- put_cpu_ptr(cb->cpu_stat);
7475 }
76
+ put_cpu();
7577 rcu_read_unlock();
7678 }
7779
....@@ -130,12 +132,12 @@
130132
131133 return cb;
132134 }
133
-EXPORT_SYMBOL_GPL(blk_stat_alloc_callback);
134135
135136 void blk_stat_add_callback(struct request_queue *q,
136137 struct blk_stat_callback *cb)
137138 {
138139 unsigned int bucket;
140
+ unsigned long flags;
139141 int cpu;
140142
141143 for_each_possible_cpu(cpu) {
....@@ -146,25 +148,25 @@
146148 blk_rq_stat_init(&cpu_stat[bucket]);
147149 }
148150
149
- spin_lock(&q->stats->lock);
151
+ spin_lock_irqsave(&q->stats->lock, flags);
150152 list_add_tail_rcu(&cb->list, &q->stats->callbacks);
151153 blk_queue_flag_set(QUEUE_FLAG_STATS, q);
152
- spin_unlock(&q->stats->lock);
154
+ spin_unlock_irqrestore(&q->stats->lock, flags);
153155 }
154
-EXPORT_SYMBOL_GPL(blk_stat_add_callback);
155156
156157 void blk_stat_remove_callback(struct request_queue *q,
157158 struct blk_stat_callback *cb)
158159 {
159
- spin_lock(&q->stats->lock);
160
+ unsigned long flags;
161
+
162
+ spin_lock_irqsave(&q->stats->lock, flags);
160163 list_del_rcu(&cb->list);
161164 if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting)
162165 blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
163
- spin_unlock(&q->stats->lock);
166
+ spin_unlock_irqrestore(&q->stats->lock, flags);
164167
165168 del_timer_sync(&cb->timer);
166169 }
167
-EXPORT_SYMBOL_GPL(blk_stat_remove_callback);
168170
169171 static void blk_stat_free_callback_rcu(struct rcu_head *head)
170172 {
....@@ -181,15 +183,17 @@
181183 if (cb)
182184 call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
183185 }
184
-EXPORT_SYMBOL_GPL(blk_stat_free_callback);
185186
186187 void blk_stat_enable_accounting(struct request_queue *q)
187188 {
188
- spin_lock(&q->stats->lock);
189
+ unsigned long flags;
190
+
191
+ spin_lock_irqsave(&q->stats->lock, flags);
189192 q->stats->enable_accounting = true;
190193 blk_queue_flag_set(QUEUE_FLAG_STATS, q);
191
- spin_unlock(&q->stats->lock);
194
+ spin_unlock_irqrestore(&q->stats->lock, flags);
192195 }
196
+EXPORT_SYMBOL_GPL(blk_stat_enable_accounting);
193197
194198 struct blk_queue_stats *blk_alloc_queue_stats(void)
195199 {