hc
2023-11-22 f743a7adbd6e230d66a6206fa115b59fec2d88eb
kernel/block/blk-mq.c
....@@ -320,6 +320,9 @@
320320 rq->extra_len = 0;
321321 rq->__deadline = 0;
322322
323
+#ifdef CONFIG_PREEMPT_RT_FULL
324
+ INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work);
325
+#endif
323326 INIT_LIST_HEAD(&rq->timeout_list);
324327 rq->timeout = 0;
325328
....@@ -547,12 +550,24 @@
547550 }
548551 EXPORT_SYMBOL(blk_mq_end_request);
549552
553
+#ifdef CONFIG_PREEMPT_RT_FULL
554
+
555
+void __blk_mq_complete_request_remote_work(struct work_struct *work)
556
+{
557
+ struct request *rq = container_of(work, struct request, work);
558
+
559
+ rq->q->softirq_done_fn(rq);
560
+}
561
+
562
+#else
563
+
550564 static void __blk_mq_complete_request_remote(void *data)
551565 {
552566 struct request *rq = data;
553567
554568 rq->q->softirq_done_fn(rq);
555569 }
570
+#endif
556571
557572 static void __blk_mq_complete_request(struct request *rq)
558573 {
....@@ -570,19 +585,27 @@
570585 return;
571586 }
572587
573
- cpu = get_cpu();
588
+ cpu = get_cpu_light();
574589 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
575590 shared = cpus_share_cache(cpu, ctx->cpu);
576591
577592 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
593
+#ifdef CONFIG_PREEMPT_RT_FULL
594
+ /*
595
+ * We could force QUEUE_FLAG_SAME_FORCE then we would not get in
596
+ * here. But we could try to invoke it one the CPU like this.
597
+ */
598
+ schedule_work_on(ctx->cpu, &rq->work);
599
+#else
578600 rq->csd.func = __blk_mq_complete_request_remote;
579601 rq->csd.info = rq;
580602 rq->csd.flags = 0;
581603 smp_call_function_single_async(ctx->cpu, &rq->csd);
604
+#endif
582605 } else {
583606 rq->q->softirq_done_fn(rq);
584607 }
585
- put_cpu();
608
+ put_cpu_light();
586609 }
587610
588611 static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)
....@@ -1387,14 +1410,14 @@
13871410 return;
13881411
13891412 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1390
- int cpu = get_cpu();
1413
+ int cpu = get_cpu_light();
13911414 if (cpumask_test_cpu(cpu, hctx->cpumask)) {
13921415 __blk_mq_run_hw_queue(hctx);
1393
- put_cpu();
1416
+ put_cpu_light();
13941417 return;
13951418 }
13961419
1397
- put_cpu();
1420
+ put_cpu_light();
13981421 }
13991422
14001423 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
....@@ -3139,10 +3162,9 @@
31393162 kt = nsecs;
31403163
31413164 mode = HRTIMER_MODE_REL;
3142
- hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
3165
+ hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode, current);
31433166 hrtimer_set_expires(&hs.timer, kt);
31443167
3145
- hrtimer_init_sleeper(&hs, current);
31463168 do {
31473169 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
31483170 break;