.. | .. |
---|
272 | 272 | unsigned long flags; |
---|
273 | 273 | |
---|
274 | 274 | /* Interrupts need to be off for FSGSBASE */ |
---|
275 | | - local_irq_save(flags); |
---|
| 275 | + local_irq_save_full(flags); |
---|
276 | 276 | save_fsgs(current); |
---|
277 | | - local_irq_restore(flags); |
---|
| 277 | + local_irq_restore_full(flags); |
---|
278 | 278 | } |
---|
279 | 279 | #if IS_ENABLED(CONFIG_KVM) |
---|
280 | 280 | EXPORT_SYMBOL_GPL(current_save_fsgs); |
---|
.. | .. |
---|
410 | 410 | if (boot_cpu_has(X86_FEATURE_FSGSBASE)) { |
---|
411 | 411 | unsigned long flags; |
---|
412 | 412 | |
---|
413 | | - local_irq_save(flags); |
---|
| 413 | + local_irq_save_full(flags); |
---|
414 | 414 | gsbase = __rdgsbase_inactive(); |
---|
415 | | - local_irq_restore(flags); |
---|
| 415 | + local_irq_restore_full(flags); |
---|
416 | 416 | } else { |
---|
417 | 417 | rdmsrl(MSR_KERNEL_GS_BASE, gsbase); |
---|
418 | 418 | } |
---|
.. | .. |
---|
425 | 425 | if (boot_cpu_has(X86_FEATURE_FSGSBASE)) { |
---|
426 | 426 | unsigned long flags; |
---|
427 | 427 | |
---|
428 | | - local_irq_save(flags); |
---|
| 428 | + local_irq_save_full(flags); |
---|
429 | 429 | __wrgsbase_inactive(gsbase); |
---|
430 | | - local_irq_restore(flags); |
---|
| 430 | + local_irq_restore_full(flags); |
---|
431 | 431 | } else { |
---|
432 | 432 | wrmsrl(MSR_KERNEL_GS_BASE, gsbase); |
---|
433 | 433 | } |
---|
.. | .. |
---|
537 | 537 | struct thread_struct *next = &next_p->thread; |
---|
538 | 538 | int cpu = smp_processor_id(); |
---|
539 | 539 | |
---|
| 540 | + /* |
---|
| 541 | + * Dovetail: Switching context on the out-of-band stage is |
---|
| 542 | + * legit, and we may have preempted an in-band (soft)irq |
---|
| 543 | + * handler earlier. Since oob handlers never switch stack, |
---|
| 544 | + * make sure to restrict the following test to in-band |
---|
| 545 | + * callers. |
---|
| 546 | + */ |
---|
540 | 547 | WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && |
---|
541 | | - this_cpu_read(irq_count) != -1); |
---|
| 548 | + running_inband() && this_cpu_read(irq_count) != -1); |
---|
| 549 | + |
---|
| 550 | + WARN_ON_ONCE(dovetail_debug() && !hard_irqs_disabled()); |
---|
542 | 551 | |
---|
543 | 552 | if (!test_thread_flag(TIF_NEED_FPU_LOAD)) |
---|
544 | 553 | switch_fpu_prepare(prev_p, cpu); |
---|
.. | .. |
---|
719 | 728 | |
---|
720 | 729 | long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2) |
---|
721 | 730 | { |
---|
| 731 | + unsigned long flags; |
---|
722 | 732 | int ret = 0; |
---|
723 | 733 | |
---|
724 | 734 | switch (option) { |
---|
.. | .. |
---|
726 | 736 | if (unlikely(arg2 >= TASK_SIZE_MAX)) |
---|
727 | 737 | return -EPERM; |
---|
728 | 738 | |
---|
729 | | - preempt_disable(); |
---|
| 739 | + flags = hard_preempt_disable(); |
---|
730 | 740 | /* |
---|
731 | 741 | * ARCH_SET_GS has always overwritten the index |
---|
732 | 742 | * and the base. Zero is the most sensible value |
---|
.. | .. |
---|
747 | 757 | task->thread.gsindex = 0; |
---|
748 | 758 | x86_gsbase_write_task(task, arg2); |
---|
749 | 759 | } |
---|
750 | | - preempt_enable(); |
---|
| 760 | + hard_preempt_enable(flags); |
---|
751 | 761 | break; |
---|
752 | 762 | } |
---|
753 | 763 | case ARCH_SET_FS: { |
---|
.. | .. |
---|
758 | 768 | if (unlikely(arg2 >= TASK_SIZE_MAX)) |
---|
759 | 769 | return -EPERM; |
---|
760 | 770 | |
---|
761 | | - preempt_disable(); |
---|
| 771 | + flags = hard_preempt_disable(); |
---|
762 | 772 | /* |
---|
763 | 773 | * Set the selector to 0 for the same reason |
---|
764 | 774 | * as %gs above. |
---|
.. | .. |
---|
776 | 786 | task->thread.fsindex = 0; |
---|
777 | 787 | x86_fsbase_write_task(task, arg2); |
---|
778 | 788 | } |
---|
779 | | - preempt_enable(); |
---|
| 789 | + hard_preempt_enable(flags); |
---|
780 | 790 | break; |
---|
781 | 791 | } |
---|
782 | 792 | case ARCH_GET_FS: { |
---|