| .. | .. |
|---|
| 272 | 272 | unsigned long flags; |
|---|
| 273 | 273 | |
|---|
| 274 | 274 | /* Interrupts need to be off for FSGSBASE */ |
|---|
| 275 | | - local_irq_save(flags); |
|---|
| 275 | + local_irq_save_full(flags); |
|---|
| 276 | 276 | save_fsgs(current); |
|---|
| 277 | | - local_irq_restore(flags); |
|---|
| 277 | + local_irq_restore_full(flags); |
|---|
| 278 | 278 | } |
|---|
| 279 | 279 | #if IS_ENABLED(CONFIG_KVM) |
|---|
| 280 | 280 | EXPORT_SYMBOL_GPL(current_save_fsgs); |
|---|
| .. | .. |
|---|
| 410 | 410 | if (boot_cpu_has(X86_FEATURE_FSGSBASE)) { |
|---|
| 411 | 411 | unsigned long flags; |
|---|
| 412 | 412 | |
|---|
| 413 | | - local_irq_save(flags); |
|---|
| 413 | + local_irq_save_full(flags); |
|---|
| 414 | 414 | gsbase = __rdgsbase_inactive(); |
|---|
| 415 | | - local_irq_restore(flags); |
|---|
| 415 | + local_irq_restore_full(flags); |
|---|
| 416 | 416 | } else { |
|---|
| 417 | 417 | rdmsrl(MSR_KERNEL_GS_BASE, gsbase); |
|---|
| 418 | 418 | } |
|---|
| .. | .. |
|---|
| 425 | 425 | if (boot_cpu_has(X86_FEATURE_FSGSBASE)) { |
|---|
| 426 | 426 | unsigned long flags; |
|---|
| 427 | 427 | |
|---|
| 428 | | - local_irq_save(flags); |
|---|
| 428 | + local_irq_save_full(flags); |
|---|
| 429 | 429 | __wrgsbase_inactive(gsbase); |
|---|
| 430 | | - local_irq_restore(flags); |
|---|
| 430 | + local_irq_restore_full(flags); |
|---|
| 431 | 431 | } else { |
|---|
| 432 | 432 | wrmsrl(MSR_KERNEL_GS_BASE, gsbase); |
|---|
| 433 | 433 | } |
|---|
| .. | .. |
|---|
| 537 | 537 | struct thread_struct *next = &next_p->thread; |
|---|
| 538 | 538 | int cpu = smp_processor_id(); |
|---|
| 539 | 539 | |
|---|
| 540 | + /* |
|---|
| 541 | + * Dovetail: Switching context on the out-of-band stage is |
|---|
| 542 | + * legit, and we may have preempted an in-band (soft)irq |
|---|
| 543 | + * handler earlier. Since oob handlers never switch stack, |
|---|
| 544 | + * make sure to restrict the following test to in-band |
|---|
| 545 | + * callers. |
|---|
| 546 | + */ |
|---|
| 540 | 547 | WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && |
|---|
| 541 | | - this_cpu_read(irq_count) != -1); |
|---|
| 548 | + running_inband() && this_cpu_read(irq_count) != -1); |
|---|
| 549 | + |
|---|
| 550 | + WARN_ON_ONCE(dovetail_debug() && !hard_irqs_disabled()); |
|---|
| 542 | 551 | |
|---|
| 543 | 552 | if (!test_thread_flag(TIF_NEED_FPU_LOAD)) |
|---|
| 544 | 553 | switch_fpu_prepare(prev_p, cpu); |
|---|
| .. | .. |
|---|
| 719 | 728 | |
|---|
| 720 | 729 | long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2) |
|---|
| 721 | 730 | { |
|---|
| 731 | + unsigned long flags; |
|---|
| 722 | 732 | int ret = 0; |
|---|
| 723 | 733 | |
|---|
| 724 | 734 | switch (option) { |
|---|
| .. | .. |
|---|
| 726 | 736 | if (unlikely(arg2 >= TASK_SIZE_MAX)) |
|---|
| 727 | 737 | return -EPERM; |
|---|
| 728 | 738 | |
|---|
| 729 | | - preempt_disable(); |
|---|
| 739 | + flags = hard_preempt_disable(); |
|---|
| 730 | 740 | /* |
|---|
| 731 | 741 | * ARCH_SET_GS has always overwritten the index |
|---|
| 732 | 742 | * and the base. Zero is the most sensible value |
|---|
| .. | .. |
|---|
| 747 | 757 | task->thread.gsindex = 0; |
|---|
| 748 | 758 | x86_gsbase_write_task(task, arg2); |
|---|
| 749 | 759 | } |
|---|
| 750 | | - preempt_enable(); |
|---|
| 760 | + hard_preempt_enable(flags); |
|---|
| 751 | 761 | break; |
|---|
| 752 | 762 | } |
|---|
| 753 | 763 | case ARCH_SET_FS: { |
|---|
| .. | .. |
|---|
| 758 | 768 | if (unlikely(arg2 >= TASK_SIZE_MAX)) |
|---|
| 759 | 769 | return -EPERM; |
|---|
| 760 | 770 | |
|---|
| 761 | | - preempt_disable(); |
|---|
| 771 | + flags = hard_preempt_disable(); |
|---|
| 762 | 772 | /* |
|---|
| 763 | 773 | * Set the selector to 0 for the same reason |
|---|
| 764 | 774 | * as %gs above. |
|---|
| .. | .. |
|---|
| 776 | 786 | task->thread.fsindex = 0; |
|---|
| 777 | 787 | x86_fsbase_write_task(task, arg2); |
|---|
| 778 | 788 | } |
|---|
| 779 | | - preempt_enable(); |
|---|
| 789 | + hard_preempt_enable(flags); |
|---|
| 780 | 790 | break; |
|---|
| 781 | 791 | } |
|---|
| 782 | 792 | case ARCH_GET_FS: { |
|---|