| .. | .. |
|---|
| 8 | 8 | #include <linux/random.h> |
|---|
| 9 | 9 | #include <linux/signal.h> |
|---|
| 10 | 10 | #include <linux/personality.h> |
|---|
| 11 | +#include <linux/irq_pipeline.h> |
|---|
| 11 | 12 | #include <linux/uaccess.h> |
|---|
| 12 | 13 | #include <linux/tracehook.h> |
|---|
| 13 | 14 | #include <linux/uprobes.h> |
|---|
| .. | .. |
|---|
| 639 | 640 | return 0; |
|---|
| 640 | 641 | } |
|---|
| 641 | 642 | |
|---|
| 643 | +static inline void do_retuser(void) |
|---|
| 644 | +{ |
|---|
| 645 | + unsigned int thread_flags; |
|---|
| 646 | + |
|---|
| 647 | + if (dovetailing()) { |
|---|
| 648 | + thread_flags = current_thread_info()->flags; |
|---|
| 649 | + if (thread_flags & _TIF_RETUSER) |
|---|
| 650 | + inband_retuser_notify(); |
|---|
| 651 | + } |
|---|
| 652 | +} |
|---|
| 653 | + |
|---|
| 642 | 654 | asmlinkage int |
|---|
| 643 | 655 | do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) |
|---|
| 644 | 656 | { |
|---|
| 657 | + WARN_ON_ONCE(irq_pipeline_debug() && |
|---|
| 658 | + (irqs_disabled() || running_oob())); |
|---|
| 659 | + |
|---|
| 645 | 660 | /* |
|---|
| 646 | 661 | * The assembly code enters us with IRQs off, but it hasn't |
|---|
| 647 | 662 | * informed the tracing code of that for efficiency reasons. |
|---|
| 648 | 663 | * Update the trace code with the current status. |
|---|
| 649 | 664 | */ |
|---|
| 650 | | - trace_hardirqs_off(); |
|---|
| 665 | + if (!irqs_pipelined()) |
|---|
| 666 | + trace_hardirqs_off(); |
|---|
| 651 | 667 | do { |
|---|
| 668 | + if (irqs_pipelined()) { |
|---|
| 669 | + local_irq_disable(); |
|---|
| 670 | + hard_cond_local_irq_enable(); |
|---|
| 671 | + } |
|---|
| 672 | + |
|---|
| 652 | 673 | if (likely(thread_flags & _TIF_NEED_RESCHED)) { |
|---|
| 653 | 674 | schedule(); |
|---|
| 654 | 675 | } else { |
|---|
| .. | .. |
|---|
| 658 | 679 | if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { |
|---|
| 659 | 680 | int restart = do_signal(regs, syscall); |
|---|
| 660 | 681 | if (unlikely(restart)) { |
|---|
| 682 | + do_retuser(); |
|---|
| 661 | 683 | /* |
|---|
| 662 | 684 | * Restart without handlers. |
|---|
| 663 | 685 | * Deal with it without leaving |
|---|
| .. | .. |
|---|
| 672 | 694 | tracehook_notify_resume(regs); |
|---|
| 673 | 695 | rseq_handle_notify_resume(NULL, regs); |
|---|
| 674 | 696 | } |
|---|
| 697 | + do_retuser(); |
|---|
| 675 | 698 | } |
|---|
| 676 | | - local_irq_disable(); |
|---|
| 699 | + hard_local_irq_disable(); |
|---|
| 700 | + |
|---|
| 701 | + /* RETUSER might have switched oob */ |
|---|
| 702 | + if (!running_inband()) |
|---|
| 703 | + break; |
|---|
| 704 | + |
|---|
| 677 | 705 | thread_flags = current_thread_info()->flags; |
|---|
| 678 | | - } while (thread_flags & _TIF_WORK_MASK); |
|---|
| 706 | + } while (inband_irq_pending() || (thread_flags & _TIF_WORK_MASK)); |
|---|
| 679 | 707 | return 0; |
|---|
| 680 | 708 | } |
|---|
| 681 | 709 | |
|---|
