.. | .. |
---|
8 | 8 | #include <linux/mm.h> |
---|
9 | 9 | #include <linux/uaccess.h> |
---|
10 | 10 | #include <linux/hardirq.h> |
---|
| 11 | +#include <linux/sched.h> |
---|
11 | 12 | |
---|
12 | 13 | #include <asm/cacheflush.h> |
---|
13 | 14 | |
---|
.. | .. |
---|
66 | 67 | |
---|
67 | 68 | static inline void *kmap_atomic(struct page *page) |
---|
68 | 69 | { |
---|
69 | | - preempt_disable(); |
---|
| 70 | + preempt_disable_nort(); |
---|
70 | 71 | pagefault_disable(); |
---|
71 | 72 | return page_address(page); |
---|
72 | 73 | } |
---|
.. | .. |
---|
75 | 76 | static inline void __kunmap_atomic(void *addr) |
---|
76 | 77 | { |
---|
77 | 78 | pagefault_enable(); |
---|
78 | | - preempt_enable(); |
---|
| 79 | + preempt_enable_nort(); |
---|
79 | 80 | } |
---|
80 | 81 | |
---|
81 | 82 | #define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn)) |
---|
.. | .. |
---|
87 | 88 | |
---|
88 | 89 | #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) |
---|
89 | 90 | |
---|
| 91 | +#ifndef CONFIG_PREEMPT_RT_FULL |
---|
90 | 92 | DECLARE_PER_CPU(int, __kmap_atomic_idx); |
---|
| 93 | +#endif |
---|
91 | 94 | |
---|
92 | 95 | static inline int kmap_atomic_idx_push(void) |
---|
93 | 96 | { |
---|
| 97 | +#ifndef CONFIG_PREEMPT_RT_FULL |
---|
94 | 98 | int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1; |
---|
95 | 99 | |
---|
96 | | -#ifdef CONFIG_DEBUG_HIGHMEM |
---|
| 100 | +# ifdef CONFIG_DEBUG_HIGHMEM |
---|
97 | 101 | WARN_ON_ONCE(in_irq() && !irqs_disabled()); |
---|
98 | 102 | BUG_ON(idx >= KM_TYPE_NR); |
---|
99 | | -#endif |
---|
| 103 | +# endif |
---|
100 | 104 | return idx; |
---|
| 105 | +#else |
---|
| 106 | + current->kmap_idx++; |
---|
| 107 | + BUG_ON(current->kmap_idx > KM_TYPE_NR); |
---|
| 108 | + return current->kmap_idx - 1; |
---|
| 109 | +#endif |
---|
101 | 110 | } |
---|
102 | 111 | |
---|
103 | 112 | static inline int kmap_atomic_idx(void) |
---|
104 | 113 | { |
---|
| 114 | +#ifndef CONFIG_PREEMPT_RT_FULL |
---|
105 | 115 | return __this_cpu_read(__kmap_atomic_idx) - 1; |
---|
| 116 | +#else |
---|
| 117 | + return current->kmap_idx - 1; |
---|
| 118 | +#endif |
---|
106 | 119 | } |
---|
107 | 120 | |
---|
108 | 121 | static inline void kmap_atomic_idx_pop(void) |
---|
109 | 122 | { |
---|
110 | | -#ifdef CONFIG_DEBUG_HIGHMEM |
---|
| 123 | +#ifndef CONFIG_PREEMPT_RT_FULL |
---|
| 124 | +# ifdef CONFIG_DEBUG_HIGHMEM |
---|
111 | 125 | int idx = __this_cpu_dec_return(__kmap_atomic_idx); |
---|
112 | 126 | |
---|
113 | 127 | BUG_ON(idx < 0); |
---|
114 | | -#else |
---|
| 128 | +# else |
---|
115 | 129 | __this_cpu_dec(__kmap_atomic_idx); |
---|
| 130 | +# endif |
---|
| 131 | +#else |
---|
| 132 | + current->kmap_idx--; |
---|
| 133 | +# ifdef CONFIG_DEBUG_HIGHMEM |
---|
| 134 | + BUG_ON(current->kmap_idx < 0); |
---|
| 135 | +# endif |
---|
116 | 136 | #endif |
---|
117 | 137 | } |
---|
118 | 138 | |
---|