| .. | .. |
|---|
| 7 | 7 | #include <linux/page-flags.h> |
|---|
| 8 | 8 | #include <linux/tracepoint-defs.h> |
|---|
| 9 | 9 | |
|---|
| 10 | | -extern struct tracepoint __tracepoint_page_ref_set; |
|---|
| 11 | | -extern struct tracepoint __tracepoint_page_ref_mod; |
|---|
| 12 | | -extern struct tracepoint __tracepoint_page_ref_mod_and_test; |
|---|
| 13 | | -extern struct tracepoint __tracepoint_page_ref_mod_and_return; |
|---|
| 14 | | -extern struct tracepoint __tracepoint_page_ref_mod_unless; |
|---|
| 15 | | -extern struct tracepoint __tracepoint_page_ref_freeze; |
|---|
| 16 | | -extern struct tracepoint __tracepoint_page_ref_unfreeze; |
|---|
| 10 | +DECLARE_TRACEPOINT(page_ref_set); |
|---|
| 11 | +DECLARE_TRACEPOINT(page_ref_mod); |
|---|
| 12 | +DECLARE_TRACEPOINT(page_ref_mod_and_test); |
|---|
| 13 | +DECLARE_TRACEPOINT(page_ref_mod_and_return); |
|---|
| 14 | +DECLARE_TRACEPOINT(page_ref_mod_unless); |
|---|
| 15 | +DECLARE_TRACEPOINT(page_ref_freeze); |
|---|
| 16 | +DECLARE_TRACEPOINT(page_ref_unfreeze); |
|---|
| 17 | 17 | |
|---|
| 18 | 18 | #ifdef CONFIG_DEBUG_PAGE_REF |
|---|
| 19 | 19 | |
|---|
| .. | .. |
|---|
| 24 | 24 | * |
|---|
| 25 | 25 | * See trace_##name##_enabled(void) in include/linux/tracepoint.h |
|---|
| 26 | 26 | */ |
|---|
| 27 | | -#define page_ref_tracepoint_active(t) static_key_false(&(t).key) |
|---|
| 27 | +#define page_ref_tracepoint_active(t) tracepoint_enabled(t) |
|---|
| 28 | 28 | |
|---|
| 29 | 29 | extern void __page_ref_set(struct page *page, int v); |
|---|
| 30 | 30 | extern void __page_ref_mod(struct page *page, int v); |
|---|
| .. | .. |
|---|
| 75 | 75 | static inline void set_page_count(struct page *page, int v) |
|---|
| 76 | 76 | { |
|---|
| 77 | 77 | atomic_set(&page->_refcount, v); |
|---|
| 78 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_set)) |
|---|
| 78 | + if (page_ref_tracepoint_active(page_ref_set)) |
|---|
| 79 | 79 | __page_ref_set(page, v); |
|---|
| 80 | 80 | } |
|---|
| 81 | 81 | |
|---|
| .. | .. |
|---|
| 91 | 91 | static inline void page_ref_add(struct page *page, int nr) |
|---|
| 92 | 92 | { |
|---|
| 93 | 93 | atomic_add(nr, &page->_refcount); |
|---|
| 94 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod)) |
|---|
| 94 | + if (page_ref_tracepoint_active(page_ref_mod)) |
|---|
| 95 | 95 | __page_ref_mod(page, nr); |
|---|
| 96 | 96 | } |
|---|
| 97 | 97 | |
|---|
| 98 | 98 | static inline void page_ref_sub(struct page *page, int nr) |
|---|
| 99 | 99 | { |
|---|
| 100 | 100 | atomic_sub(nr, &page->_refcount); |
|---|
| 101 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod)) |
|---|
| 101 | + if (page_ref_tracepoint_active(page_ref_mod)) |
|---|
| 102 | 102 | __page_ref_mod(page, -nr); |
|---|
| 103 | +} |
|---|
| 104 | + |
|---|
| 105 | +static inline int page_ref_sub_return(struct page *page, int nr) |
|---|
| 106 | +{ |
|---|
| 107 | + int ret = atomic_sub_return(nr, &page->_refcount); |
|---|
| 108 | + |
|---|
| 109 | + if (page_ref_tracepoint_active(page_ref_mod_and_return)) |
|---|
| 110 | + __page_ref_mod_and_return(page, -nr, ret); |
|---|
| 111 | + return ret; |
|---|
| 103 | 112 | } |
|---|
| 104 | 113 | |
|---|
| 105 | 114 | static inline void page_ref_inc(struct page *page) |
|---|
| 106 | 115 | { |
|---|
| 107 | 116 | atomic_inc(&page->_refcount); |
|---|
| 108 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod)) |
|---|
| 117 | + if (page_ref_tracepoint_active(page_ref_mod)) |
|---|
| 109 | 118 | __page_ref_mod(page, 1); |
|---|
| 110 | 119 | } |
|---|
| 111 | 120 | |
|---|
| 112 | 121 | static inline void page_ref_dec(struct page *page) |
|---|
| 113 | 122 | { |
|---|
| 114 | 123 | atomic_dec(&page->_refcount); |
|---|
| 115 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod)) |
|---|
| 124 | + if (page_ref_tracepoint_active(page_ref_mod)) |
|---|
| 116 | 125 | __page_ref_mod(page, -1); |
|---|
| 117 | 126 | } |
|---|
| 118 | 127 | |
|---|
| .. | .. |
|---|
| 120 | 129 | { |
|---|
| 121 | 130 | int ret = atomic_sub_and_test(nr, &page->_refcount); |
|---|
| 122 | 131 | |
|---|
| 123 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test)) |
|---|
| 132 | + if (page_ref_tracepoint_active(page_ref_mod_and_test)) |
|---|
| 124 | 133 | __page_ref_mod_and_test(page, -nr, ret); |
|---|
| 125 | 134 | return ret; |
|---|
| 126 | 135 | } |
|---|
| .. | .. |
|---|
| 129 | 138 | { |
|---|
| 130 | 139 | int ret = atomic_inc_return(&page->_refcount); |
|---|
| 131 | 140 | |
|---|
| 132 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return)) |
|---|
| 141 | + if (page_ref_tracepoint_active(page_ref_mod_and_return)) |
|---|
| 133 | 142 | __page_ref_mod_and_return(page, 1, ret); |
|---|
| 134 | 143 | return ret; |
|---|
| 135 | 144 | } |
|---|
| .. | .. |
|---|
| 138 | 147 | { |
|---|
| 139 | 148 | int ret = atomic_dec_and_test(&page->_refcount); |
|---|
| 140 | 149 | |
|---|
| 141 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test)) |
|---|
| 150 | + if (page_ref_tracepoint_active(page_ref_mod_and_test)) |
|---|
| 142 | 151 | __page_ref_mod_and_test(page, -1, ret); |
|---|
| 143 | 152 | return ret; |
|---|
| 144 | 153 | } |
|---|
| .. | .. |
|---|
| 147 | 156 | { |
|---|
| 148 | 157 | int ret = atomic_dec_return(&page->_refcount); |
|---|
| 149 | 158 | |
|---|
| 150 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return)) |
|---|
| 159 | + if (page_ref_tracepoint_active(page_ref_mod_and_return)) |
|---|
| 151 | 160 | __page_ref_mod_and_return(page, -1, ret); |
|---|
| 152 | 161 | return ret; |
|---|
| 153 | 162 | } |
|---|
| .. | .. |
|---|
| 156 | 165 | { |
|---|
| 157 | 166 | int ret = atomic_add_unless(&page->_refcount, nr, u); |
|---|
| 158 | 167 | |
|---|
| 159 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_unless)) |
|---|
| 168 | + if (page_ref_tracepoint_active(page_ref_mod_unless)) |
|---|
| 160 | 169 | __page_ref_mod_unless(page, nr, ret); |
|---|
| 161 | 170 | return ret; |
|---|
| 162 | 171 | } |
|---|
| .. | .. |
|---|
| 165 | 174 | { |
|---|
| 166 | 175 | int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count); |
|---|
| 167 | 176 | |
|---|
| 168 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_freeze)) |
|---|
| 177 | + if (page_ref_tracepoint_active(page_ref_freeze)) |
|---|
| 169 | 178 | __page_ref_freeze(page, count, ret); |
|---|
| 170 | 179 | return ret; |
|---|
| 171 | 180 | } |
|---|
| .. | .. |
|---|
| 176 | 185 | VM_BUG_ON(count == 0); |
|---|
| 177 | 186 | |
|---|
| 178 | 187 | atomic_set_release(&page->_refcount, count); |
|---|
| 179 | | - if (page_ref_tracepoint_active(__tracepoint_page_ref_unfreeze)) |
|---|
| 188 | + if (page_ref_tracepoint_active(page_ref_unfreeze)) |
|---|
| 180 | 189 | __page_ref_unfreeze(page, count); |
|---|
| 181 | 190 | } |
|---|
| 182 | 191 | |
|---|