| .. | .. |
|---|
| 4 | 4 | |
|---|
| 5 | 5 | #include <linux/hardirq.h> |
|---|
| 6 | 6 | #include <linux/uaccess.h> |
|---|
| 7 | +#include <linux/refcount.h> |
|---|
| 7 | 8 | |
|---|
| 8 | 9 | /* Buffer handling */ |
|---|
| 9 | 10 | |
|---|
| 10 | 11 | #define RING_BUFFER_WRITABLE 0x01 |
|---|
| 11 | 12 | |
|---|
| 12 | | -struct ring_buffer { |
|---|
| 13 | | - atomic_t refcount; |
|---|
| 13 | +struct perf_buffer { |
|---|
| 14 | + refcount_t refcount; |
|---|
| 14 | 15 | struct rcu_head rcu_head; |
|---|
| 15 | 16 | #ifdef CONFIG_PERF_USE_VMALLOC |
|---|
| 16 | 17 | struct work_struct work; |
|---|
| .. | .. |
|---|
| 23 | 24 | atomic_t poll; /* POLL_ for wakeups */ |
|---|
| 24 | 25 | |
|---|
| 25 | 26 | local_t head; /* write position */ |
|---|
| 26 | | - local_t nest; /* nested writers */ |
|---|
| 27 | + unsigned int nest; /* nested writers */ |
|---|
| 27 | 28 | local_t events; /* event limit */ |
|---|
| 28 | 29 | local_t wakeup; /* wakeup stamp */ |
|---|
| 29 | 30 | local_t lost; /* nr records lost */ |
|---|
| .. | .. |
|---|
| 40 | 41 | |
|---|
| 41 | 42 | /* AUX area */ |
|---|
| 42 | 43 | long aux_head; |
|---|
| 43 | | - local_t aux_nest; |
|---|
| 44 | + unsigned int aux_nest; |
|---|
| 44 | 45 | long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */ |
|---|
| 45 | 46 | unsigned long aux_pgoff; |
|---|
| 46 | 47 | int aux_nr_pages; |
|---|
| .. | .. |
|---|
| 48 | 49 | atomic_t aux_mmap_count; |
|---|
| 49 | 50 | unsigned long aux_mmap_locked; |
|---|
| 50 | 51 | void (*free_aux)(void *); |
|---|
| 51 | | - atomic_t aux_refcount; |
|---|
| 52 | + refcount_t aux_refcount; |
|---|
| 53 | + int aux_in_sampling; |
|---|
| 52 | 54 | void **aux_pages; |
|---|
| 53 | 55 | void *aux_priv; |
|---|
| 54 | 56 | |
|---|
| 55 | 57 | struct perf_event_mmap_page *user_page; |
|---|
| 56 | | - void *data_pages[0]; |
|---|
| 58 | + void *data_pages[]; |
|---|
| 57 | 59 | }; |
|---|
| 58 | 60 | |
|---|
| 59 | | -extern void rb_free(struct ring_buffer *rb); |
|---|
| 61 | +extern void rb_free(struct perf_buffer *rb); |
|---|
| 60 | 62 | |
|---|
| 61 | 63 | static inline void rb_free_rcu(struct rcu_head *rcu_head) |
|---|
| 62 | 64 | { |
|---|
| 63 | | - struct ring_buffer *rb; |
|---|
| 65 | + struct perf_buffer *rb; |
|---|
| 64 | 66 | |
|---|
| 65 | | - rb = container_of(rcu_head, struct ring_buffer, rcu_head); |
|---|
| 67 | + rb = container_of(rcu_head, struct perf_buffer, rcu_head); |
|---|
| 66 | 68 | rb_free(rb); |
|---|
| 67 | 69 | } |
|---|
| 68 | 70 | |
|---|
| 69 | | -static inline void rb_toggle_paused(struct ring_buffer *rb, bool pause) |
|---|
| 71 | +static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause) |
|---|
| 70 | 72 | { |
|---|
| 71 | 73 | if (!pause && rb->nr_pages) |
|---|
| 72 | 74 | rb->paused = 0; |
|---|
| .. | .. |
|---|
| 74 | 76 | rb->paused = 1; |
|---|
| 75 | 77 | } |
|---|
| 76 | 78 | |
|---|
| 77 | | -extern struct ring_buffer * |
|---|
| 79 | +extern struct perf_buffer * |
|---|
| 78 | 80 | rb_alloc(int nr_pages, long watermark, int cpu, int flags); |
|---|
| 79 | 81 | extern void perf_event_wakeup(struct perf_event *event); |
|---|
| 80 | | -extern int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event, |
|---|
| 82 | +extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, |
|---|
| 81 | 83 | pgoff_t pgoff, int nr_pages, long watermark, int flags); |
|---|
| 82 | | -extern void rb_free_aux(struct ring_buffer *rb); |
|---|
| 83 | | -extern struct ring_buffer *ring_buffer_get(struct perf_event *event); |
|---|
| 84 | | -extern void ring_buffer_put(struct ring_buffer *rb); |
|---|
| 84 | +extern void rb_free_aux(struct perf_buffer *rb); |
|---|
| 85 | +extern struct perf_buffer *ring_buffer_get(struct perf_event *event); |
|---|
| 86 | +extern void ring_buffer_put(struct perf_buffer *rb); |
|---|
| 85 | 87 | |
|---|
| 86 | | -static inline bool rb_has_aux(struct ring_buffer *rb) |
|---|
| 88 | +static inline bool rb_has_aux(struct perf_buffer *rb) |
|---|
| 87 | 89 | { |
|---|
| 88 | 90 | return !!rb->aux_nr_pages; |
|---|
| 89 | 91 | } |
|---|
| .. | .. |
|---|
| 92 | 94 | unsigned long size, u64 flags); |
|---|
| 93 | 95 | |
|---|
| 94 | 96 | extern struct page * |
|---|
| 95 | | -perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff); |
|---|
| 97 | +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff); |
|---|
| 96 | 98 | |
|---|
| 97 | 99 | #ifdef CONFIG_PERF_USE_VMALLOC |
|---|
| 98 | 100 | /* |
|---|
| .. | .. |
|---|
| 101 | 103 | * Required for architectures that have d-cache aliasing issues. |
|---|
| 102 | 104 | */ |
|---|
| 103 | 105 | |
|---|
| 104 | | -static inline int page_order(struct ring_buffer *rb) |
|---|
| 106 | +static inline int page_order(struct perf_buffer *rb) |
|---|
| 105 | 107 | { |
|---|
| 106 | 108 | return rb->page_order; |
|---|
| 107 | 109 | } |
|---|
| 108 | 110 | |
|---|
| 109 | 111 | #else |
|---|
| 110 | 112 | |
|---|
| 111 | | -static inline int page_order(struct ring_buffer *rb) |
|---|
| 113 | +static inline int page_order(struct perf_buffer *rb) |
|---|
| 112 | 114 | { |
|---|
| 113 | 115 | return 0; |
|---|
| 114 | 116 | } |
|---|
| 115 | 117 | #endif |
|---|
| 116 | 118 | |
|---|
| 117 | | -static inline unsigned long perf_data_size(struct ring_buffer *rb) |
|---|
| 119 | +static inline int data_page_nr(struct perf_buffer *rb) |
|---|
| 120 | +{ |
|---|
| 121 | + return rb->nr_pages << page_order(rb); |
|---|
| 122 | +} |
|---|
| 123 | + |
|---|
| 124 | +static inline unsigned long perf_data_size(struct perf_buffer *rb) |
|---|
| 118 | 125 | { |
|---|
| 119 | 126 | return rb->nr_pages << (PAGE_SHIFT + page_order(rb)); |
|---|
| 120 | 127 | } |
|---|
| 121 | 128 | |
|---|
| 122 | | -static inline unsigned long perf_aux_size(struct ring_buffer *rb) |
|---|
| 129 | +static inline unsigned long perf_aux_size(struct perf_buffer *rb) |
|---|
| 123 | 130 | { |
|---|
| 124 | 131 | return rb->aux_nr_pages << PAGE_SHIFT; |
|---|
| 125 | 132 | } |
|---|
| .. | .. |
|---|
| 139 | 146 | buf += written; \ |
|---|
| 140 | 147 | handle->size -= written; \ |
|---|
| 141 | 148 | if (!handle->size) { \ |
|---|
| 142 | | - struct ring_buffer *rb = handle->rb; \ |
|---|
| 149 | + struct perf_buffer *rb = handle->rb; \ |
|---|
| 143 | 150 | \ |
|---|
| 144 | 151 | handle->page++; \ |
|---|
| 145 | 152 | handle->page &= rb->nr_pages - 1; \ |
|---|
| .. | .. |
|---|
| 203 | 210 | |
|---|
| 204 | 211 | static inline int get_recursion_context(int *recursion) |
|---|
| 205 | 212 | { |
|---|
| 206 | | - int rctx; |
|---|
| 213 | + unsigned int pc = preempt_count(); |
|---|
| 214 | + unsigned char rctx = 0; |
|---|
| 207 | 215 | |
|---|
| 208 | | - if (unlikely(in_nmi())) |
|---|
| 209 | | - rctx = 3; |
|---|
| 210 | | - else if (in_irq()) |
|---|
| 211 | | - rctx = 2; |
|---|
| 212 | | - else if (in_serving_softirq()) |
|---|
| 213 | | - rctx = 1; |
|---|
| 214 | | - else |
|---|
| 215 | | - rctx = 0; |
|---|
| 216 | + rctx += !!(pc & (NMI_MASK)); |
|---|
| 217 | + rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK)); |
|---|
| 218 | + rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)); |
|---|
| 216 | 219 | |
|---|
| 217 | 220 | if (recursion[rctx]) |
|---|
| 218 | 221 | return -1; |
|---|