.. | .. |
---|
4 | 4 | |
---|
5 | 5 | #include <linux/hardirq.h> |
---|
6 | 6 | #include <linux/uaccess.h> |
---|
| 7 | +#include <linux/refcount.h> |
---|
7 | 8 | |
---|
8 | 9 | /* Buffer handling */ |
---|
9 | 10 | |
---|
10 | 11 | #define RING_BUFFER_WRITABLE 0x01 |
---|
11 | 12 | |
---|
12 | | -struct ring_buffer { |
---|
13 | | - atomic_t refcount; |
---|
| 13 | +struct perf_buffer { |
---|
| 14 | + refcount_t refcount; |
---|
14 | 15 | struct rcu_head rcu_head; |
---|
15 | 16 | #ifdef CONFIG_PERF_USE_VMALLOC |
---|
16 | 17 | struct work_struct work; |
---|
.. | .. |
---|
23 | 24 | atomic_t poll; /* POLL_ for wakeups */ |
---|
24 | 25 | |
---|
25 | 26 | local_t head; /* write position */ |
---|
26 | | - local_t nest; /* nested writers */ |
---|
| 27 | + unsigned int nest; /* nested writers */ |
---|
27 | 28 | local_t events; /* event limit */ |
---|
28 | 29 | local_t wakeup; /* wakeup stamp */ |
---|
29 | 30 | local_t lost; /* nr records lost */ |
---|
.. | .. |
---|
40 | 41 | |
---|
41 | 42 | /* AUX area */ |
---|
42 | 43 | long aux_head; |
---|
43 | | - local_t aux_nest; |
---|
| 44 | + unsigned int aux_nest; |
---|
44 | 45 | long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */ |
---|
45 | 46 | unsigned long aux_pgoff; |
---|
46 | 47 | int aux_nr_pages; |
---|
.. | .. |
---|
48 | 49 | atomic_t aux_mmap_count; |
---|
49 | 50 | unsigned long aux_mmap_locked; |
---|
50 | 51 | void (*free_aux)(void *); |
---|
51 | | - atomic_t aux_refcount; |
---|
| 52 | + refcount_t aux_refcount; |
---|
| 53 | + int aux_in_sampling; |
---|
52 | 54 | void **aux_pages; |
---|
53 | 55 | void *aux_priv; |
---|
54 | 56 | |
---|
55 | 57 | struct perf_event_mmap_page *user_page; |
---|
56 | | - void *data_pages[0]; |
---|
| 58 | + void *data_pages[]; |
---|
57 | 59 | }; |
---|
58 | 60 | |
---|
59 | | -extern void rb_free(struct ring_buffer *rb); |
---|
| 61 | +extern void rb_free(struct perf_buffer *rb); |
---|
60 | 62 | |
---|
61 | 63 | static inline void rb_free_rcu(struct rcu_head *rcu_head) |
---|
62 | 64 | { |
---|
63 | | - struct ring_buffer *rb; |
---|
| 65 | + struct perf_buffer *rb; |
---|
64 | 66 | |
---|
65 | | - rb = container_of(rcu_head, struct ring_buffer, rcu_head); |
---|
| 67 | + rb = container_of(rcu_head, struct perf_buffer, rcu_head); |
---|
66 | 68 | rb_free(rb); |
---|
67 | 69 | } |
---|
68 | 70 | |
---|
69 | | -static inline void rb_toggle_paused(struct ring_buffer *rb, bool pause) |
---|
| 71 | +static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause) |
---|
70 | 72 | { |
---|
71 | 73 | if (!pause && rb->nr_pages) |
---|
72 | 74 | rb->paused = 0; |
---|
.. | .. |
---|
74 | 76 | rb->paused = 1; |
---|
75 | 77 | } |
---|
76 | 78 | |
---|
77 | | -extern struct ring_buffer * |
---|
| 79 | +extern struct perf_buffer * |
---|
78 | 80 | rb_alloc(int nr_pages, long watermark, int cpu, int flags); |
---|
79 | 81 | extern void perf_event_wakeup(struct perf_event *event); |
---|
80 | | -extern int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event, |
---|
| 82 | +extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, |
---|
81 | 83 | pgoff_t pgoff, int nr_pages, long watermark, int flags); |
---|
82 | | -extern void rb_free_aux(struct ring_buffer *rb); |
---|
83 | | -extern struct ring_buffer *ring_buffer_get(struct perf_event *event); |
---|
84 | | -extern void ring_buffer_put(struct ring_buffer *rb); |
---|
| 84 | +extern void rb_free_aux(struct perf_buffer *rb); |
---|
| 85 | +extern struct perf_buffer *ring_buffer_get(struct perf_event *event); |
---|
| 86 | +extern void ring_buffer_put(struct perf_buffer *rb); |
---|
85 | 87 | |
---|
86 | | -static inline bool rb_has_aux(struct ring_buffer *rb) |
---|
| 88 | +static inline bool rb_has_aux(struct perf_buffer *rb) |
---|
87 | 89 | { |
---|
88 | 90 | return !!rb->aux_nr_pages; |
---|
89 | 91 | } |
---|
.. | .. |
---|
92 | 94 | unsigned long size, u64 flags); |
---|
93 | 95 | |
---|
94 | 96 | extern struct page * |
---|
95 | | -perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff); |
---|
| 97 | +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff); |
---|
96 | 98 | |
---|
97 | 99 | #ifdef CONFIG_PERF_USE_VMALLOC |
---|
98 | 100 | /* |
---|
.. | .. |
---|
101 | 103 | * Required for architectures that have d-cache aliasing issues. |
---|
102 | 104 | */ |
---|
103 | 105 | |
---|
104 | | -static inline int page_order(struct ring_buffer *rb) |
---|
| 106 | +static inline int page_order(struct perf_buffer *rb) |
---|
105 | 107 | { |
---|
106 | 108 | return rb->page_order; |
---|
107 | 109 | } |
---|
108 | 110 | |
---|
109 | 111 | #else |
---|
110 | 112 | |
---|
111 | | -static inline int page_order(struct ring_buffer *rb) |
---|
| 113 | +static inline int page_order(struct perf_buffer *rb) |
---|
112 | 114 | { |
---|
113 | 115 | return 0; |
---|
114 | 116 | } |
---|
115 | 117 | #endif |
---|
116 | 118 | |
---|
117 | | -static inline unsigned long perf_data_size(struct ring_buffer *rb) |
---|
| 119 | +static inline int data_page_nr(struct perf_buffer *rb) |
---|
| 120 | +{ |
---|
| 121 | + return rb->nr_pages << page_order(rb); |
---|
| 122 | +} |
---|
| 123 | + |
---|
| 124 | +static inline unsigned long perf_data_size(struct perf_buffer *rb) |
---|
118 | 125 | { |
---|
119 | 126 | return rb->nr_pages << (PAGE_SHIFT + page_order(rb)); |
---|
120 | 127 | } |
---|
121 | 128 | |
---|
122 | | -static inline unsigned long perf_aux_size(struct ring_buffer *rb) |
---|
| 129 | +static inline unsigned long perf_aux_size(struct perf_buffer *rb) |
---|
123 | 130 | { |
---|
124 | 131 | return rb->aux_nr_pages << PAGE_SHIFT; |
---|
125 | 132 | } |
---|
.. | .. |
---|
139 | 146 | buf += written; \ |
---|
140 | 147 | handle->size -= written; \ |
---|
141 | 148 | if (!handle->size) { \ |
---|
142 | | - struct ring_buffer *rb = handle->rb; \ |
---|
| 149 | + struct perf_buffer *rb = handle->rb; \ |
---|
143 | 150 | \ |
---|
144 | 151 | handle->page++; \ |
---|
145 | 152 | handle->page &= rb->nr_pages - 1; \ |
---|
.. | .. |
---|
203 | 210 | |
---|
204 | 211 | static inline int get_recursion_context(int *recursion) |
---|
205 | 212 | { |
---|
206 | | - int rctx; |
---|
| 213 | + unsigned int pc = preempt_count(); |
---|
| 214 | + unsigned char rctx = 0; |
---|
207 | 215 | |
---|
208 | | - if (unlikely(in_nmi())) |
---|
209 | | - rctx = 3; |
---|
210 | | - else if (in_irq()) |
---|
211 | | - rctx = 2; |
---|
212 | | - else if (in_serving_softirq()) |
---|
213 | | - rctx = 1; |
---|
214 | | - else |
---|
215 | | - rctx = 0; |
---|
| 216 | + rctx += !!(pc & (NMI_MASK)); |
---|
| 217 | + rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK)); |
---|
| 218 | + rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)); |
---|
216 | 219 | |
---|
217 | 220 | if (recursion[rctx]) |
---|
218 | 221 | return -1; |
---|