hc
2024-05-10 748e4f3d702def1a4bff191e0cf93b6a05340f01
kernel/kernel/events/internal.h
....@@ -4,13 +4,14 @@
44
55 #include <linux/hardirq.h>
66 #include <linux/uaccess.h>
7
+#include <linux/refcount.h>
78
89 /* Buffer handling */
910
1011 #define RING_BUFFER_WRITABLE 0x01
1112
12
-struct ring_buffer {
13
- atomic_t refcount;
13
+struct perf_buffer {
14
+ refcount_t refcount;
1415 struct rcu_head rcu_head;
1516 #ifdef CONFIG_PERF_USE_VMALLOC
1617 struct work_struct work;
....@@ -23,7 +24,7 @@
2324 atomic_t poll; /* POLL_ for wakeups */
2425
2526 local_t head; /* write position */
26
- local_t nest; /* nested writers */
27
+ unsigned int nest; /* nested writers */
2728 local_t events; /* event limit */
2829 local_t wakeup; /* wakeup stamp */
2930 local_t lost; /* nr records lost */
....@@ -40,7 +41,7 @@
4041
4142 /* AUX area */
4243 long aux_head;
43
- local_t aux_nest;
44
+ unsigned int aux_nest;
4445 long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */
4546 unsigned long aux_pgoff;
4647 int aux_nr_pages;
....@@ -48,25 +49,26 @@
4849 atomic_t aux_mmap_count;
4950 unsigned long aux_mmap_locked;
5051 void (*free_aux)(void *);
51
- atomic_t aux_refcount;
52
+ refcount_t aux_refcount;
53
+ int aux_in_sampling;
5254 void **aux_pages;
5355 void *aux_priv;
5456
5557 struct perf_event_mmap_page *user_page;
56
- void *data_pages[0];
58
+ void *data_pages[];
5759 };
5860
59
-extern void rb_free(struct ring_buffer *rb);
61
+extern void rb_free(struct perf_buffer *rb);
6062
6163 static inline void rb_free_rcu(struct rcu_head *rcu_head)
6264 {
63
- struct ring_buffer *rb;
65
+ struct perf_buffer *rb;
6466
65
- rb = container_of(rcu_head, struct ring_buffer, rcu_head);
67
+ rb = container_of(rcu_head, struct perf_buffer, rcu_head);
6668 rb_free(rb);
6769 }
6870
69
-static inline void rb_toggle_paused(struct ring_buffer *rb, bool pause)
71
+static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause)
7072 {
7173 if (!pause && rb->nr_pages)
7274 rb->paused = 0;
....@@ -74,16 +76,16 @@
7476 rb->paused = 1;
7577 }
7678
77
-extern struct ring_buffer *
79
+extern struct perf_buffer *
7880 rb_alloc(int nr_pages, long watermark, int cpu, int flags);
7981 extern void perf_event_wakeup(struct perf_event *event);
80
-extern int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event,
82
+extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,
8183 pgoff_t pgoff, int nr_pages, long watermark, int flags);
82
-extern void rb_free_aux(struct ring_buffer *rb);
83
-extern struct ring_buffer *ring_buffer_get(struct perf_event *event);
84
-extern void ring_buffer_put(struct ring_buffer *rb);
84
+extern void rb_free_aux(struct perf_buffer *rb);
85
+extern struct perf_buffer *ring_buffer_get(struct perf_event *event);
86
+extern void ring_buffer_put(struct perf_buffer *rb);
8587
86
-static inline bool rb_has_aux(struct ring_buffer *rb)
88
+static inline bool rb_has_aux(struct perf_buffer *rb)
8789 {
8890 return !!rb->aux_nr_pages;
8991 }
....@@ -92,7 +94,7 @@
9294 unsigned long size, u64 flags);
9395
9496 extern struct page *
95
-perf_mmap_to_page(struct ring_buffer *rb, unsigned long pgoff);
97
+perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff);
9698
9799 #ifdef CONFIG_PERF_USE_VMALLOC
98100 /*
....@@ -101,25 +103,30 @@
101103 * Required for architectures that have d-cache aliasing issues.
102104 */
103105
104
-static inline int page_order(struct ring_buffer *rb)
106
+static inline int page_order(struct perf_buffer *rb)
105107 {
106108 return rb->page_order;
107109 }
108110
109111 #else
110112
111
-static inline int page_order(struct ring_buffer *rb)
113
+static inline int page_order(struct perf_buffer *rb)
112114 {
113115 return 0;
114116 }
115117 #endif
116118
117
-static inline unsigned long perf_data_size(struct ring_buffer *rb)
119
+static inline int data_page_nr(struct perf_buffer *rb)
120
+{
121
+ return rb->nr_pages << page_order(rb);
122
+}
123
+
124
+static inline unsigned long perf_data_size(struct perf_buffer *rb)
118125 {
119126 return rb->nr_pages << (PAGE_SHIFT + page_order(rb));
120127 }
121128
122
-static inline unsigned long perf_aux_size(struct ring_buffer *rb)
129
+static inline unsigned long perf_aux_size(struct perf_buffer *rb)
123130 {
124131 return rb->aux_nr_pages << PAGE_SHIFT;
125132 }
....@@ -139,7 +146,7 @@
139146 buf += written; \
140147 handle->size -= written; \
141148 if (!handle->size) { \
142
- struct ring_buffer *rb = handle->rb; \
149
+ struct perf_buffer *rb = handle->rb; \
143150 \
144151 handle->page++; \
145152 handle->page &= rb->nr_pages - 1; \
....@@ -203,16 +210,12 @@
203210
204211 static inline int get_recursion_context(int *recursion)
205212 {
206
- int rctx;
213
+ unsigned int pc = preempt_count();
214
+ unsigned char rctx = 0;
207215
208
- if (unlikely(in_nmi()))
209
- rctx = 3;
210
- else if (in_irq())
211
- rctx = 2;
212
- else if (in_serving_softirq())
213
- rctx = 1;
214
- else
215
- rctx = 0;
216
+ rctx += !!(pc & (NMI_MASK));
217
+ rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK));
218
+ rctx += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));
216219
217220 if (recursion[rctx])
218221 return -1;