forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-09-20 cf4ce59b3b70238352c7f1729f0f7223214828ad 
 kernel/arch/arm64/kernel/perf_event.c
....@@ -1,22 +1,11 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
2
- * PMU support
3
+ * ARMv8 PMUv3 Performance Events handling code.
34  *
45  * Copyright (C) 2012 ARM Limited
56  * Author: Will Deacon <will.deacon@arm.com>
67  *
78  * This code is based heavily on the ARMv7 perf event code.
8
- *
9
- * This program is free software; you can redistribute it and/or modify
10
- * it under the terms of the GNU General Public License version 2 as
11
- * published by the Free Software Foundation.
12
- *
13
- * This program is distributed in the hope that it will be useful,
14
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
15
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16
- * GNU General Public License for more details.
17
- *
18
- * You should have received a copy of the GNU General Public License
19
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
209  */
2110 
2211 #include <asm/irq_regs.h>
....@@ -24,154 +13,16 @@
2413 #include <asm/sysreg.h>
2514 #include <asm/virt.h>
2615 
16
+#include <clocksource/arm_arch_timer.h>
17
+
2718 #include <linux/acpi.h>
2819 #include <linux/clocksource.h>
20
+#include <linux/kvm_host.h>
2921 #include <linux/of.h>
3022 #include <linux/perf/arm_pmu.h>
3123 #include <linux/platform_device.h>
32
-
33
-/*
34
- * ARMv8 PMUv3 Performance Events handling code.
35
- * Common event types (some are defined in asm/perf_event.h).
36
- */
37
-
38
-/* At least one of the following is required. */
39
-#define ARMV8_PMUV3_PERFCTR_INST_RETIRED			0x08
40
-#define ARMV8_PMUV3_PERFCTR_INST_SPEC				0x1B
41
-
42
-/* Common architectural events. */
43
-#define ARMV8_PMUV3_PERFCTR_LD_RETIRED				0x06
44
-#define ARMV8_PMUV3_PERFCTR_ST_RETIRED				0x07
45
-#define ARMV8_PMUV3_PERFCTR_EXC_TAKEN				0x09
46
-#define ARMV8_PMUV3_PERFCTR_EXC_RETURN				0x0A
47
-#define ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED			0x0B
48
-#define ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED			0x0C
49
-#define ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED			0x0D
50
-#define ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED			0x0E
51
-#define ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED		0x0F
52
-#define ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED			0x1C
53
-#define ARMV8_PMUV3_PERFCTR_CHAIN				0x1E
54
-#define ARMV8_PMUV3_PERFCTR_BR_RETIRED				0x21
55
-
56
-/* Common microarchitectural events. */
57
-#define ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL			0x01
58
-#define ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL			0x02
59
-#define ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL			0x05
60
-#define ARMV8_PMUV3_PERFCTR_MEM_ACCESS				0x13
61
-#define ARMV8_PMUV3_PERFCTR_L1I_CACHE				0x14
62
-#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB			0x15
63
-#define ARMV8_PMUV3_PERFCTR_L2D_CACHE				0x16
64
-#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL			0x17
65
-#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB			0x18
66
-#define ARMV8_PMUV3_PERFCTR_BUS_ACCESS				0x19
67
-#define ARMV8_PMUV3_PERFCTR_MEMORY_ERROR			0x1A
68
-#define ARMV8_PMUV3_PERFCTR_BUS_CYCLES				0x1D
69
-#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE			0x1F
70
-#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE			0x20
71
-#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED			0x22
72
-#define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND			0x23
73
-#define ARMV8_PMUV3_PERFCTR_STALL_BACKEND			0x24
74
-#define ARMV8_PMUV3_PERFCTR_L1D_TLB				0x25
75
-#define ARMV8_PMUV3_PERFCTR_L1I_TLB				0x26
76
-#define ARMV8_PMUV3_PERFCTR_L2I_CACHE				0x27
77
-#define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL			0x28
78
-#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE			0x29
79
-#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL			0x2A
80
-#define ARMV8_PMUV3_PERFCTR_L3D_CACHE				0x2B
81
-#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB			0x2C
82
-#define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL			0x2D
83
-#define ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL			0x2E
84
-#define ARMV8_PMUV3_PERFCTR_L2D_TLB				0x2F
85
-#define ARMV8_PMUV3_PERFCTR_L2I_TLB				0x30
86
-
87
-/* ARMv8 recommended implementation defined event types */
88
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD			0x40
89
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR			0x41
90
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD		0x42
91
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR		0x43
92
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_INNER		0x44
93
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_OUTER		0x45
94
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_VICTIM		0x46
95
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_CLEAN			0x47
96
-#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_INVAL			0x48
97
-
98
-#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD			0x4C
99
-#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR			0x4D
100
-#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD				0x4E
101
-#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR				0x4F
102
-#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_RD			0x50
103
-#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WR			0x51
104
-#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_RD		0x52
105
-#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_WR		0x53
106
-
107
-#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_VICTIM		0x56
108
-#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_CLEAN			0x57
109
-#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_INVAL			0x58
110
-
111
-#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_RD			0x5C
112
-#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_WR			0x5D
113
-#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_RD				0x5E
114
-#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_WR				0x5F
115
-
116
-#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD			0x60
117
-#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR			0x61
118
-#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_SHARED			0x62
119
-#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NOT_SHARED		0x63
120
-#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NORMAL			0x64
121
-#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_PERIPH			0x65
122
-
123
-#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_RD			0x66
124
-#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_WR			0x67
125
-#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LD_SPEC			0x68
126
-#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_ST_SPEC			0x69
127
-#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LDST_SPEC		0x6A
128
-
129
-#define ARMV8_IMPDEF_PERFCTR_LDREX_SPEC				0x6C
130
-#define ARMV8_IMPDEF_PERFCTR_STREX_PASS_SPEC			0x6D
131
-#define ARMV8_IMPDEF_PERFCTR_STREX_FAIL_SPEC			0x6E
132
-#define ARMV8_IMPDEF_PERFCTR_STREX_SPEC				0x6F
133
-#define ARMV8_IMPDEF_PERFCTR_LD_SPEC				0x70
134
-#define ARMV8_IMPDEF_PERFCTR_ST_SPEC				0x71
135
-#define ARMV8_IMPDEF_PERFCTR_LDST_SPEC				0x72
136
-#define ARMV8_IMPDEF_PERFCTR_DP_SPEC				0x73
137
-#define ARMV8_IMPDEF_PERFCTR_ASE_SPEC				0x74
138
-#define ARMV8_IMPDEF_PERFCTR_VFP_SPEC				0x75
139
-#define ARMV8_IMPDEF_PERFCTR_PC_WRITE_SPEC			0x76
140
-#define ARMV8_IMPDEF_PERFCTR_CRYPTO_SPEC			0x77
141
-#define ARMV8_IMPDEF_PERFCTR_BR_IMMED_SPEC			0x78
142
-#define ARMV8_IMPDEF_PERFCTR_BR_RETURN_SPEC			0x79
143
-#define ARMV8_IMPDEF_PERFCTR_BR_INDIRECT_SPEC			0x7A
144
-
145
-#define ARMV8_IMPDEF_PERFCTR_ISB_SPEC				0x7C
146
-#define ARMV8_IMPDEF_PERFCTR_DSB_SPEC				0x7D
147
-#define ARMV8_IMPDEF_PERFCTR_DMB_SPEC				0x7E
148
-
149
-#define ARMV8_IMPDEF_PERFCTR_EXC_UNDEF				0x81
150
-#define ARMV8_IMPDEF_PERFCTR_EXC_SVC				0x82
151
-#define ARMV8_IMPDEF_PERFCTR_EXC_PABORT				0x83
152
-#define ARMV8_IMPDEF_PERFCTR_EXC_DABORT				0x84
153
-
154
-#define ARMV8_IMPDEF_PERFCTR_EXC_IRQ				0x86
155
-#define ARMV8_IMPDEF_PERFCTR_EXC_FIQ				0x87
156
-#define ARMV8_IMPDEF_PERFCTR_EXC_SMC				0x88
157
-
158
-#define ARMV8_IMPDEF_PERFCTR_EXC_HVC				0x8A
159
-#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_PABORT			0x8B
160
-#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_DABORT			0x8C
161
-#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_OTHER			0x8D
162
-#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_IRQ			0x8E
163
-#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_FIQ			0x8F
164
-#define ARMV8_IMPDEF_PERFCTR_RC_LD_SPEC				0x90
165
-#define ARMV8_IMPDEF_PERFCTR_RC_ST_SPEC				0x91
166
-
167
-#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_RD			0xA0
168
-#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WR			0xA1
169
-#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_RD		0xA2
170
-#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_WR		0xA3
171
-
172
-#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_VICTIM		0xA6
173
-#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_CLEAN			0xA7
174
-#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_INVAL			0xA8
24
+#include <linux/sched_clock.h>
25
+#include <linux/smp.h>
17526 
17627 /* ARMv8 Cortex-A53 specific event types. */
17728 #define ARMV8_A53_PERFCTR_PREF_LINEFILL				0xC2
....@@ -183,12 +34,10 @@
18334 #define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS		0xEC
18435 #define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS		0xED
18536 
186
-/* PMUv3 HW events mapping. */
187
-
18837 /*
18938  * ARMv8 Architectural defined events, not all of these may
190
- * be supported on any given implementation. Undefined events will
191
- * be disabled at run-time.
39
+ * be supported on any given implementation. Unsupported events will
40
+ * be disabled at run-time based on the PMCEID registers.
19241  */
19342 static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
19443 	PERF_MAP_ALL_UNSUPPORTED,
....@@ -210,8 +59,6 @@
21059 
21160 	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE,
21261 	[C(L1D)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
213
-	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE,
214
-	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
21562 
21663 	[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1I_CACHE,
21764 	[C(L1I)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
....@@ -222,10 +69,11 @@
22269 	[C(ITLB)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
22370 	[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1I_TLB,
22471 
72
+	[C(LL)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD,
73
+	[C(LL)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_LL_CACHE_RD,
74
+
22575 	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_BR_PRED,
22676 	[C(BPU)][C(OP_READ)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
227
-	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_BR_PRED,
228
-	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
22977 };
23078 
23179 static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
....@@ -313,113 +161,96 @@
313161 
314162 	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
315163 
316
-	return sprintf(page, "event=0x%03llx\n", pmu_attr->id);
164
+	return sprintf(page, "event=0x%04llx\n", pmu_attr->id);
317165 }
318166 
319
-#define ARMV8_EVENT_ATTR_RESOLVE(m) #m
320
-#define ARMV8_EVENT_ATTR(name, config) \
321
-	PMU_EVENT_ATTR(name, armv8_event_attr_##name, \
322
-		       config, armv8pmu_events_sysfs_show)
323
-
324
-ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR);
325
-ARMV8_EVENT_ATTR(l1i_cache_refill, ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL);
326
-ARMV8_EVENT_ATTR(l1i_tlb_refill, ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL);
327
-ARMV8_EVENT_ATTR(l1d_cache_refill, ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL);
328
-ARMV8_EVENT_ATTR(l1d_cache, ARMV8_PMUV3_PERFCTR_L1D_CACHE);
329
-ARMV8_EVENT_ATTR(l1d_tlb_refill, ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL);
330
-ARMV8_EVENT_ATTR(ld_retired, ARMV8_PMUV3_PERFCTR_LD_RETIRED);
331
-ARMV8_EVENT_ATTR(st_retired, ARMV8_PMUV3_PERFCTR_ST_RETIRED);
332
-ARMV8_EVENT_ATTR(inst_retired, ARMV8_PMUV3_PERFCTR_INST_RETIRED);
333
-ARMV8_EVENT_ATTR(exc_taken, ARMV8_PMUV3_PERFCTR_EXC_TAKEN);
334
-ARMV8_EVENT_ATTR(exc_return, ARMV8_PMUV3_PERFCTR_EXC_RETURN);
335
-ARMV8_EVENT_ATTR(cid_write_retired, ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED);
336
-ARMV8_EVENT_ATTR(pc_write_retired, ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED);
337
-ARMV8_EVENT_ATTR(br_immed_retired, ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED);
338
-ARMV8_EVENT_ATTR(br_return_retired, ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED);
339
-ARMV8_EVENT_ATTR(unaligned_ldst_retired, ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED);
340
-ARMV8_EVENT_ATTR(br_mis_pred, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED);
341
-ARMV8_EVENT_ATTR(cpu_cycles, ARMV8_PMUV3_PERFCTR_CPU_CYCLES);
342
-ARMV8_EVENT_ATTR(br_pred, ARMV8_PMUV3_PERFCTR_BR_PRED);
343
-ARMV8_EVENT_ATTR(mem_access, ARMV8_PMUV3_PERFCTR_MEM_ACCESS);
344
-ARMV8_EVENT_ATTR(l1i_cache, ARMV8_PMUV3_PERFCTR_L1I_CACHE);
345
-ARMV8_EVENT_ATTR(l1d_cache_wb, ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB);
346
-ARMV8_EVENT_ATTR(l2d_cache, ARMV8_PMUV3_PERFCTR_L2D_CACHE);
347
-ARMV8_EVENT_ATTR(l2d_cache_refill, ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL);
348
-ARMV8_EVENT_ATTR(l2d_cache_wb, ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB);
349
-ARMV8_EVENT_ATTR(bus_access, ARMV8_PMUV3_PERFCTR_BUS_ACCESS);
350
-ARMV8_EVENT_ATTR(memory_error, ARMV8_PMUV3_PERFCTR_MEMORY_ERROR);
351
-ARMV8_EVENT_ATTR(inst_spec, ARMV8_PMUV3_PERFCTR_INST_SPEC);
352
-ARMV8_EVENT_ATTR(ttbr_write_retired, ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED);
353
-ARMV8_EVENT_ATTR(bus_cycles, ARMV8_PMUV3_PERFCTR_BUS_CYCLES);
354
-/* Don't expose the chain event in /sys, since it's useless in isolation */
355
-ARMV8_EVENT_ATTR(l1d_cache_allocate, ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE);
356
-ARMV8_EVENT_ATTR(l2d_cache_allocate, ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE);
357
-ARMV8_EVENT_ATTR(br_retired, ARMV8_PMUV3_PERFCTR_BR_RETIRED);
358
-ARMV8_EVENT_ATTR(br_mis_pred_retired, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED);
359
-ARMV8_EVENT_ATTR(stall_frontend, ARMV8_PMUV3_PERFCTR_STALL_FRONTEND);
360
-ARMV8_EVENT_ATTR(stall_backend, ARMV8_PMUV3_PERFCTR_STALL_BACKEND);
361
-ARMV8_EVENT_ATTR(l1d_tlb, ARMV8_PMUV3_PERFCTR_L1D_TLB);
362
-ARMV8_EVENT_ATTR(l1i_tlb, ARMV8_PMUV3_PERFCTR_L1I_TLB);
363
-ARMV8_EVENT_ATTR(l2i_cache, ARMV8_PMUV3_PERFCTR_L2I_CACHE);
364
-ARMV8_EVENT_ATTR(l2i_cache_refill, ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL);
365
-ARMV8_EVENT_ATTR(l3d_cache_allocate, ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE);
366
-ARMV8_EVENT_ATTR(l3d_cache_refill, ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL);
367
-ARMV8_EVENT_ATTR(l3d_cache, ARMV8_PMUV3_PERFCTR_L3D_CACHE);
368
-ARMV8_EVENT_ATTR(l3d_cache_wb, ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB);
369
-ARMV8_EVENT_ATTR(l2d_tlb_refill, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL);
370
-ARMV8_EVENT_ATTR(l2i_tlb_refill, ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL);
371
-ARMV8_EVENT_ATTR(l2d_tlb, ARMV8_PMUV3_PERFCTR_L2D_TLB);
372
-ARMV8_EVENT_ATTR(l2i_tlb, ARMV8_PMUV3_PERFCTR_L2I_TLB);
167
+#define ARMV8_EVENT_ATTR(name, config)						\
168
+	(&((struct perf_pmu_events_attr) {					\
169
+		.attr = __ATTR(name, 0444, armv8pmu_events_sysfs_show, NULL),	\
170
+		.id = config,							\
171
+	}).attr.attr)
373172 
374173 static struct attribute *armv8_pmuv3_event_attrs[] = {
375
-	&armv8_event_attr_sw_incr.attr.attr,
376
-	&armv8_event_attr_l1i_cache_refill.attr.attr,
377
-	&armv8_event_attr_l1i_tlb_refill.attr.attr,
378
-	&armv8_event_attr_l1d_cache_refill.attr.attr,
379
-	&armv8_event_attr_l1d_cache.attr.attr,
380
-	&armv8_event_attr_l1d_tlb_refill.attr.attr,
381
-	&armv8_event_attr_ld_retired.attr.attr,
382
-	&armv8_event_attr_st_retired.attr.attr,
383
-	&armv8_event_attr_inst_retired.attr.attr,
384
-	&armv8_event_attr_exc_taken.attr.attr,
385
-	&armv8_event_attr_exc_return.attr.attr,
386
-	&armv8_event_attr_cid_write_retired.attr.attr,
387
-	&armv8_event_attr_pc_write_retired.attr.attr,
388
-	&armv8_event_attr_br_immed_retired.attr.attr,
389
-	&armv8_event_attr_br_return_retired.attr.attr,
390
-	&armv8_event_attr_unaligned_ldst_retired.attr.attr,
391
-	&armv8_event_attr_br_mis_pred.attr.attr,
392
-	&armv8_event_attr_cpu_cycles.attr.attr,
393
-	&armv8_event_attr_br_pred.attr.attr,
394
-	&armv8_event_attr_mem_access.attr.attr,
395
-	&armv8_event_attr_l1i_cache.attr.attr,
396
-	&armv8_event_attr_l1d_cache_wb.attr.attr,
397
-	&armv8_event_attr_l2d_cache.attr.attr,
398
-	&armv8_event_attr_l2d_cache_refill.attr.attr,
399
-	&armv8_event_attr_l2d_cache_wb.attr.attr,
400
-	&armv8_event_attr_bus_access.attr.attr,
401
-	&armv8_event_attr_memory_error.attr.attr,
402
-	&armv8_event_attr_inst_spec.attr.attr,
403
-	&armv8_event_attr_ttbr_write_retired.attr.attr,
404
-	&armv8_event_attr_bus_cycles.attr.attr,
405
-	&armv8_event_attr_l1d_cache_allocate.attr.attr,
406
-	&armv8_event_attr_l2d_cache_allocate.attr.attr,
407
-	&armv8_event_attr_br_retired.attr.attr,
408
-	&armv8_event_attr_br_mis_pred_retired.attr.attr,
409
-	&armv8_event_attr_stall_frontend.attr.attr,
410
-	&armv8_event_attr_stall_backend.attr.attr,
411
-	&armv8_event_attr_l1d_tlb.attr.attr,
412
-	&armv8_event_attr_l1i_tlb.attr.attr,
413
-	&armv8_event_attr_l2i_cache.attr.attr,
414
-	&armv8_event_attr_l2i_cache_refill.attr.attr,
415
-	&armv8_event_attr_l3d_cache_allocate.attr.attr,
416
-	&armv8_event_attr_l3d_cache_refill.attr.attr,
417
-	&armv8_event_attr_l3d_cache.attr.attr,
418
-	&armv8_event_attr_l3d_cache_wb.attr.attr,
419
-	&armv8_event_attr_l2d_tlb_refill.attr.attr,
420
-	&armv8_event_attr_l2i_tlb_refill.attr.attr,
421
-	&armv8_event_attr_l2d_tlb.attr.attr,
422
-	&armv8_event_attr_l2i_tlb.attr.attr,
174
+	ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR),
175
+	ARMV8_EVENT_ATTR(l1i_cache_refill, ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL),
176
+	ARMV8_EVENT_ATTR(l1i_tlb_refill, ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL),
177
+	ARMV8_EVENT_ATTR(l1d_cache_refill, ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL),
178
+	ARMV8_EVENT_ATTR(l1d_cache, ARMV8_PMUV3_PERFCTR_L1D_CACHE),
179
+	ARMV8_EVENT_ATTR(l1d_tlb_refill, ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL),
180
+	ARMV8_EVENT_ATTR(ld_retired, ARMV8_PMUV3_PERFCTR_LD_RETIRED),
181
+	ARMV8_EVENT_ATTR(st_retired, ARMV8_PMUV3_PERFCTR_ST_RETIRED),
182
+	ARMV8_EVENT_ATTR(inst_retired, ARMV8_PMUV3_PERFCTR_INST_RETIRED),
183
+	ARMV8_EVENT_ATTR(exc_taken, ARMV8_PMUV3_PERFCTR_EXC_TAKEN),
184
+	ARMV8_EVENT_ATTR(exc_return, ARMV8_PMUV3_PERFCTR_EXC_RETURN),
185
+	ARMV8_EVENT_ATTR(cid_write_retired, ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED),
186
+	ARMV8_EVENT_ATTR(pc_write_retired, ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED),
187
+	ARMV8_EVENT_ATTR(br_immed_retired, ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED),
188
+	ARMV8_EVENT_ATTR(br_return_retired, ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED),
189
+	ARMV8_EVENT_ATTR(unaligned_ldst_retired, ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED),
190
+	ARMV8_EVENT_ATTR(br_mis_pred, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED),
191
+	ARMV8_EVENT_ATTR(cpu_cycles, ARMV8_PMUV3_PERFCTR_CPU_CYCLES),
192
+	ARMV8_EVENT_ATTR(br_pred, ARMV8_PMUV3_PERFCTR_BR_PRED),
193
+	ARMV8_EVENT_ATTR(mem_access, ARMV8_PMUV3_PERFCTR_MEM_ACCESS),
194
+	ARMV8_EVENT_ATTR(l1i_cache, ARMV8_PMUV3_PERFCTR_L1I_CACHE),
195
+	ARMV8_EVENT_ATTR(l1d_cache_wb, ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB),
196
+	ARMV8_EVENT_ATTR(l2d_cache, ARMV8_PMUV3_PERFCTR_L2D_CACHE),
197
+	ARMV8_EVENT_ATTR(l2d_cache_refill, ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL),
198
+	ARMV8_EVENT_ATTR(l2d_cache_wb, ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB),
199
+	ARMV8_EVENT_ATTR(bus_access, ARMV8_PMUV3_PERFCTR_BUS_ACCESS),
200
+	ARMV8_EVENT_ATTR(memory_error, ARMV8_PMUV3_PERFCTR_MEMORY_ERROR),
201
+	ARMV8_EVENT_ATTR(inst_spec, ARMV8_PMUV3_PERFCTR_INST_SPEC),
202
+	ARMV8_EVENT_ATTR(ttbr_write_retired, ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED),
203
+	ARMV8_EVENT_ATTR(bus_cycles, ARMV8_PMUV3_PERFCTR_BUS_CYCLES),
204
+	/* Don't expose the chain event in /sys, since it's useless in isolation */
205
+	ARMV8_EVENT_ATTR(l1d_cache_allocate, ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE),
206
+	ARMV8_EVENT_ATTR(l2d_cache_allocate, ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE),
207
+	ARMV8_EVENT_ATTR(br_retired, ARMV8_PMUV3_PERFCTR_BR_RETIRED),
208
+	ARMV8_EVENT_ATTR(br_mis_pred_retired, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED),
209
+	ARMV8_EVENT_ATTR(stall_frontend, ARMV8_PMUV3_PERFCTR_STALL_FRONTEND),
210
+	ARMV8_EVENT_ATTR(stall_backend, ARMV8_PMUV3_PERFCTR_STALL_BACKEND),
211
+	ARMV8_EVENT_ATTR(l1d_tlb, ARMV8_PMUV3_PERFCTR_L1D_TLB),
212
+	ARMV8_EVENT_ATTR(l1i_tlb, ARMV8_PMUV3_PERFCTR_L1I_TLB),
213
+	ARMV8_EVENT_ATTR(l2i_cache, ARMV8_PMUV3_PERFCTR_L2I_CACHE),
214
+	ARMV8_EVENT_ATTR(l2i_cache_refill, ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL),
215
+	ARMV8_EVENT_ATTR(l3d_cache_allocate, ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE),
216
+	ARMV8_EVENT_ATTR(l3d_cache_refill, ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL),
217
+	ARMV8_EVENT_ATTR(l3d_cache, ARMV8_PMUV3_PERFCTR_L3D_CACHE),
218
+	ARMV8_EVENT_ATTR(l3d_cache_wb, ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB),
219
+	ARMV8_EVENT_ATTR(l2d_tlb_refill, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL),
220
+	ARMV8_EVENT_ATTR(l2i_tlb_refill, ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL),
221
+	ARMV8_EVENT_ATTR(l2d_tlb, ARMV8_PMUV3_PERFCTR_L2D_TLB),
222
+	ARMV8_EVENT_ATTR(l2i_tlb, ARMV8_PMUV3_PERFCTR_L2I_TLB),
223
+	ARMV8_EVENT_ATTR(remote_access, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS),
224
+	ARMV8_EVENT_ATTR(ll_cache, ARMV8_PMUV3_PERFCTR_LL_CACHE),
225
+	ARMV8_EVENT_ATTR(ll_cache_miss, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS),
226
+	ARMV8_EVENT_ATTR(dtlb_walk, ARMV8_PMUV3_PERFCTR_DTLB_WALK),
227
+	ARMV8_EVENT_ATTR(itlb_walk, ARMV8_PMUV3_PERFCTR_ITLB_WALK),
228
+	ARMV8_EVENT_ATTR(ll_cache_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_RD),
229
+	ARMV8_EVENT_ATTR(ll_cache_miss_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD),
230
+	ARMV8_EVENT_ATTR(remote_access_rd, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD),
231
+	ARMV8_EVENT_ATTR(l1d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L1D_CACHE_LMISS_RD),
232
+	ARMV8_EVENT_ATTR(op_retired, ARMV8_PMUV3_PERFCTR_OP_RETIRED),
233
+	ARMV8_EVENT_ATTR(op_spec, ARMV8_PMUV3_PERFCTR_OP_SPEC),
234
+	ARMV8_EVENT_ATTR(stall, ARMV8_PMUV3_PERFCTR_STALL),
235
+	ARMV8_EVENT_ATTR(stall_slot_backend, ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND),
236
+	ARMV8_EVENT_ATTR(stall_slot_frontend, ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND),
237
+	ARMV8_EVENT_ATTR(stall_slot, ARMV8_PMUV3_PERFCTR_STALL_SLOT),
238
+	ARMV8_EVENT_ATTR(sample_pop, ARMV8_SPE_PERFCTR_SAMPLE_POP),
239
+	ARMV8_EVENT_ATTR(sample_feed, ARMV8_SPE_PERFCTR_SAMPLE_FEED),
240
+	ARMV8_EVENT_ATTR(sample_filtrate, ARMV8_SPE_PERFCTR_SAMPLE_FILTRATE),
241
+	ARMV8_EVENT_ATTR(sample_collision, ARMV8_SPE_PERFCTR_SAMPLE_COLLISION),
242
+	ARMV8_EVENT_ATTR(cnt_cycles, ARMV8_AMU_PERFCTR_CNT_CYCLES),
243
+	ARMV8_EVENT_ATTR(stall_backend_mem, ARMV8_AMU_PERFCTR_STALL_BACKEND_MEM),
244
+	ARMV8_EVENT_ATTR(l1i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L1I_CACHE_LMISS),
245
+	ARMV8_EVENT_ATTR(l2d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L2D_CACHE_LMISS_RD),
246
+	ARMV8_EVENT_ATTR(l2i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L2I_CACHE_LMISS),
247
+	ARMV8_EVENT_ATTR(l3d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L3D_CACHE_LMISS_RD),
248
+	ARMV8_EVENT_ATTR(ldst_align_lat, ARMV8_PMUV3_PERFCTR_LDST_ALIGN_LAT),
249
+	ARMV8_EVENT_ATTR(ld_align_lat, ARMV8_PMUV3_PERFCTR_LD_ALIGN_LAT),
250
+	ARMV8_EVENT_ATTR(st_align_lat, ARMV8_PMUV3_PERFCTR_ST_ALIGN_LAT),
251
+	ARMV8_EVENT_ATTR(mem_access_checked, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED),
252
+	ARMV8_EVENT_ATTR(mem_access_checked_rd, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_RD),
253
+	ARMV8_EVENT_ATTR(mem_access_checked_wr, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_WR),
423254 	NULL,
424255 };
425256 
....@@ -434,8 +265,17 @@
434265 
435266 	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
436267 
437
-	if (test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))
268
+	if (pmu_attr->id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&
269
+	    test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))
438270 		return attr->mode;
271
+
272
+	if (pmu_attr->id >= ARMV8_PMUV3_EXT_COMMON_EVENT_BASE) {
273
+		u64 id = pmu_attr->id - ARMV8_PMUV3_EXT_COMMON_EVENT_BASE;
274
+
275
+		if (id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&
276
+		    test_bit(id, cpu_pmu->pmceid_ext_bitmap))
277
+			return attr->mode;
278
+	}
439279 
440280 	return 0;
441281 }
....@@ -465,13 +305,44 @@
465305 	.attrs = armv8_pmuv3_format_attrs,
466306 };
467307 
308
+static ssize_t slots_show(struct device *dev, struct device_attribute *attr,
309
+			  char *page)
310
+{
311
+	struct pmu *pmu = dev_get_drvdata(dev);
312
+	struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
313
+	u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK;
314
+
315
+	return sysfs_emit(page, "0x%08x\n", slots);
316
+}
317
+
318
+static DEVICE_ATTR_RO(slots);
319
+
320
+static struct attribute *armv8_pmuv3_caps_attrs[] = {
321
+	&dev_attr_slots.attr,
322
+	NULL,
323
+};
324
+
325
+static struct attribute_group armv8_pmuv3_caps_attr_group = {
326
+	.name = "caps",
327
+	.attrs = armv8_pmuv3_caps_attrs,
328
+};
329
+
468330 /*
469331  * Perf Events' indices
470332  */
471333 #define	ARMV8_IDX_CYCLE_COUNTER	0
472334 #define	ARMV8_IDX_COUNTER0	1
473
-#define	ARMV8_IDX_COUNTER_LAST(cpu_pmu) \
474
-	(ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
335
+
336
+
337
+/*
338
+ * We unconditionally enable ARMv8.5-PMU long event counter support
339
+ * (64-bit events) where supported. Indicate if this arm_pmu has long
340
+ * event counter support.
341
+ */
342
+static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu)
343
+{
344
+	return (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5);
345
+}
475346 
476347 /*
477348  * We must chain two programmable counters for 64 bit events,
....@@ -482,9 +353,11 @@
482353 static inline bool armv8pmu_event_is_chained(struct perf_event *event)
483354 {
484355 	int idx = event->hw.idx;
356
+	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
485357 
486358 	return !WARN_ON(idx < 0) &&
487359 	       armv8pmu_event_is_64bit(event) &&
360
+	       !armv8pmu_has_long_event(cpu_pmu) &&
488361 	       (idx != ARMV8_IDX_CYCLE_COUNTER);
489362 }
490363 
....@@ -497,6 +370,73 @@
497370  */
498371 #define	ARMV8_IDX_TO_COUNTER(x)	\
499372 	(((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK)
373
+
374
+/*
375
+ * This code is really good
376
+ */
377
+
378
+#define PMEVN_CASE(n, case_macro) \
379
+	case n: case_macro(n); break
380
+
381
+#define PMEVN_SWITCH(x, case_macro)				\
382
+	do {							\
383
+		switch (x) {					\
384
+		PMEVN_CASE(0,  case_macro);			\
385
+		PMEVN_CASE(1,  case_macro);			\
386
+		PMEVN_CASE(2,  case_macro);			\
387
+		PMEVN_CASE(3,  case_macro);			\
388
+		PMEVN_CASE(4,  case_macro);			\
389
+		PMEVN_CASE(5,  case_macro);			\
390
+		PMEVN_CASE(6,  case_macro);			\
391
+		PMEVN_CASE(7,  case_macro);			\
392
+		PMEVN_CASE(8,  case_macro);			\
393
+		PMEVN_CASE(9,  case_macro);			\
394
+		PMEVN_CASE(10, case_macro);			\
395
+		PMEVN_CASE(11, case_macro);			\
396
+		PMEVN_CASE(12, case_macro);			\
397
+		PMEVN_CASE(13, case_macro);			\
398
+		PMEVN_CASE(14, case_macro);			\
399
+		PMEVN_CASE(15, case_macro);			\
400
+		PMEVN_CASE(16, case_macro);			\
401
+		PMEVN_CASE(17, case_macro);			\
402
+		PMEVN_CASE(18, case_macro);			\
403
+		PMEVN_CASE(19, case_macro);			\
404
+		PMEVN_CASE(20, case_macro);			\
405
+		PMEVN_CASE(21, case_macro);			\
406
+		PMEVN_CASE(22, case_macro);			\
407
+		PMEVN_CASE(23, case_macro);			\
408
+		PMEVN_CASE(24, case_macro);			\
409
+		PMEVN_CASE(25, case_macro);			\
410
+		PMEVN_CASE(26, case_macro);			\
411
+		PMEVN_CASE(27, case_macro);			\
412
+		PMEVN_CASE(28, case_macro);			\
413
+		PMEVN_CASE(29, case_macro);			\
414
+		PMEVN_CASE(30, case_macro);			\
415
+		default: WARN(1, "Invalid PMEV* index\n");	\
416
+		}						\
417
+	} while (0)
418
+
419
+#define RETURN_READ_PMEVCNTRN(n) \
420
+	return read_sysreg(pmevcntr##n##_el0)
421
+static unsigned long read_pmevcntrn(int n)
422
+{
423
+	PMEVN_SWITCH(n, RETURN_READ_PMEVCNTRN);
424
+	return 0;
425
+}
426
+
427
+#define WRITE_PMEVCNTRN(n) \
428
+	write_sysreg(val, pmevcntr##n##_el0)
429
+static void write_pmevcntrn(int n, unsigned long val)
430
+{
431
+	PMEVN_SWITCH(n, WRITE_PMEVCNTRN);
432
+}
433
+
434
+#define WRITE_PMEVTYPERN(n) \
435
+	write_sysreg(val, pmevtyper##n##_el0)
436
+static void write_pmevtypern(int n, unsigned long val)
437
+{
438
+	PMEVN_SWITCH(n, WRITE_PMEVTYPERN);
439
+}
500440 
501441 static inline u32 armv8pmu_pmcr_read(void)
502442 {
....@@ -515,28 +455,16 @@
515455 	return pmovsr & ARMV8_PMU_OVERFLOWED_MASK;
516456 }
517457 
518
-static inline int armv8pmu_counter_valid(struct arm_pmu *cpu_pmu, int idx)
519
-{
520
-	return idx >= ARMV8_IDX_CYCLE_COUNTER &&
521
-		idx <= ARMV8_IDX_COUNTER_LAST(cpu_pmu);
522
-}
523
-
524458 static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
525459 {
526460 	return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx));
527461 }
528462 
529
-static inline void armv8pmu_select_counter(int idx)
463
+static inline u64 armv8pmu_read_evcntr(int idx)
530464 {
531465 	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
532
-	write_sysreg(counter, pmselr_el0);
533
-	isb();
534
-}
535466 
536
-static inline u32 armv8pmu_read_evcntr(int idx)
537
-{
538
-	armv8pmu_select_counter(idx);
539
-	return read_sysreg(pmxevcntr_el0);
467
+	return read_pmevcntrn(counter);
540468 }
541469 
542470 static inline u64 armv8pmu_read_hw_counter(struct perf_event *event)
....@@ -550,28 +478,63 @@
550478 	return val;
551479 }
552480 
553
-static inline u64 armv8pmu_read_counter(struct perf_event *event)
481
+/*
482
+ * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP
483
+ * is set the event counters also become 64-bit counters. Unless the
484
+ * user has requested a long counter (attr.config1) then we want to
485
+ * interrupt upon 32-bit overflow - we achieve this by applying a bias.
486
+ */
487
+static bool armv8pmu_event_needs_bias(struct perf_event *event)
554488 {
555489 	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
556490 	struct hw_perf_event *hwc = &event->hw;
557491 	int idx = hwc->idx;
558
-	u64 value = 0;
559492 
560
-	if (!armv8pmu_counter_valid(cpu_pmu, idx))
561
-		pr_err("CPU%u reading wrong counter %d\n",
562
-			smp_processor_id(), idx);
563
-	else if (idx == ARMV8_IDX_CYCLE_COUNTER)
564
-		value = read_sysreg(pmccntr_el0);
565
-	else
566
-		value = armv8pmu_read_hw_counter(event);
493
+	if (armv8pmu_event_is_64bit(event))
494
+		return false;
495
+
496
+	if (armv8pmu_has_long_event(cpu_pmu) ||
497
+	    idx == ARMV8_IDX_CYCLE_COUNTER)
498
+		return true;
499
+
500
+	return false;
501
+}
502
+
503
+static u64 armv8pmu_bias_long_counter(struct perf_event *event, u64 value)
504
+{
505
+	if (armv8pmu_event_needs_bias(event))
506
+		value |= GENMASK(63, 32);
567507 
568508 	return value;
569509 }
570510 
571
-static inline void armv8pmu_write_evcntr(int idx, u32 value)
511
+static u64 armv8pmu_unbias_long_counter(struct perf_event *event, u64 value)
572512 {
573
-	armv8pmu_select_counter(idx);
574
-	write_sysreg(value, pmxevcntr_el0);
513
+	if (armv8pmu_event_needs_bias(event))
514
+		value &= ~GENMASK(63, 32);
515
+
516
+	return value;
517
+}
518
+
519
+static u64 armv8pmu_read_counter(struct perf_event *event)
520
+{
521
+	struct hw_perf_event *hwc = &event->hw;
522
+	int idx = hwc->idx;
523
+	u64 value = 0;
524
+
525
+	if (idx == ARMV8_IDX_CYCLE_COUNTER)
526
+		value = read_sysreg(pmccntr_el0);
527
+	else
528
+		value = armv8pmu_read_hw_counter(event);
529
+
530
+	return  armv8pmu_unbias_long_counter(event, value);
531
+}
532
+
533
+static inline void armv8pmu_write_evcntr(int idx, u64 value)
534
+{
535
+	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
536
+
537
+	write_pmevcntrn(counter, value);
575538 }
576539 
577540 static inline void armv8pmu_write_hw_counter(struct perf_event *event,
....@@ -587,34 +550,25 @@
587550 	}
588551 }
589552 
590
-static inline void armv8pmu_write_counter(struct perf_event *event, u64 value)
553
+static void armv8pmu_write_counter(struct perf_event *event, u64 value)
591554 {
592
-	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
593555 	struct hw_perf_event *hwc = &event->hw;
594556 	int idx = hwc->idx;
595557 
596
-	if (!armv8pmu_counter_valid(cpu_pmu, idx))
597
-		pr_err("CPU%u writing wrong counter %d\n",
598
-			smp_processor_id(), idx);
599
-	else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
600
-		/*
601
-		 * The cycles counter is really a 64-bit counter.
602
-		 * When treating it as a 32-bit counter, we only count
603
-		 * the lower 32 bits, and set the upper 32-bits so that
604
-		 * we get an interrupt upon 32-bit overflow.
605
-		 */
606
-		if (!armv8pmu_event_is_64bit(event))
607
-			value |= 0xffffffff00000000ULL;
558
+	value = armv8pmu_bias_long_counter(event, value);
559
+
560
+	if (idx == ARMV8_IDX_CYCLE_COUNTER)
608561 		write_sysreg(value, pmccntr_el0);
609
-	} else
562
+	else
610563 		armv8pmu_write_hw_counter(event, value);
611564 }
612565 
613566 static inline void armv8pmu_write_evtype(int idx, u32 val)
614567 {
615
-	armv8pmu_select_counter(idx);
568
+	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
569
+
616570 	val &= ARMV8_PMU_EVTYPE_MASK;
617
-	write_sysreg(val, pmxevtyper_el0);
571
+	write_pmevtypern(counter, val);
618572 }
619573 
620574 static inline void armv8pmu_write_event_type(struct perf_event *event)
....@@ -634,71 +588,91 @@
634588 		armv8pmu_write_evtype(idx - 1, hwc->config_base);
635589 		armv8pmu_write_evtype(idx, chain_evt);
636590 	} else {
637
-		armv8pmu_write_evtype(idx, hwc->config_base);
591
+		if (idx == ARMV8_IDX_CYCLE_COUNTER)
592
+			write_sysreg(hwc->config_base, pmccfiltr_el0);
593
+		else
594
+			armv8pmu_write_evtype(idx, hwc->config_base);
638595 	}
639596 }
640597 
641
-static inline int armv8pmu_enable_counter(int idx)
598
+static u32 armv8pmu_event_cnten_mask(struct perf_event *event)
642599 {
643
-	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
644
-	write_sysreg(BIT(counter), pmcntenset_el0);
645
-	return idx;
600
+	int counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
601
+	u32 mask = BIT(counter);
602
+
603
+	if (armv8pmu_event_is_chained(event))
604
+		mask |= BIT(counter - 1);
605
+	return mask;
606
+}
607
+
608
+static inline void armv8pmu_enable_counter(u32 mask)
609
+{
610
+	/*
611
+	 * Make sure event configuration register writes are visible before we
612
+	 * enable the counter.
613
+	 * */
614
+	isb();
615
+	write_sysreg(mask, pmcntenset_el0);
646616 }
647617 
648618 static inline void armv8pmu_enable_event_counter(struct perf_event *event)
649619 {
650
-	int idx = event->hw.idx;
620
+	struct perf_event_attr *attr = &event->attr;
621
+	u32 mask = armv8pmu_event_cnten_mask(event);
651622 
652
-	armv8pmu_enable_counter(idx);
653
-	if (armv8pmu_event_is_chained(event))
654
-		armv8pmu_enable_counter(idx - 1);
655
-	isb();
623
+	kvm_set_pmu_events(mask, attr);
624
+
625
+	/* We rely on the hypervisor switch code to enable guest counters */
626
+	if (!kvm_pmu_counter_deferred(attr))
627
+		armv8pmu_enable_counter(mask);
656628 }
657629 
658
-static inline int armv8pmu_disable_counter(int idx)
630
+static inline void armv8pmu_disable_counter(u32 mask)
659631 {
660
-	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
661
-	write_sysreg(BIT(counter), pmcntenclr_el0);
662
-	return idx;
632
+	write_sysreg(mask, pmcntenclr_el0);
633
+	/*
634
+	 * Make sure the effects of disabling the counter are visible before we
635
+	 * start configuring the event.
636
+	 */
637
+	isb();
663638 }
664639 
665640 static inline void armv8pmu_disable_event_counter(struct perf_event *event)
666641 {
667
-	struct hw_perf_event *hwc = &event->hw;
668
-	int idx = hwc->idx;
642
+	struct perf_event_attr *attr = &event->attr;
643
+	u32 mask = armv8pmu_event_cnten_mask(event);
669644 
670
-	if (armv8pmu_event_is_chained(event))
671
-		armv8pmu_disable_counter(idx - 1);
672
-	armv8pmu_disable_counter(idx);
645
+	kvm_clr_pmu_events(mask);
646
+
647
+	/* We rely on the hypervisor switch code to disable guest counters */
648
+	if (!kvm_pmu_counter_deferred(attr))
649
+		armv8pmu_disable_counter(mask);
673650 }
674651 
675
-static inline int armv8pmu_enable_intens(int idx)
652
+static inline void armv8pmu_enable_intens(u32 mask)
676653 {
677
-	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
678
-	write_sysreg(BIT(counter), pmintenset_el1);
679
-	return idx;
654
+	write_sysreg(mask, pmintenset_el1);
680655 }
681656 
682
-static inline int armv8pmu_enable_event_irq(struct perf_event *event)
657
+static inline void armv8pmu_enable_event_irq(struct perf_event *event)
683658 {
684
-	return armv8pmu_enable_intens(event->hw.idx);
659
+	u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
660
+	armv8pmu_enable_intens(BIT(counter));
685661 }
686662 
687
-static inline int armv8pmu_disable_intens(int idx)
663
+static inline void armv8pmu_disable_intens(u32 mask)
688664 {
689
-	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
690
-	write_sysreg(BIT(counter), pmintenclr_el1);
665
+	write_sysreg(mask, pmintenclr_el1);
691666 	isb();
692667 	/* Clear the overflow flag in case an interrupt is pending. */
693
-	write_sysreg(BIT(counter), pmovsclr_el0);
668
+	write_sysreg(mask, pmovsclr_el0);
694669 	isb();
695
-
696
-	return idx;
697670 }
698671 
699
-static inline int armv8pmu_disable_event_irq(struct perf_event *event)
672
+static inline void armv8pmu_disable_event_irq(struct perf_event *event)
700673 {
701
-	return armv8pmu_disable_intens(event->hw.idx);
674
+	u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
675
+	armv8pmu_disable_intens(BIT(counter));
702676 }
703677 
704678 static inline u32 armv8pmu_getreset_flags(void)
....@@ -717,15 +691,10 @@
717691 
718692 static void armv8pmu_enable_event(struct perf_event *event)
719693 {
720
-	unsigned long flags;
721
-	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
722
-	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
723
-
724694 	/*
725695 	 * Enable counter and interrupt, and set the counter to count
726696 	 * the event that we're interested in.
727697 	 */
728
-	raw_spin_lock_irqsave(&events->pmu_lock, flags);
729698 
730699 	/*
731700 	 * Disable counter
....@@ -733,7 +702,7 @@
733702 	armv8pmu_disable_event_counter(event);
734703 
735704 	/*
736
-	 * Set event (if destined for PMNx counters).
705
+	 * Set event.
737706 	 */
738707 	armv8pmu_write_event_type(event);
739708 
....@@ -746,21 +715,10 @@
746715 	 * Enable counter
747716 	 */
748717 	armv8pmu_enable_event_counter(event);
749
-
750
-	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
751718 }
752719 
753720 static void armv8pmu_disable_event(struct perf_event *event)
754721 {
755
-	unsigned long flags;
756
-	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
757
-	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
758
-
759
-	/*
760
-	 * Disable counter and interrupt
761
-	 */
762
-	raw_spin_lock_irqsave(&events->pmu_lock, flags);
763
-
764722 	/*
765723 	 * Disable counter
766724 	 */
....@@ -770,30 +728,18 @@
770728 	 * Disable interrupt for this counter
771729 	 */
772730 	armv8pmu_disable_event_irq(event);
773
-
774
-	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
775731 }
776732 
777733 static void armv8pmu_start(struct arm_pmu *cpu_pmu)
778734 {
779
-	unsigned long flags;
780
-	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
781
-
782
-	raw_spin_lock_irqsave(&events->pmu_lock, flags);
783735 	/* Enable all counters */
784736 	armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
785
-	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
786737 }
787738 
788739 static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
789740 {
790
-	unsigned long flags;
791
-	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
792
-
793
-	raw_spin_lock_irqsave(&events->pmu_lock, flags);
794741 	/* Disable all counters */
795742 	armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
796
-	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
797743 }
798744 
799745 static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
....@@ -846,19 +792,15 @@
846792 		if (!armpmu_event_set_period(event))
847793 			continue;
848794 
795
+		/*
796
+		 * Perf event overflow will queue the processing of the event as
797
+		 * an irq_work which will be taken care of in the handling of
798
+		 * IPI_IRQ_WORK.
799
+		 */
849800 		if (perf_event_overflow(event, &data, regs))
850801 			cpu_pmu->disable(event);
851802 	}
852803 	armv8pmu_start(cpu_pmu);
853
-
854
-	/*
855
-	 * Handle the pending perf events.
856
-	 *
857
-	 * Note: this call *must* be run with interrupts disabled. For
858
-	 * platforms that can have the PMU interrupts raised as an NMI, this
859
-	 * will not work.
860
-	 */
861
-	irq_work_run();
862804 
863805 	return IRQ_HANDLED;
864806 }
....@@ -912,7 +854,8 @@
912854 	/*
913855 	 * Otherwise use events counters
914856 	 */
915
-	if (armv8pmu_event_is_64bit(event))
857
+	if (armv8pmu_event_is_64bit(event) &&
858
+	    !armv8pmu_has_long_event(cpu_pmu))
916859 		return	armv8pmu_get_chain_idx(cpuc, cpu_pmu);
917860 	else
918861 		return armv8pmu_get_single_idx(cpuc, cpu_pmu);
....@@ -929,7 +872,7 @@
929872 }
930873 
931874 /*
932
- * Add an event filter to a given event. This will only work for PMUv2 PMUs.
875
+ * Add an event filter to a given event.
933876  */
934877 static int armv8pmu_set_event_filter(struct hw_perf_event *event,
935878 				     struct perf_event_attr *attr)
....@@ -946,14 +889,23 @@
946889 	 * with other architectures (x86 and Power).
947890 	 */
948891 	if (is_kernel_in_hyp_mode()) {
949
-		if (!attr->exclude_kernel)
892
+		if (!attr->exclude_kernel && !attr->exclude_host)
950893 			config_base |= ARMV8_PMU_INCLUDE_EL2;
951
-	} else {
952
-		if (attr->exclude_kernel)
894
+		if (attr->exclude_guest)
953895 			config_base |= ARMV8_PMU_EXCLUDE_EL1;
954
-		if (!attr->exclude_hv)
896
+		if (attr->exclude_host)
897
+			config_base |= ARMV8_PMU_EXCLUDE_EL0;
898
+	} else {
899
+		if (!attr->exclude_hv && !attr->exclude_host)
955900 			config_base |= ARMV8_PMU_INCLUDE_EL2;
956901 	}
902
+
903
+	/*
904
+	 * Filter out !VHE kernels and guest kernels
905
+	 */
906
+	if (attr->exclude_kernel)
907
+		config_base |= ARMV8_PMU_EXCLUDE_EL1;
908
+
957909 	if (attr->exclude_user)
958910 		config_base |= ARMV8_PMU_EXCLUDE_EL0;
959911 
....@@ -975,20 +927,26 @@
975927 static void armv8pmu_reset(void *info)
976928 {
977929 	struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
978
-	u32 idx, nb_cnt = cpu_pmu->num_events;
930
+	u32 pmcr;
979931 
980932 	/* The counter and interrupt enable registers are unknown at reset. */
981
-	for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
982
-		armv8pmu_disable_counter(idx);
983
-		armv8pmu_disable_intens(idx);
984
-	}
933
+	armv8pmu_disable_counter(U32_MAX);
934
+	armv8pmu_disable_intens(U32_MAX);
935
+
936
+	/* Clear the counters we flip at guest entry/exit */
937
+	kvm_clr_pmu_events(U32_MAX);
985938 
986939 	/*
987940 	 * Initialize & Reset PMNC. Request overflow interrupt for
988941 	 * 64 bit cycle counter but cheat in armv8pmu_write_counter().
989942 	 */
990
-	armv8pmu_pmcr_write(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C |
991
-			    ARMV8_PMU_PMCR_LC);
943
+	pmcr = ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C | ARMV8_PMU_PMCR_LC;
944
+
945
+	/* Enable long event counter support where available */
946
+	if (armv8pmu_has_long_event(cpu_pmu))
947
+		pmcr |= ARMV8_PMU_PMCR_LP;
948
+
949
+	armv8pmu_pmcr_write(pmcr);
992950 }
993951 
994952 static int __armv8_pmuv3_map_event(struct perf_event *event,
....@@ -1009,7 +967,7 @@
1009967 	if (armv8pmu_event_is_64bit(event))
1010968 		event->hw.flags |= ARMPMU_EVT_64BIT;
1011969 
1012
-	/* Onl expose micro/arch events supported by this PMU */
970
+	/* Only expose micro/arch events supported by this PMU */
1013971 	if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
1014972 	    && test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
1015973 		return hw_event_id;
....@@ -1061,6 +1019,7 @@
10611019 	struct armv8pmu_probe_info *probe = info;
10621020 	struct arm_pmu *cpu_pmu = probe->pmu;
10631021 	u64 dfr0;
1022
+	u64 pmceid_raw[2];
10641023 	u32 pmceid[2];
10651024 	int pmuver;
10661025 
....@@ -1070,6 +1029,7 @@
10701029 	if (pmuver == 0xf || pmuver == 0)
10711030 		return;
10721031 
1032
+	cpu_pmu->pmuver = pmuver;
10731033 	probe->present = true;
10741034 
10751035 	/* Read the nb of CNTx counters supported from PMNC */
....@@ -1079,11 +1039,23 @@
10791039 	/* Add the CPU cycles counter */
10801040 	cpu_pmu->num_events += 1;
10811041 
1082
-	pmceid[0] = read_sysreg(pmceid0_el0);
1083
-	pmceid[1] = read_sysreg(pmceid1_el0);
1042
+	pmceid[0] = pmceid_raw[0] = read_sysreg(pmceid0_el0);
1043
+	pmceid[1] = pmceid_raw[1] = read_sysreg(pmceid1_el0);
10841044 
10851045 	bitmap_from_arr32(cpu_pmu->pmceid_bitmap,
10861046 			     pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
1047
+
1048
+	pmceid[0] = pmceid_raw[0] >> 32;
1049
+	pmceid[1] = pmceid_raw[1] >> 32;
1050
+
1051
+	bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap,
1052
+			     pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
1053
+
1054
+	/* store PMMIR_EL1 register for sysfs */
1055
+	if (pmuver >= ID_AA64DFR0_PMUVER_8_4 && (pmceid_raw[1] & BIT(31)))
1056
+		cpu_pmu->reg_pmmir = read_cpuid(PMMIR_EL1);
1057
+	else
1058
+		cpu_pmu->reg_pmmir = 0;
10871059 }
10881060 
10891061 static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
....@@ -1103,165 +1075,142 @@
11031075 	return probe.present ? 0 : -ENODEV;
11041076 }
11051077 
1106
-static int armv8_pmu_init(struct arm_pmu *cpu_pmu)
1078
+static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name,
1079
+			  int (*map_event)(struct perf_event *event),
1080
+			  const struct attribute_group *events,
1081
+			  const struct attribute_group *format,
1082
+			  const struct attribute_group *caps)
11071083 {
11081084 	int ret = armv8pmu_probe_pmu(cpu_pmu);
11091085 	if (ret)
11101086 		return ret;
11111087 
1112
-	cpu_pmu->handle_irq		= armv8pmu_handle_irq,
1113
-	cpu_pmu->enable			= armv8pmu_enable_event,
1114
-	cpu_pmu->disable		= armv8pmu_disable_event,
1115
-	cpu_pmu->read_counter		= armv8pmu_read_counter,
1116
-	cpu_pmu->write_counter		= armv8pmu_write_counter,
1117
-	cpu_pmu->get_event_idx		= armv8pmu_get_event_idx,
1118
-	cpu_pmu->clear_event_idx	= armv8pmu_clear_event_idx,
1119
-	cpu_pmu->start			= armv8pmu_start,
1120
-	cpu_pmu->stop			= armv8pmu_stop,
1121
-	cpu_pmu->reset			= armv8pmu_reset,
1088
+	cpu_pmu->handle_irq		= armv8pmu_handle_irq;
1089
+	cpu_pmu->enable			= armv8pmu_enable_event;
1090
+	cpu_pmu->disable		= armv8pmu_disable_event;
1091
+	cpu_pmu->read_counter		= armv8pmu_read_counter;
1092
+	cpu_pmu->write_counter		= armv8pmu_write_counter;
1093
+	cpu_pmu->get_event_idx		= armv8pmu_get_event_idx;
1094
+	cpu_pmu->clear_event_idx	= armv8pmu_clear_event_idx;
1095
+	cpu_pmu->start			= armv8pmu_start;
1096
+	cpu_pmu->stop			= armv8pmu_stop;
1097
+	cpu_pmu->reset			= armv8pmu_reset;
11221098 	cpu_pmu->set_event_filter	= armv8pmu_set_event_filter;
11231099 	cpu_pmu->filter_match		= armv8pmu_filter_match;
11241100 
1101
+	cpu_pmu->name			= name;
1102
+	cpu_pmu->map_event		= map_event;
1103
+	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = events ?
1104
+			events : &armv8_pmuv3_events_attr_group;
1105
+	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = format ?
1106
+			format : &armv8_pmuv3_format_attr_group;
1107
+	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = caps ?
1108
+			caps : &armv8_pmuv3_caps_attr_group;
1109
+
11251110 	return 0;
11261111 }
11271112 
1128
-static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu)
1113
+static int armv8_pmu_init_nogroups(struct arm_pmu *cpu_pmu, char *name,
1114
+				   int (*map_event)(struct perf_event *event))
11291115 {
1130
-	int ret = armv8_pmu_init(cpu_pmu);
1131
-	if (ret)
1132
-		return ret;
1133
-
1134
-	cpu_pmu->name			= "armv8_pmuv3";
1135
-	cpu_pmu->map_event		= armv8_pmuv3_map_event;
1136
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1137
-		&armv8_pmuv3_events_attr_group;
1138
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1139
-		&armv8_pmuv3_format_attr_group;
1140
-
1141
-	return 0;
1116
+	return armv8_pmu_init(cpu_pmu, name, map_event, NULL, NULL, NULL);
11421117 }
1118
+
1119
+#define PMUV3_INIT_SIMPLE(name)						\
1120
+static int name##_pmu_init(struct arm_pmu *cpu_pmu)			\
1121
+{									\
1122
+	return armv8_pmu_init_nogroups(cpu_pmu, #name, armv8_pmuv3_map_event);\
1123
+}
1124
+
1125
+PMUV3_INIT_SIMPLE(armv8_pmuv3)
1126
+
1127
+PMUV3_INIT_SIMPLE(armv8_cortex_a34)
1128
+PMUV3_INIT_SIMPLE(armv8_cortex_a55)
1129
+PMUV3_INIT_SIMPLE(armv8_cortex_a65)
1130
+PMUV3_INIT_SIMPLE(armv8_cortex_a75)
1131
+PMUV3_INIT_SIMPLE(armv8_cortex_a76)
1132
+PMUV3_INIT_SIMPLE(armv8_cortex_a77)
1133
+PMUV3_INIT_SIMPLE(armv8_cortex_a78)
1134
+PMUV3_INIT_SIMPLE(armv9_cortex_a510)
1135
+PMUV3_INIT_SIMPLE(armv9_cortex_a710)
1136
+PMUV3_INIT_SIMPLE(armv8_cortex_x1)
1137
+PMUV3_INIT_SIMPLE(armv9_cortex_x2)
1138
+PMUV3_INIT_SIMPLE(armv8_neoverse_e1)
1139
+PMUV3_INIT_SIMPLE(armv8_neoverse_n1)
1140
+PMUV3_INIT_SIMPLE(armv9_neoverse_n2)
1141
+PMUV3_INIT_SIMPLE(armv8_neoverse_v1)
1142
+
1143
+PMUV3_INIT_SIMPLE(armv8_nvidia_carmel)
1144
+PMUV3_INIT_SIMPLE(armv8_nvidia_denver)
11431145 
11441146 static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)
11451147 {
1146
-	int ret = armv8_pmu_init(cpu_pmu);
1147
-	if (ret)
1148
-		return ret;
1149
-
1150
-	cpu_pmu->name			= "armv8_cortex_a35";
1151
-	cpu_pmu->map_event		= armv8_a53_map_event;
1152
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1153
-		&armv8_pmuv3_events_attr_group;
1154
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1155
-		&armv8_pmuv3_format_attr_group;
1156
-
1157
-	return 0;
1148
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a35",
1149
+				       armv8_a53_map_event);
11581150 }
11591151 
11601152 static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
11611153 {
1162
-	int ret = armv8_pmu_init(cpu_pmu);
1163
-	if (ret)
1164
-		return ret;
1165
-
1166
-	cpu_pmu->name			= "armv8_cortex_a53";
1167
-	cpu_pmu->map_event		= armv8_a53_map_event;
1168
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1169
-		&armv8_pmuv3_events_attr_group;
1170
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1171
-		&armv8_pmuv3_format_attr_group;
1172
-
1173
-	return 0;
1154
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a53",
1155
+				       armv8_a53_map_event);
11741156 }
11751157 
11761158 static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
11771159 {
1178
-	int ret = armv8_pmu_init(cpu_pmu);
1179
-	if (ret)
1180
-		return ret;
1181
-
1182
-	cpu_pmu->name			= "armv8_cortex_a57";
1183
-	cpu_pmu->map_event		= armv8_a57_map_event;
1184
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1185
-		&armv8_pmuv3_events_attr_group;
1186
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1187
-		&armv8_pmuv3_format_attr_group;
1188
-
1189
-	return 0;
1160
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a57",
1161
+				       armv8_a57_map_event);
11901162 }
11911163 
11921164 static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)
11931165 {
1194
-	int ret = armv8_pmu_init(cpu_pmu);
1195
-	if (ret)
1196
-		return ret;
1197
-
1198
-	cpu_pmu->name			= "armv8_cortex_a72";
1199
-	cpu_pmu->map_event		= armv8_a57_map_event;
1200
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1201
-		&armv8_pmuv3_events_attr_group;
1202
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1203
-		&armv8_pmuv3_format_attr_group;
1204
-
1205
-	return 0;
1166
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a72",
1167
+				       armv8_a57_map_event);
12061168 }
12071169 
12081170 static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)
12091171 {
1210
-	int ret = armv8_pmu_init(cpu_pmu);
1211
-	if (ret)
1212
-		return ret;
1213
-
1214
-	cpu_pmu->name			= "armv8_cortex_a73";
1215
-	cpu_pmu->map_event		= armv8_a73_map_event;
1216
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1217
-		&armv8_pmuv3_events_attr_group;
1218
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1219
-		&armv8_pmuv3_format_attr_group;
1220
-
1221
-	return 0;
1172
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a73",
1173
+				       armv8_a73_map_event);
12221174 }
12231175 
12241176 static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
12251177 {
1226
-	int ret = armv8_pmu_init(cpu_pmu);
1227
-	if (ret)
1228
-		return ret;
1229
-
1230
-	cpu_pmu->name			= "armv8_cavium_thunder";
1231
-	cpu_pmu->map_event		= armv8_thunder_map_event;
1232
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1233
-		&armv8_pmuv3_events_attr_group;
1234
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1235
-		&armv8_pmuv3_format_attr_group;
1236
-
1237
-	return 0;
1178
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cavium_thunder",
1179
+				       armv8_thunder_map_event);
12381180 }
12391181 
12401182 static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)
12411183 {
1242
-	int ret = armv8_pmu_init(cpu_pmu);
1243
-	if (ret)
1244
-		return ret;
1245
-
1246
-	cpu_pmu->name			= "armv8_brcm_vulcan";
1247
-	cpu_pmu->map_event		= armv8_vulcan_map_event;
1248
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1249
-		&armv8_pmuv3_events_attr_group;
1250
-	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1251
-		&armv8_pmuv3_format_attr_group;
1252
-
1253
-	return 0;
1184
+	return armv8_pmu_init_nogroups(cpu_pmu, "armv8_brcm_vulcan",
1185
+				       armv8_vulcan_map_event);
12541186 }
12551187 
12561188 static const struct of_device_id armv8_pmu_of_device_ids[] = {
1257
-	{.compatible = "arm,armv8-pmuv3",	.data = armv8_pmuv3_init},
1189
+	{.compatible = "arm,armv8-pmuv3",	.data = armv8_pmuv3_pmu_init},
1190
+	{.compatible = "arm,cortex-a34-pmu",	.data = armv8_cortex_a34_pmu_init},
12581191 	{.compatible = "arm,cortex-a35-pmu",	.data = armv8_a35_pmu_init},
12591192 	{.compatible = "arm,cortex-a53-pmu",	.data = armv8_a53_pmu_init},
1193
+	{.compatible = "arm,cortex-a55-pmu",	.data = armv8_cortex_a55_pmu_init},
12601194 	{.compatible = "arm,cortex-a57-pmu",	.data = armv8_a57_pmu_init},
1195
+	{.compatible = "arm,cortex-a65-pmu",	.data = armv8_cortex_a65_pmu_init},
12611196 	{.compatible = "arm,cortex-a72-pmu",	.data = armv8_a72_pmu_init},
12621197 	{.compatible = "arm,cortex-a73-pmu",	.data = armv8_a73_pmu_init},
1198
+	{.compatible = "arm,cortex-a75-pmu",	.data = armv8_cortex_a75_pmu_init},
1199
+	{.compatible = "arm,cortex-a76-pmu",	.data = armv8_cortex_a76_pmu_init},
1200
+	{.compatible = "arm,cortex-a77-pmu",	.data = armv8_cortex_a77_pmu_init},
1201
+	{.compatible = "arm,cortex-a78-pmu",	.data = armv8_cortex_a78_pmu_init},
1202
+	{.compatible = "arm,cortex-a510-pmu",	.data = armv9_cortex_a510_pmu_init},
1203
+	{.compatible = "arm,cortex-a710-pmu",	.data = armv9_cortex_a710_pmu_init},
1204
+	{.compatible = "arm,cortex-x1-pmu",	.data = armv8_cortex_x1_pmu_init},
1205
+	{.compatible = "arm,cortex-x2-pmu",	.data = armv9_cortex_x2_pmu_init},
1206
+	{.compatible = "arm,neoverse-e1-pmu",	.data = armv8_neoverse_e1_pmu_init},
1207
+	{.compatible = "arm,neoverse-n1-pmu",	.data = armv8_neoverse_n1_pmu_init},
1208
+	{.compatible = "arm,neoverse-n2-pmu",	.data = armv9_neoverse_n2_pmu_init},
1209
+	{.compatible = "arm,neoverse-v1-pmu",	.data = armv8_neoverse_v1_pmu_init},
12631210 	{.compatible = "cavium,thunder-pmu",	.data = armv8_thunder_pmu_init},
12641211 	{.compatible = "brcm,vulcan-pmu",	.data = armv8_vulcan_pmu_init},
1212
+	{.compatible = "nvidia,carmel-pmu",	.data = armv8_nvidia_carmel_pmu_init},
1213
+	{.compatible = "nvidia,denver-pmu",	.data = armv8_nvidia_denver_pmu_init},
12651214 	{},
12661215 };
12671216 
....@@ -1284,35 +1233,61 @@
12841233 	if (acpi_disabled)
12851234 		return platform_driver_register(&armv8_pmu_driver);
12861235 	else
1287
-		return arm_pmu_acpi_probe(armv8_pmuv3_init);
1236
+		return arm_pmu_acpi_probe(armv8_pmuv3_pmu_init);
12881237 }
12891238 device_initcall(armv8_pmu_driver_init)
12901239 
12911240 void arch_perf_update_userpage(struct perf_event *event,
12921241 			       struct perf_event_mmap_page *userpg, u64 now)
12931242 {
1294
-	u32 freq;
1295
-	u32 shift;
1243
+	struct clock_read_data *rd;
1244
+	unsigned int seq;
1245
+	u64 ns;
12961246 
1297
-	/*
1298
-	 * Internal timekeeping for enabled/running/stopped times
1299
-	 * is always computed with the sched_clock.
1300
-	 */
1301
-	freq = arch_timer_get_rate();
1302
-	userpg->cap_user_time = 1;
1247
+	userpg->cap_user_time = 0;
1248
+	userpg->cap_user_time_zero = 0;
1249
+	userpg->cap_user_time_short = 0;
13031250 
1304
-	clocks_calc_mult_shift(&userpg->time_mult, &shift, freq,
1305
-			NSEC_PER_SEC, 0);
1251
+	do {
1252
+		rd = sched_clock_read_begin(&seq);
1253
+
1254
+		if (rd->read_sched_clock != arch_timer_read_counter)
1255
+			return;
1256
+
1257
+		userpg->time_mult = rd->mult;
1258
+		userpg->time_shift = rd->shift;
1259
+		userpg->time_zero = rd->epoch_ns;
1260
+		userpg->time_cycles = rd->epoch_cyc;
1261
+		userpg->time_mask = rd->sched_clock_mask;
1262
+
1263
+		/*
1264
+		 * Subtract the cycle base, such that software that
1265
+		 * doesn't know about cap_user_time_short still 'works'
1266
+		 * assuming no wraps.
1267
+		 */
1268
+		ns = mul_u64_u32_shr(rd->epoch_cyc, rd->mult, rd->shift);
1269
+		userpg->time_zero -= ns;
1270
+
1271
+	} while (sched_clock_read_retry(seq));
1272
+
1273
+	userpg->time_offset = userpg->time_zero - now;
1274
+
13061275 	/*
13071276 	 * time_shift is not expected to be greater than 31 due to
13081277 	 * the original published conversion algorithm shifting a
13091278 	 * 32-bit value (now specifies a 64-bit value) - refer
13101279 	 * perf_event_mmap_page documentation in perf_event.h.
13111280 	 */
1312
-	if (shift == 32) {
1313
-		shift = 31;
1281
+	if (userpg->time_shift == 32) {
1282
+		userpg->time_shift = 31;
13141283 		userpg->time_mult >>= 1;
13151284 	}
1316
-	userpg->time_shift = (u16)shift;
1317
-	userpg->time_offset = -now;
1285
+
1286
+	/*
1287
+	 * Internal timekeeping for enabled/running/stopped times
1288
+	 * is always computed with the sched_clock.
1289
+	 */
1290
+	userpg->cap_user_time = 1;
1291
+	userpg->cap_user_time_zero = 1;
1292
+	userpg->cap_user_time_short = 1;
13181293 }