forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-11 04dd17822334871b23ea2862f7798fb0e0007777 
 kernel/arch/powerpc/perf/core-book3s.c
....@@ -1,15 +1,12 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  * Performance event support - powerpc architecture code
34  *
45  * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
5
- *
6
- * This program is free software; you can redistribute it and/or
7
- * modify it under the terms of the GNU General Public License
8
- * as published by the Free Software Foundation; either version
9
- * 2 of the License, or (at your option) any later version.
106  */
117 #include <linux/kernel.h>
128 #include <linux/sched.h>
9
+#include <linux/sched/clock.h>
1310 #include <linux/perf_event.h>
1411 #include <linux/percpu.h>
1512 #include <linux/hardirq.h>
....@@ -20,6 +17,10 @@
2017 #include <asm/firmware.h>
2118 #include <asm/ptrace.h>
2219 #include <asm/code-patching.h>
20
+
21
+#ifdef CONFIG_PPC64
22
+#include "internal.h"
23
+#endif
2324 
2425 #define BHRB_MAX_ENTRIES	32
2526 #define BHRB_TARGET		0x0000000000000002
....@@ -36,12 +37,7 @@
3637 	struct perf_event *event[MAX_HWEVENTS];
3738 	u64 events[MAX_HWEVENTS];
3839 	unsigned int flags[MAX_HWEVENTS];
39
-	/*
40
-	 * The order of the MMCR array is:
41
-	 *  - 64-bit, MMCR0, MMCR1, MMCRA, MMCR2
42
-	 *  - 32-bit, MMCR0, MMCR1, MMCR2
43
-	 */
44
-	unsigned long mmcr[4];
40
+	struct mmcr_regs mmcr;
4541 	struct perf_event *limited_counter[MAX_LIMITED_HWCOUNTERS];
4642 	u8  limited_hwidx[MAX_LIMITED_HWCOUNTERS];
4743 	u64 alternatives[MAX_HWEVENTS][MAX_EVENT_ALTERNATIVES];
....@@ -76,6 +72,11 @@
7672 /*
7773  * 32-bit doesn't have MMCRA but does have an MMCR2,
7874  * and a few other names are different.
75
+ * Also 32-bit doesn't have MMCR3, SIER2 and SIER3.
76
+ * Define them as zero knowing that any code path accessing
77
+ * these registers (via mtspr/mfspr) are done under ppmu flag
78
+ * check for PPMU_ARCH_31 and we will not enter that code path
79
+ * for 32-bit.
7980  */
8081 #ifdef CONFIG_PPC32
8182 
....@@ -89,7 +90,12 @@
8990 #define MMCR0_PMCC_U6		0
9091 
9192 #define SPRN_MMCRA		SPRN_MMCR2
93
+#define SPRN_MMCR3		0
94
+#define SPRN_SIER2		0
95
+#define SPRN_SIER3		0
9296 #define MMCRA_SAMPLE_ENABLE	0
97
+#define MMCRA_BHRB_DISABLE     0
98
+#define MMCR0_PMCCEXT		0
9399 
94100 static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
95101 {
....@@ -104,10 +110,6 @@
104110 {
105111 	regs->result = 0;
106112 }
107
-static inline int perf_intr_is_nmi(struct pt_regs *regs)
108
-{
109
-	return 0;
110
-}
111113 
112114 static inline int siar_valid(struct pt_regs *regs)
113115 {
....@@ -120,7 +122,7 @@
120122 static void ebb_switch_out(unsigned long mmcr0) { }
121123 static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
122124 {
123
-	return cpuhw->mmcr[0];
125
+	return cpuhw->mmcr.mmcr0;
124126 }
125127 
126128 static inline void power_pmu_bhrb_enable(struct perf_event *event) {}
....@@ -129,6 +131,17 @@
129131 static inline void power_pmu_bhrb_read(struct perf_event *event, struct cpu_hw_events *cpuhw) {}
130132 static void pmao_restore_workaround(bool ebb) { }
131133 #endif /* CONFIG_PPC32 */
134
+
135
+bool is_sier_available(void)
136
+{
137
+	if (!ppmu)
138
+		return false;
139
+
140
+	if (ppmu->flags & PPMU_HAS_SIER)
141
+		return true;
142
+
143
+	return false;
144
+}
132145 
133146 static bool regs_use_siar(struct pt_regs *regs)
134147 {
....@@ -195,7 +208,7 @@
195208 	if (!(mmcra & MMCRA_SAMPLE_ENABLE) || sdar_valid)
196209 		*addrp = mfspr(SPRN_SDAR);
197210 
198
-	if (is_kernel_addr(mfspr(SPRN_SDAR)) && perf_allow_kernel(&event->attr) != 0)
211
+	if (is_kernel_addr(mfspr(SPRN_SDAR)) && event->attr.exclude_kernel)
199212 		*addrp = 0;
200213 }
201214 
....@@ -316,15 +329,6 @@
316329 }
317330 
318331 /*
319
- * If interrupts were soft-disabled when a PMU interrupt occurs, treat
320
- * it as an NMI.
321
- */
322
-static inline int perf_intr_is_nmi(struct pt_regs *regs)
323
-{
324
-	return (regs->softe & IRQS_DISABLED);
325
-}
326
-
327
-/*
328332  * On processors like P7+ that have the SIAR-Valid bit, marked instructions
329333  * must be sampled only if the SIAR-valid bit is set.
330334  *
....@@ -406,26 +410,22 @@
406410 static __u64 power_pmu_bhrb_to(u64 addr)
407411 {
408412 	unsigned int instr;
409
-	int ret;
410413 	__u64 target;
411414 
412415 	if (is_kernel_addr(addr)) {
413
-		if (probe_kernel_read(&instr, (void *)addr, sizeof(instr)))
416
+		if (copy_from_kernel_nofault(&instr, (void *)addr,
417
+				sizeof(instr)))
414418 			return 0;
415419 
416
-		return branch_target(&instr);
420
+		return branch_target((struct ppc_inst *)&instr);
417421 	}
418422 
419423 	/* Userspace: need copy instruction here then translate it */
420
-	pagefault_disable();
421
-	ret = __get_user_inatomic(instr, (unsigned int __user *)addr);
422
-	if (ret) {
423
-		pagefault_enable();
424
+	if (copy_from_user_nofault(&instr, (unsigned int __user *)addr,
425
+			sizeof(instr)))
424426 		return 0;
425
-	}
426
-	pagefault_enable();
427427 
428
-	target = branch_target(&instr);
428
+	target = branch_target((struct ppc_inst *)&instr);
429429 	if ((!target) || (instr & BRANCH_ABSOLUTE))
430430 		return target;
431431 
....@@ -461,8 +461,11 @@
461461 			 * addresses at this point. Check the privileges before
462462 			 * exporting it to userspace (avoid exposure of regions
463463 			 * where we could have speculative execution)
464
+			 * Incase of ISA v3.1, BHRB will capture only user-space
465
+			 * addresses, hence include a check before filtering code
464466 			 */
465
-			if (is_kernel_addr(addr) && perf_allow_kernel(&event->attr) != 0)
467
+			if (!(ppmu->flags & PPMU_ARCH_31) &&
468
+			    is_kernel_addr(addr) && event->attr.exclude_kernel)
466469 				continue;
467470 
468471 			/* Branches are read most recent first (ie. mfbhrb 0 is
....@@ -515,6 +518,7 @@
515518 		}
516519 	}
517520 	cpuhw->bhrb_stack.nr = u_index;
521
+	cpuhw->bhrb_stack.hw_idx = -1ULL;
518522 	return;
519523 }
520524 
....@@ -580,11 +584,16 @@
580584 	current->thread.sdar  = mfspr(SPRN_SDAR);
581585 	current->thread.mmcr0 = mmcr0 & MMCR0_USER_MASK;
582586 	current->thread.mmcr2 = mfspr(SPRN_MMCR2) & MMCR2_USER_MASK;
587
+	if (ppmu->flags & PPMU_ARCH_31) {
588
+		current->thread.mmcr3 = mfspr(SPRN_MMCR3);
589
+		current->thread.sier2 = mfspr(SPRN_SIER2);
590
+		current->thread.sier3 = mfspr(SPRN_SIER3);
591
+	}
583592 }
584593 
585594 static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw)
586595 {
587
-	unsigned long mmcr0 = cpuhw->mmcr[0];
596
+	unsigned long mmcr0 = cpuhw->mmcr.mmcr0;
588597 
589598 	if (!ebb)
590599 		goto out;
....@@ -618,7 +627,13 @@
618627 	 * unfreeze counters, it should not set exclude_xxx in its events and
619628 	 * instead manage the MMCR2 entirely by itself.
620629 	 */
621
-	mtspr(SPRN_MMCR2, cpuhw->mmcr[3] | current->thread.mmcr2);
630
+	mtspr(SPRN_MMCR2, cpuhw->mmcr.mmcr2 | current->thread.mmcr2);
631
+
632
+	if (ppmu->flags & PPMU_ARCH_31) {
633
+		mtspr(SPRN_MMCR3, current->thread.mmcr3);
634
+		mtspr(SPRN_SIER2, current->thread.sier2);
635
+		mtspr(SPRN_SIER3, current->thread.sier3);
636
+	}
622637 out:
623638 	return mmcr0;
624639 }
....@@ -790,6 +805,19 @@
790805 	}
791806 }
792807 
808
+static int any_pmc_overflown(struct cpu_hw_events *cpuhw)
809
+{
810
+	int i, idx;
811
+
812
+	for (i = 0; i < cpuhw->n_events; i++) {
813
+		idx = cpuhw->event[i]->hw.idx;
814
+		if ((idx) && ((int)read_pmc(idx) < 0))
815
+			return idx;
816
+	}
817
+
818
+	return 0;
819
+}
820
+
793821 /* Called from sysrq_handle_showregs() */
794822 void perf_event_print_debug(void)
795823 {
....@@ -839,6 +867,11 @@
839867 		pr_info("EBBRR: %016lx BESCR: %016lx\n",
840868 			mfspr(SPRN_EBBRR), mfspr(SPRN_BESCR));
841869 	}
870
+
871
+	if (ppmu->flags & PPMU_ARCH_31) {
872
+		pr_info("MMCR3: %016lx SIER2: %016lx SIER3: %016lx\n",
873
+			mfspr(SPRN_MMCR3), mfspr(SPRN_SIER2), mfspr(SPRN_SIER3));
874
+	}
842875 #endif
843876 	pr_info("SIAR:  %016lx SDAR:  %016lx SIER:  %016lx\n",
844877 		mfspr(SPRN_SIAR), sdar, sier);
....@@ -862,6 +895,8 @@
862895 	int i, j;
863896 	unsigned long addf = ppmu->add_fields;
864897 	unsigned long tadd = ppmu->test_adder;
898
+	unsigned long grp_mask = ppmu->group_constraint_mask;
899
+	unsigned long grp_val = ppmu->group_constraint_val;
865900 
866901 	if (n_ev > ppmu->n_counter)
867902 		return -1;
....@@ -882,15 +917,23 @@
882917 	for (i = 0; i < n_ev; ++i) {
883918 		nv = (value | cpuhw->avalues[i][0]) +
884919 			(value & cpuhw->avalues[i][0] & addf);
885
-		if ((((nv + tadd) ^ value) & mask) != 0 ||
886
-		    (((nv + tadd) ^ cpuhw->avalues[i][0]) &
887
-		     cpuhw->amasks[i][0]) != 0)
920
+
921
+		if (((((nv + tadd) ^ value) & mask) & (~grp_mask)) != 0)
888922 			break;
923
+
924
+		if (((((nv + tadd) ^ cpuhw->avalues[i][0]) & cpuhw->amasks[i][0])
925
+			& (~grp_mask)) != 0)
926
+			break;
927
+
889928 		value = nv;
890929 		mask |= cpuhw->amasks[i][0];
891930 	}
892
-	if (i == n_ev)
893
-		return 0;	/* all OK */
931
+	if (i == n_ev) {
932
+		if ((value & mask & grp_mask) != (mask & grp_val))
933
+			return -1;
934
+		else
935
+			return 0;	/* all OK */
936
+	}
894937 
895938 	/* doesn't work, gather alternatives... */
896939 	if (!ppmu->get_alternatives)
....@@ -1180,7 +1223,7 @@
11801223 static void power_pmu_disable(struct pmu *pmu)
11811224 {
11821225 	struct cpu_hw_events *cpuhw;
1183
-	unsigned long flags, mmcr0, val;
1226
+	unsigned long flags, mmcr0, val, mmcra;
11841227 
11851228 	if (!ppmu)
11861229 		return;
....@@ -1198,11 +1241,16 @@
11981241 
11991242 		/*
12001243 		 * Set the 'freeze counters' bit, clear EBE/BHRBA/PMCC/PMAO/FC56
1244
+		 * Also clear PMXE to disable PMI's getting triggered in some
1245
+		 * corner cases during PMU disable.
12011246 		 */
12021247 		val  = mmcr0 = mfspr(SPRN_MMCR0);
12031248 		val |= MMCR0_FC;
12041249 		val &= ~(MMCR0_EBE | MMCR0_BHRBA | MMCR0_PMCC | MMCR0_PMAO |
1205
-			 MMCR0_FC56);
1250
+			 MMCR0_PMXE | MMCR0_FC56);
1251
+		/* Set mmcr0 PMCCEXT for p10 */
1252
+		if (ppmu->flags & PPMU_ARCH_31)
1253
+			val |= MMCR0_PMCCEXT;
12061254 
12071255 		/*
12081256 		 * The barrier is to make sure the mtspr has been
....@@ -1214,11 +1262,46 @@
12141262 		isync();
12151263 
12161264 		/*
1265
+		 * Some corner cases could clear the PMU counter overflow
1266
+		 * while a masked PMI is pending. One such case is when
1267
+		 * a PMI happens during interrupt replay and perf counter
1268
+		 * values are cleared by PMU callbacks before replay.
1269
+		 *
1270
+		 * Disable the interrupt by clearing the paca bit for PMI
1271
+		 * since we are disabling the PMU now. Otherwise provide a
1272
+		 * warning if there is PMI pending, but no counter is found
1273
+		 * overflown.
1274
+		 *
1275
+		 * Since power_pmu_disable runs under local_irq_save, it
1276
+		 * could happen that code hits a PMC overflow without PMI
1277
+		 * pending in paca. Hence only clear PMI pending if it was
1278
+		 * set.
1279
+		 *
1280
+		 * If a PMI is pending, then MSR[EE] must be disabled (because
1281
+		 * the masked PMI handler disabling EE). So it is safe to
1282
+		 * call clear_pmi_irq_pending().
1283
+		 */
1284
+		if (pmi_irq_pending())
1285
+			clear_pmi_irq_pending();
1286
+
1287
+		val = mmcra = cpuhw->mmcr.mmcra;
1288
+
1289
+		/*
12171290 		 * Disable instruction sampling if it was enabled
12181291 		 */
1219
-		if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
1220
-			mtspr(SPRN_MMCRA,
1221
-			      cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1292
+		if (cpuhw->mmcr.mmcra & MMCRA_SAMPLE_ENABLE)
1293
+			val &= ~MMCRA_SAMPLE_ENABLE;
1294
+
1295
+		/* Disable BHRB via mmcra (BHRBRD) for p10 */
1296
+		if (ppmu->flags & PPMU_ARCH_31)
1297
+			val |= MMCRA_BHRB_DISABLE;
1298
+
1299
+		/*
1300
+		 * Write SPRN_MMCRA if mmcra has either disabled
1301
+		 * instruction sampling or BHRB.
1302
+		 */
1303
+		if (val != mmcra) {
1304
+			mtspr(SPRN_MMCRA, mmcra);
12221305 			mb();
12231306 			isync();
12241307 		}
....@@ -1292,18 +1375,29 @@
12921375 	 * (possibly updated for removal of events).
12931376 	 */
12941377 	if (!cpuhw->n_added) {
1295
-		mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1296
-		mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
1378
+		/*
1379
+		 * If there is any active event with an overflown PMC
1380
+		 * value, set back PACA_IRQ_PMI which would have been
1381
+		 * cleared in power_pmu_disable().
1382
+		 */
1383
+		hard_irq_disable();
1384
+		if (any_pmc_overflown(cpuhw))
1385
+			set_pmi_irq_pending();
1386
+
1387
+		mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra & ~MMCRA_SAMPLE_ENABLE);
1388
+		mtspr(SPRN_MMCR1, cpuhw->mmcr.mmcr1);
1389
+		if (ppmu->flags & PPMU_ARCH_31)
1390
+			mtspr(SPRN_MMCR3, cpuhw->mmcr.mmcr3);
12971391 		goto out_enable;
12981392 	}
12991393 
13001394 	/*
13011395 	 * Clear all MMCR settings and recompute them for the new set of events.
13021396 	 */
1303
-	memset(cpuhw->mmcr, 0, sizeof(cpuhw->mmcr));
1397
+	memset(&cpuhw->mmcr, 0, sizeof(cpuhw->mmcr));
13041398 
13051399 	if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index,
1306
-			       cpuhw->mmcr, cpuhw->event)) {
1400
+			       &cpuhw->mmcr, cpuhw->event)) {
13071401 		/* shouldn't ever get here */
13081402 		printk(KERN_ERR "oops compute_mmcr failed\n");
13091403 		goto out;
....@@ -1317,11 +1411,11 @@
13171411 		 */
13181412 		event = cpuhw->event[0];
13191413 		if (event->attr.exclude_user)
1320
-			cpuhw->mmcr[0] |= MMCR0_FCP;
1414
+			cpuhw->mmcr.mmcr0 |= MMCR0_FCP;
13211415 		if (event->attr.exclude_kernel)
1322
-			cpuhw->mmcr[0] |= freeze_events_kernel;
1416
+			cpuhw->mmcr.mmcr0 |= freeze_events_kernel;
13231417 		if (event->attr.exclude_hv)
1324
-			cpuhw->mmcr[0] |= MMCR0_FCHV;
1418
+			cpuhw->mmcr.mmcr0 |= MMCR0_FCHV;
13251419 	}
13261420 
13271421 	/*
....@@ -1330,12 +1424,15 @@
13301424 	 * Then unfreeze the events.
13311425 	 */
13321426 	ppc_set_pmu_inuse(1);
1333
-	mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
1334
-	mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
1335
-	mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
1427
+	mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra & ~MMCRA_SAMPLE_ENABLE);
1428
+	mtspr(SPRN_MMCR1, cpuhw->mmcr.mmcr1);
1429
+	mtspr(SPRN_MMCR0, (cpuhw->mmcr.mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
13361430 				| MMCR0_FC);
13371431 	if (ppmu->flags & PPMU_ARCH_207S)
1338
-		mtspr(SPRN_MMCR2, cpuhw->mmcr[3]);
1432
+		mtspr(SPRN_MMCR2, cpuhw->mmcr.mmcr2);
1433
+
1434
+	if (ppmu->flags & PPMU_ARCH_31)
1435
+		mtspr(SPRN_MMCR3, cpuhw->mmcr.mmcr3);
13391436 
13401437 	/*
13411438 	 * Read off any pre-existing events that need to move
....@@ -1386,7 +1483,7 @@
13861483 		perf_event_update_userpage(event);
13871484 	}
13881485 	cpuhw->n_limited = n_lim;
1389
-	cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
1486
+	cpuhw->mmcr.mmcr0 |= MMCR0_PMXE | MMCR0_FCECE;
13901487 
13911488  out_enable:
13921489 	pmao_restore_workaround(ebb);
....@@ -1402,9 +1499,9 @@
14021499 	/*
14031500 	 * Enable instruction sampling if necessary
14041501 	 */
1405
-	if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
1502
+	if (cpuhw->mmcr.mmcra & MMCRA_SAMPLE_ENABLE) {
14061503 		mb();
1407
-		mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
1504
+		mtspr(SPRN_MMCRA, cpuhw->mmcr.mmcra);
14081505 	}
14091506 
14101507  out:
....@@ -1501,9 +1598,16 @@
15011598 	ret = 0;
15021599  out:
15031600 	if (has_branch_stack(event)) {
1504
-		power_pmu_bhrb_enable(event);
1505
-		cpuhw->bhrb_filter = ppmu->bhrb_filter_map(
1506
-					event->attr.branch_sample_type);
1601
+		u64 bhrb_filter = -1;
1602
+
1603
+		if (ppmu->bhrb_filter_map)
1604
+			bhrb_filter = ppmu->bhrb_filter_map(
1605
+				event->attr.branch_sample_type);
1606
+
1607
+		if (bhrb_filter != -1) {
1608
+			cpuhw->bhrb_filter = bhrb_filter;
1609
+			power_pmu_bhrb_enable(event);
1610
+		}
15071611 	}
15081612 
15091613 	perf_pmu_enable(event->pmu);
....@@ -1534,7 +1638,7 @@
15341638 				cpuhw->flags[i-1] = cpuhw->flags[i];
15351639 			}
15361640 			--cpuhw->n_events;
1537
-			ppmu->disable_pmc(event->hw.idx - 1, cpuhw->mmcr);
1641
+			ppmu->disable_pmc(event->hw.idx - 1, &cpuhw->mmcr);
15381642 			if (event->hw.idx) {
15391643 				write_pmc(event->hw.idx, 0);
15401644 				event->hw.idx = 0;
....@@ -1555,7 +1659,7 @@
15551659 	}
15561660 	if (cpuhw->n_events == 0) {
15571661 		/* disable exceptions if no events are running */
1558
-		cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
1662
+		cpuhw->mmcr.mmcr0 &= ~(MMCR0_PMXE | MMCR0_FCECE);
15591663 	}
15601664 
15611665 	if (has_branch_stack(event))
....@@ -1779,7 +1883,7 @@
17791883 static int hw_perf_cache_event(u64 config, u64 *eventp)
17801884 {
17811885 	unsigned long type, op, result;
1782
-	int ev;
1886
+	u64 ev;
17831887 
17841888 	if (!ppmu->cache_events)
17851889 		return -EINVAL;
....@@ -1818,14 +1922,13 @@
18181922 static int power_pmu_event_init(struct perf_event *event)
18191923 {
18201924 	u64 ev;
1821
-	unsigned long flags;
1925
+	unsigned long flags, irq_flags;
18221926 	struct perf_event *ctrs[MAX_HWEVENTS];
18231927 	u64 events[MAX_HWEVENTS];
18241928 	unsigned int cflags[MAX_HWEVENTS];
18251929 	int n;
18261930 	int err;
18271931 	struct cpu_hw_events *cpuhw;
1828
-	u64 bhrb_filter;
18291932 
18301933 	if (!ppmu)
18311934 		return -ENOENT;
....@@ -1927,21 +2030,26 @@
19272030 	if (check_excludes(ctrs, cflags, n, 1))
19282031 		return -EINVAL;
19292032 
1930
-	cpuhw = &get_cpu_var(cpu_hw_events);
2033
+	local_irq_save(irq_flags);
2034
+	cpuhw = this_cpu_ptr(&cpu_hw_events);
2035
+
19312036 	err = power_check_constraints(cpuhw, events, cflags, n + 1);
19322037 
19332038 	if (has_branch_stack(event)) {
1934
-		bhrb_filter = ppmu->bhrb_filter_map(
2039
+		u64 bhrb_filter = -1;
2040
+
2041
+		if (ppmu->bhrb_filter_map)
2042
+			bhrb_filter = ppmu->bhrb_filter_map(
19352043 					event->attr.branch_sample_type);
19362044 
19372045 		if (bhrb_filter == -1) {
1938
-			put_cpu_var(cpu_hw_events);
2046
+			local_irq_restore(irq_flags);
19392047 			return -EOPNOTSUPP;
19402048 		}
19412049 		cpuhw->bhrb_filter = bhrb_filter;
19422050 	}
19432051 
1944
-	put_cpu_var(cpu_hw_events);
2052
+	local_irq_restore(irq_flags);
19452053 	if (err)
19462054 		return -EINVAL;
19472055 
....@@ -2173,26 +2281,19 @@
21732281 /*
21742282  * Performance monitor interrupt stuff
21752283  */
2176
-static void perf_event_interrupt(struct pt_regs *regs)
2284
+static void __perf_event_interrupt(struct pt_regs *regs)
21772285 {
21782286 	int i, j;
21792287 	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
21802288 	struct perf_event *event;
21812289 	unsigned long val[8];
21822290 	int found, active;
2183
-	int nmi;
21842291 
21852292 	if (cpuhw->n_limited)
21862293 		freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
21872294 					mfspr(SPRN_PMC6));
21882295 
21892296 	perf_read_regs(regs);
2190
-
2191
-	nmi = perf_intr_is_nmi(regs);
2192
-	if (nmi)
2193
-		nmi_enter();
2194
-	else
2195
-		irq_enter();
21962297 
21972298 	/* Read all the PMCs since we'll need them a bunch of times */
21982299 	for (i = 0; i < ppmu->n_counter; ++i)
....@@ -2220,6 +2321,14 @@
22202321 				break;
22212322 			}
22222323 		}
2324
+
2325
+		/*
2326
+		 * Clear PACA_IRQ_PMI in case it was set by
2327
+		 * set_pmi_irq_pending() when PMU was enabled
2328
+		 * after accounting for interrupts.
2329
+		 */
2330
+		clear_pmi_irq_pending();
2331
+
22232332 		if (!active)
22242333 			/* reset non active counters that have overflowed */
22252334 			write_pmc(i + 1, 0);
....@@ -2239,8 +2348,15 @@
22392348 			}
22402349 		}
22412350 	}
2242
-	if (!found && !nmi && printk_ratelimit())
2243
-		printk(KERN_WARNING "Can't find PMC that caused IRQ\n");
2351
+
2352
+	/*
2353
+	 * During system wide profling or while specific CPU is monitored for an
2354
+	 * event, some corner cases could cause PMC to overflow in idle path. This
2355
+	 * will trigger a PMI after waking up from idle. Since counter values are _not_
2356
+	 * saved/restored in idle path, can lead to below "Can't find PMC" message.
2357
+	 */
2358
+	if (unlikely(!found) && !arch_irq_disabled_regs(regs))
2359
+		printk_ratelimited(KERN_WARNING "Can't find PMC that caused IRQ\n");
22442360 
22452361 	/*
22462362 	 * Reset MMCR0 to its normal value.  This will set PMXE and
....@@ -2249,12 +2365,15 @@
22492365 	 * XXX might want to use MSR.PM to keep the events frozen until
22502366 	 * we get back out of this interrupt.
22512367 	 */
2252
-	write_mmcr0(cpuhw, cpuhw->mmcr[0]);
2368
+	write_mmcr0(cpuhw, cpuhw->mmcr.mmcr0);
2369
+}
22532370 
2254
-	if (nmi)
2255
-		nmi_exit();
2256
-	else
2257
-		irq_exit();
2371
+static void perf_event_interrupt(struct pt_regs *regs)
2372
+{
2373
+	u64 start_clock = sched_clock();
2374
+
2375
+	__perf_event_interrupt(regs);
2376
+	perf_sample_event_took(sched_clock() - start_clock);
22582377 }
22592378 
22602379 static int power_pmu_prepare_cpu(unsigned int cpu)
....@@ -2263,7 +2382,7 @@
22632382 
22642383 	if (ppmu) {
22652384 		memset(cpuhw, 0, sizeof(*cpuhw));
2266
-		cpuhw->mmcr[0] = MMCR0_FC;
2385
+		cpuhw->mmcr.mmcr0 = MMCR0_FC;
22672386 	}
22682387 	return 0;
22692388 }
....@@ -2278,6 +2397,7 @@
22782397 		pmu->name);
22792398 
22802399 	power_pmu.attr_groups = ppmu->attr_groups;
2400
+	power_pmu.capabilities |= (ppmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS);
22812401 
22822402 #ifdef MSR_HV
22832403 	/*
....@@ -2292,3 +2412,29 @@
22922412 			  power_pmu_prepare_cpu, NULL);
22932413 	return 0;
22942414 }
2415
+
2416
+#ifdef CONFIG_PPC64
2417
+static int __init init_ppc64_pmu(void)
2418
+{
2419
+	/* run through all the pmu drivers one at a time */
2420
+	if (!init_power5_pmu())
2421
+		return 0;
2422
+	else if (!init_power5p_pmu())
2423
+		return 0;
2424
+	else if (!init_power6_pmu())
2425
+		return 0;
2426
+	else if (!init_power7_pmu())
2427
+		return 0;
2428
+	else if (!init_power8_pmu())
2429
+		return 0;
2430
+	else if (!init_power9_pmu())
2431
+		return 0;
2432
+	else if (!init_power10_pmu())
2433
+		return 0;
2434
+	else if (!init_ppc970_pmu())
2435
+		return 0;
2436
+	else
2437
+		return init_generic_compat_pmu();
2438
+}
2439
+early_initcall(init_ppc64_pmu);
2440
+#endif